You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/09/16 02:01:27 UTC
(datafusion-comet) branch gh-readonly-queue/main/pr-5950-6a12c7c708789433cd1edb185c02cd9925a8dcb0 created (now 23f5b63b14)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a change to branch gh-readonly-queue/main/pr-5950-6a12c7c708789433cd1edb185c02cd9925a8dcb0
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
at 23f5b63b14 test: cover datetime and timezone routing configurations (#5950)
This branch includes the following new commits:
new 23f5b63b14 test: cover datetime and timezone routing configurations (#5950)
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org
(datafusion-comet) 01/01: test: cover datetime and timezone routing configurations (#5950)
Posted by gi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-5950-6a12c7c708789433cd1edb185c02cd9925a8dcb0
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
commit 23f5b63b14a09802a9c8b98858b84f320bd517fb
Author: KUAN-HAO HUANG <10...@users.noreply.github.com>
AuthorDate: Tue Sep 15 22:46:55 2026 +0000
test: cover datetime and timezone routing configurations (#5950)
* test: cover datetime expression routing configurations
* test: cover timezone and collated datetime routing
---
.../routing_datetime_collation_disabled.sql | 61 ++++++++++++++++++++
.../routing_datetime_collation_enabled.sql | 61 ++++++++++++++++++++
.../datetime/routing_temporal_disabled.sql | 66 +++++++++++++++++++++
.../datetime/routing_temporal_enabled.sql | 66 +++++++++++++++++++++
.../datetime/routing_temporal_opt_in.sql | 67 ++++++++++++++++++++++
.../datetime/routing_timezone_disabled.sql | 30 ++++++++++
.../datetime/routing_timezone_enabled.sql | 30 ++++++++++
.../datetime/routing_timezone_opt_in.sql | 30 ++++++++++
.../comet/CometTemporalExpressionSuite.scala | 31 ++++++----
9 files changed, 432 insertions(+), 10 deletions(-)
diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/routing_datetime_collation_disabled.sql b/spark/src/test/resources/sql-tests/expressions/datetime/routing_datetime_collation_disabled.sql
new file mode 100644
index 0000000000..9e6603d46b
--- /dev/null
+++ b/spark/src/test/resources/sql-tests/expressions/datetime/routing_datetime_collation_disabled.sql
@@ -0,0 +1,61 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied. See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- MinSparkVersion: 4.0
+-- Config: spark.comet.exec.scalaUDF.codegen.enabled=false
+-- Config: spark.sql.session.timeZone=UTC
+-- Config: spark.comet.expression.NextDay.allowIncompatible=false
+-- Config: spark.comet.expression.TruncDate.allowIncompatible=false
+-- Config: spark.comet.expression.TruncTimestamp.allowIncompatible=false
+-- Config: spark.comet.expression.DateFormatClass.allowIncompatible=false
+-- Config: spark.comet.expression.ConvertTimezone.allowIncompatible=false
+-- Config: spark.comet.expression.FromUnixTime.allowIncompatible=false
+-- Config: spark.comet.expression.MakeTimestamp.allowIncompatible=false
+-- Config: spark.comet.expression.ToUnixTimestamp.allowIncompatible=false
+
+statement
+CREATE TABLE routing_datetime_collation(ts TIMESTAMP, d DATE, fmt STRING, day STRING, zone STRING, str_ts STRING, seconds BIGINT) USING parquet
+
+statement
+INSERT INTO routing_datetime_collation VALUES (TIMESTAMP '2024-06-15 12:34:56', DATE '2024-06-15', 'YEAR', 'MON', 'UTC', '2024-06-15', 0), (NULL, NULL, NULL, NULL, NULL, NULL, NULL)
+
+query expect_native(next_day)
+SELECT next_day(d, day) FROM routing_datetime_collation
+
+query expect_fallback(next_day: spark.comet.exec.scalaUDF.codegen.enabled=false)
+SELECT next_day(d, day COLLATE UTF8_LCASE) FROM routing_datetime_collation
+
+query expect_fallback(trunc: spark.comet.exec.scalaUDF.codegen.enabled=false)
+SELECT trunc(d, fmt COLLATE UTF8_LCASE) FROM routing_datetime_collation
+
+query expect_fallback(date_trunc: spark.comet.exec.scalaUDF.codegen.enabled=false)
+SELECT date_trunc(fmt COLLATE UTF8_LCASE, ts) FROM routing_datetime_collation
+
+query expect_fallback(date_format: spark.comet.exec.scalaUDF.codegen.enabled=false)
+SELECT date_format(ts, 'yyyy-MM-dd' COLLATE UTF8_LCASE) FROM routing_datetime_collation
+
+query expect_fallback(from_unixtime: spark.comet.exec.scalaUDF.codegen.enabled=false)
+SELECT from_unixtime(seconds, 'yyyy-MM-dd' COLLATE UTF8_LCASE) FROM routing_datetime_collation
+
+query expect_fallback(convert_timezone: spark.comet.exec.scalaUDF.codegen.enabled=false)
+SELECT convert_timezone(zone COLLATE UTF8_LCASE, 'America/Los_Angeles', CAST(ts AS TIMESTAMP_NTZ)) FROM routing_datetime_collation
+
+query expect_fallback(make_timestamp: spark.comet.exec.scalaUDF.codegen.enabled=false)
+SELECT make_timestamp(2024, 6, 15, 12, 0, 0, zone COLLATE UTF8_LCASE) FROM routing_datetime_collation
+
+query expect_fallback(to_unix_timestamp: spark.comet.exec.scalaUDF.codegen.enabled=false)
+SELECT to_unix_timestamp(str_ts, 'yyyy-MM-dd' COLLATE UTF8_LCASE) FROM routing_datetime_collation
diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/routing_datetime_collation_enabled.sql b/spark/src/test/resources/sql-tests/expressions/datetime/routing_datetime_collation_enabled.sql
new file mode 100644
index 0000000000..ec4be718c3
--- /dev/null
+++ b/spark/src/test/resources/sql-tests/expressions/datetime/routing_datetime_collation_enabled.sql
@@ -0,0 +1,61 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied. See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- MinSparkVersion: 4.0
+-- Config: spark.comet.exec.scalaUDF.codegen.enabled=true
+-- Config: spark.sql.session.timeZone=UTC
+-- Config: spark.comet.expression.NextDay.allowIncompatible=false
+-- Config: spark.comet.expression.TruncDate.allowIncompatible=false
+-- Config: spark.comet.expression.TruncTimestamp.allowIncompatible=false
+-- Config: spark.comet.expression.DateFormatClass.allowIncompatible=false
+-- Config: spark.comet.expression.ConvertTimezone.allowIncompatible=false
+-- Config: spark.comet.expression.FromUnixTime.allowIncompatible=false
+-- Config: spark.comet.expression.MakeTimestamp.allowIncompatible=false
+-- Config: spark.comet.expression.ToUnixTimestamp.allowIncompatible=false
+
+statement
+CREATE TABLE routing_datetime_collation(ts TIMESTAMP, d DATE, fmt STRING, day STRING, zone STRING, str_ts STRING, seconds BIGINT) USING parquet
+
+statement
+INSERT INTO routing_datetime_collation VALUES (TIMESTAMP '2024-06-15 12:34:56', DATE '2024-06-15', 'YEAR', 'MON', 'UTC', '2024-06-15', 0), (NULL, NULL, NULL, NULL, NULL, NULL, NULL)
+
+query expect_native(next_day)
+SELECT next_day(d, day) FROM routing_datetime_collation
+
+query expect_dispatch(next_day)
+SELECT next_day(d, day COLLATE UTF8_LCASE) FROM routing_datetime_collation
+
+query expect_dispatch(trunc)
+SELECT trunc(d, fmt COLLATE UTF8_LCASE) FROM routing_datetime_collation
+
+query expect_dispatch(date_trunc)
+SELECT date_trunc(fmt COLLATE UTF8_LCASE, ts) FROM routing_datetime_collation
+
+query expect_dispatch(date_format)
+SELECT date_format(ts, 'yyyy-MM-dd' COLLATE UTF8_LCASE) FROM routing_datetime_collation
+
+query expect_dispatch(from_unixtime)
+SELECT from_unixtime(seconds, 'yyyy-MM-dd' COLLATE UTF8_LCASE) FROM routing_datetime_collation
+
+query expect_dispatch(convert_timezone)
+SELECT convert_timezone(zone COLLATE UTF8_LCASE, 'America/Los_Angeles', CAST(ts AS TIMESTAMP_NTZ)) FROM routing_datetime_collation
+
+query expect_dispatch(make_timestamp)
+SELECT make_timestamp(2024, 6, 15, 12, 0, 0, zone COLLATE UTF8_LCASE) FROM routing_datetime_collation
+
+query expect_dispatch(to_unix_timestamp)
+SELECT to_unix_timestamp(str_ts, 'yyyy-MM-dd' COLLATE UTF8_LCASE) FROM routing_datetime_collation
diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/routing_temporal_disabled.sql b/spark/src/test/resources/sql-tests/expressions/datetime/routing_temporal_disabled.sql
new file mode 100644
index 0000000000..003d531179
--- /dev/null
+++ b/spark/src/test/resources/sql-tests/expressions/datetime/routing_temporal_disabled.sql
@@ -0,0 +1,66 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied. See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- Config: spark.comet.exec.scalaUDF.codegen.enabled=false
+-- Config: spark.sql.session.timeZone=UTC
+-- Config: spark.comet.expression.FromUTCTimestamp.allowIncompatible=false
+-- Config: spark.comet.expression.ToUTCTimestamp.allowIncompatible=false
+-- Config: spark.comet.expression.FromUnixTime.allowIncompatible=false
+-- Config: spark.comet.expression.TruncDate.allowIncompatible=false
+-- Config: spark.comet.expression.TruncTimestamp.allowIncompatible=false
+
+statement
+CREATE TABLE routing_temporal(ts TIMESTAMP, d DATE, fmt STRING, seconds BIGINT) USING parquet
+
+statement
+INSERT INTO routing_temporal VALUES (TIMESTAMP '2024-06-15 12:34:56', DATE '2024-06-15', 'YEAR', 0), (NULL, NULL, NULL, NULL)
+
+query expect_native(trunc)
+SELECT trunc(d, 'YEAR') FROM routing_temporal
+
+query expect_native(date_trunc)
+SELECT date_trunc('YEAR', ts) FROM routing_temporal
+
+query expect_native(date_format)
+SELECT date_format(ts, 'yyyy-MM-dd') FROM routing_temporal
+
+query expect_fallback(from_utc_timestamp: spark.comet.exec.scalaUDF.codegen.enabled=false)
+SELECT from_utc_timestamp(ts, 'UTC') FROM routing_temporal
+
+query expect_fallback(to_utc_timestamp: spark.comet.exec.scalaUDF.codegen.enabled=false)
+SELECT to_utc_timestamp(ts, 'UTC') FROM routing_temporal
+
+query expect_fallback(from_unixtime: spark.comet.exec.scalaUDF.codegen.enabled=false)
+SELECT from_unixtime(seconds) FROM routing_temporal
+
+query expect_fallback(from_unixtime: spark.comet.exec.scalaUDF.codegen.enabled=false)
+SELECT from_unixtime(seconds, 'yyyy') FROM routing_temporal
+
+query expect_fallback(trunc: spark.comet.exec.scalaUDF.codegen.enabled=false)
+SELECT trunc(d, fmt) FROM routing_temporal
+
+query expect_fallback(trunc: spark.comet.exec.scalaUDF.codegen.enabled=false)
+SELECT trunc(d, 'day') FROM routing_temporal
+
+query expect_fallback(date_trunc: spark.comet.exec.scalaUDF.codegen.enabled=false)
+SELECT date_trunc(fmt, ts) FROM routing_temporal
+
+query expect_fallback(date_trunc: spark.comet.exec.scalaUDF.codegen.enabled=false)
+SELECT date_trunc('not_a_unit', ts) FROM routing_temporal
+
+query expect_fallback(date_format: spark.comet.exec.scalaUDF.codegen.enabled=false)
+SELECT date_format(ts, 'yyyy-MM-dd EEEE') FROM routing_temporal
diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/routing_temporal_enabled.sql b/spark/src/test/resources/sql-tests/expressions/datetime/routing_temporal_enabled.sql
new file mode 100644
index 0000000000..00e8b7ffa5
--- /dev/null
+++ b/spark/src/test/resources/sql-tests/expressions/datetime/routing_temporal_enabled.sql
@@ -0,0 +1,66 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied. See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- Config: spark.comet.exec.scalaUDF.codegen.enabled=true
+-- Config: spark.sql.session.timeZone=UTC
+-- Config: spark.comet.expression.FromUTCTimestamp.allowIncompatible=false
+-- Config: spark.comet.expression.ToUTCTimestamp.allowIncompatible=false
+-- Config: spark.comet.expression.FromUnixTime.allowIncompatible=false
+-- Config: spark.comet.expression.TruncDate.allowIncompatible=false
+-- Config: spark.comet.expression.TruncTimestamp.allowIncompatible=false
+
+statement
+CREATE TABLE routing_temporal(ts TIMESTAMP, d DATE, fmt STRING, seconds BIGINT) USING parquet
+
+statement
+INSERT INTO routing_temporal VALUES (TIMESTAMP '2024-06-15 12:34:56', DATE '2024-06-15', 'YEAR', 0), (NULL, NULL, NULL, NULL)
+
+query expect_native(trunc)
+SELECT trunc(d, 'YEAR') FROM routing_temporal
+
+query expect_native(date_trunc)
+SELECT date_trunc('YEAR', ts) FROM routing_temporal
+
+query expect_native(date_format)
+SELECT date_format(ts, 'yyyy-MM-dd') FROM routing_temporal
+
+query expect_dispatch(from_utc_timestamp)
+SELECT from_utc_timestamp(ts, 'UTC') FROM routing_temporal
+
+query expect_dispatch(to_utc_timestamp)
+SELECT to_utc_timestamp(ts, 'UTC') FROM routing_temporal
+
+query expect_dispatch(from_unixtime)
+SELECT from_unixtime(seconds) FROM routing_temporal
+
+query expect_dispatch(from_unixtime)
+SELECT from_unixtime(seconds, 'yyyy') FROM routing_temporal
+
+query expect_dispatch(trunc)
+SELECT trunc(d, fmt) FROM routing_temporal
+
+query expect_dispatch(trunc)
+SELECT trunc(d, 'day') FROM routing_temporal
+
+query expect_dispatch(date_trunc)
+SELECT date_trunc(fmt, ts) FROM routing_temporal
+
+query expect_dispatch(date_trunc)
+SELECT date_trunc('not_a_unit', ts) FROM routing_temporal
+
+query expect_dispatch(date_format)
+SELECT date_format(ts, 'yyyy-MM-dd EEEE') FROM routing_temporal
diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/routing_temporal_opt_in.sql b/spark/src/test/resources/sql-tests/expressions/datetime/routing_temporal_opt_in.sql
new file mode 100644
index 0000000000..d006b417b0
--- /dev/null
+++ b/spark/src/test/resources/sql-tests/expressions/datetime/routing_temporal_opt_in.sql
@@ -0,0 +1,67 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied. See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- Config: spark.comet.exec.scalaUDF.codegen.enabled=false
+-- Config: spark.sql.session.timeZone=UTC
+-- Config: spark.comet.expression.FromUTCTimestamp.allowIncompatible=true
+-- Config: spark.comet.expression.ToUTCTimestamp.allowIncompatible=true
+-- Config: spark.comet.expression.FromUnixTime.allowIncompatible=true
+-- Config: spark.comet.expression.TruncDate.allowIncompatible=true
+-- Config: spark.comet.expression.TruncTimestamp.allowIncompatible=true
+
+statement
+CREATE TABLE routing_temporal(ts TIMESTAMP, d DATE, fmt STRING, seconds BIGINT) USING parquet
+
+statement
+INSERT INTO routing_temporal VALUES (TIMESTAMP '2024-06-15 12:34:56', DATE '2024-06-15', 'YEAR', 0), (NULL, NULL, NULL, NULL)
+
+query expect_native(trunc)
+SELECT trunc(d, 'YEAR') FROM routing_temporal
+
+query expect_native(date_trunc)
+SELECT date_trunc('YEAR', ts) FROM routing_temporal
+
+query expect_native(date_format)
+SELECT date_format(ts, 'yyyy-MM-dd') FROM routing_temporal
+
+query expect_native(from_utc_timestamp)
+SELECT from_utc_timestamp(ts, 'UTC') FROM routing_temporal
+
+query expect_native(to_utc_timestamp)
+SELECT to_utc_timestamp(ts, 'UTC') FROM routing_temporal
+
+query expect_native(from_unixtime)
+SELECT from_unixtime(seconds) FROM routing_temporal
+
+query expect_fallback(from_unixtime: spark.comet.exec.scalaUDF.codegen.enabled=false)
+SELECT from_unixtime(seconds, 'yyyy') FROM routing_temporal
+
+-- Use non-null formats for native opt-in; the dispatcher cases retain null-format coverage.
+query expect_native(trunc)
+SELECT trunc(d, fmt) FROM routing_temporal WHERE fmt IS NOT NULL
+
+query expect_fallback(trunc: spark.comet.exec.scalaUDF.codegen.enabled=false)
+SELECT trunc(d, 'day') FROM routing_temporal
+
+query expect_native(date_trunc)
+SELECT date_trunc(fmt, ts) FROM routing_temporal WHERE fmt IS NOT NULL
+
+query expect_fallback(date_trunc: spark.comet.exec.scalaUDF.codegen.enabled=false)
+SELECT date_trunc('not_a_unit', ts) FROM routing_temporal
+
+query expect_fallback(date_format: spark.comet.exec.scalaUDF.codegen.enabled=false)
+SELECT date_format(ts, 'yyyy-MM-dd EEEE') FROM routing_temporal
diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/routing_timezone_disabled.sql b/spark/src/test/resources/sql-tests/expressions/datetime/routing_timezone_disabled.sql
new file mode 100644
index 0000000000..caa4d87ce6
--- /dev/null
+++ b/spark/src/test/resources/sql-tests/expressions/datetime/routing_timezone_disabled.sql
@@ -0,0 +1,30 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied. See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- MinSparkVersion: 3.5
+-- Config: spark.comet.exec.scalaUDF.codegen.enabled=false
+-- Config: spark.sql.session.timeZone=UTC
+-- Config: spark.comet.expression.ConvertTimezone.allowIncompatible=false
+
+statement
+CREATE TABLE routing_timezone(ts TIMESTAMP_NTZ) USING parquet
+
+statement
+INSERT INTO routing_timezone VALUES (TIMESTAMP_NTZ '2024-06-15 12:34:56'), (NULL)
+
+query expect_fallback(convert_timezone: spark.comet.exec.scalaUDF.codegen.enabled=false)
+SELECT convert_timezone('UTC', 'America/Los_Angeles', ts) FROM routing_timezone
diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/routing_timezone_enabled.sql b/spark/src/test/resources/sql-tests/expressions/datetime/routing_timezone_enabled.sql
new file mode 100644
index 0000000000..ed0ed79f21
--- /dev/null
+++ b/spark/src/test/resources/sql-tests/expressions/datetime/routing_timezone_enabled.sql
@@ -0,0 +1,30 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied. See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- MinSparkVersion: 3.5
+-- Config: spark.comet.exec.scalaUDF.codegen.enabled=true
+-- Config: spark.sql.session.timeZone=UTC
+-- Config: spark.comet.expression.ConvertTimezone.allowIncompatible=false
+
+statement
+CREATE TABLE routing_timezone(ts TIMESTAMP_NTZ) USING parquet
+
+statement
+INSERT INTO routing_timezone VALUES (TIMESTAMP_NTZ '2024-06-15 12:34:56'), (NULL)
+
+query expect_dispatch(convert_timezone)
+SELECT convert_timezone('UTC', 'America/Los_Angeles', ts) FROM routing_timezone
diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/routing_timezone_opt_in.sql b/spark/src/test/resources/sql-tests/expressions/datetime/routing_timezone_opt_in.sql
new file mode 100644
index 0000000000..f61c34ddcb
--- /dev/null
+++ b/spark/src/test/resources/sql-tests/expressions/datetime/routing_timezone_opt_in.sql
@@ -0,0 +1,30 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied. See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- MinSparkVersion: 3.5
+-- Config: spark.comet.exec.scalaUDF.codegen.enabled=false
+-- Config: spark.sql.session.timeZone=UTC
+-- Config: spark.comet.expression.ConvertTimezone.allowIncompatible=true
+
+statement
+CREATE TABLE routing_timezone(ts TIMESTAMP_NTZ) USING parquet
+
+statement
+INSERT INTO routing_timezone VALUES (TIMESTAMP_NTZ '2024-06-15 12:34:56'), (NULL)
+
+query expect_native(convert_timezone)
+SELECT convert_timezone('UTC', 'America/Los_Angeles', ts) FROM routing_timezone
diff --git a/spark/src/test/scala/org/apache/comet/CometTemporalExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometTemporalExpressionSuite.scala
index fce36496aa..9b2b73f97e 100644
--- a/spark/src/test/scala/org/apache/comet/CometTemporalExpressionSuite.scala
+++ b/spark/src/test/scala/org/apache/comet/CometTemporalExpressionSuite.scala
@@ -176,8 +176,10 @@ class CometTemporalExpressionSuite extends CometTestBase with AdaptiveSparkPlanH
SQLConf.SESSION_LOCAL_TIMEZONE.key -> tz,
CometConf.COMET_SCALA_UDF_CODEGEN_ENABLED.key -> "true") {
for (format <- CometTruncTimestamp.supportedFormats) {
- checkSparkAnswerAndOperator(
- s"SELECT c0, date_trunc('$format', c0) from tbl order by c0")
+ checkSparkAnswerAndImpl(
+ s"SELECT c0, date_trunc('$format', c0) from tbl order by c0",
+ native = Seq.empty,
+ dispatched = Seq("date_trunc"))
}
}
}
@@ -213,10 +215,13 @@ class CometTemporalExpressionSuite extends CometTestBase with AdaptiveSparkPlanH
for (tz <- nonUtcTimezones) {
withSQLConf(
SQLConf.SESSION_LOCAL_TIMEZONE.key -> tz,
- "spark.comet.expression.TruncTimestamp.allowIncompatible" -> "true") {
+ "spark.comet.expression.TruncTimestamp.allowIncompatible" -> "true",
+ CometConf.COMET_SCALA_UDF_CODEGEN_ENABLED.key -> "false") {
for (format <- CometTruncTimestamp.supportedFormats) {
- checkSparkAnswerAndOperator(
- s"SELECT c0, date_trunc('$format', c0) from tbl order by c0")
+ checkSparkAnswerAndImpl(
+ s"SELECT c0, date_trunc('$format', c0) from tbl order by c0",
+ native = Seq("date_trunc"),
+ dispatched = Seq.empty)
}
}
}
@@ -602,8 +607,10 @@ class CometTemporalExpressionSuite extends CometTestBase with AdaptiveSparkPlanH
withSQLConf(
SQLConf.SESSION_LOCAL_TIMEZONE.key -> "UTC",
CometConf.COMET_SCALA_UDF_CODEGEN_ENABLED.key -> "true") {
- checkSparkAnswerAndOperator(
- "SELECT c0, date_format(c0, 'yyyy-MM-dd EEEE') from tbl order by c0")
+ checkSparkAnswerAndImpl(
+ "SELECT c0, date_format(c0, 'yyyy-MM-dd EEEE') from tbl order by c0",
+ native = Seq.empty,
+ dispatched = Seq("date_format"))
}
}
@@ -629,8 +636,10 @@ class CometTemporalExpressionSuite extends CometTestBase with AdaptiveSparkPlanH
withSQLConf(
SQLConf.SESSION_LOCAL_TIMEZONE.key -> tz,
CometConf.COMET_SCALA_UDF_CODEGEN_ENABLED.key -> "true") {
- checkSparkAnswerAndOperator(
- "SELECT c0, date_format(c0, 'yyyy-MM-dd HH:mm:ss') from tbl order by c0")
+ checkSparkAnswerAndImpl(
+ "SELECT c0, date_format(c0, 'yyyy-MM-dd HH:mm:ss') from tbl order by c0",
+ native = Seq.empty,
+ dispatched = Seq("date_format"))
}
}
}
@@ -659,13 +668,15 @@ class CometTemporalExpressionSuite extends CometTestBase with AdaptiveSparkPlanH
for (tz <- nonUtcTimezones) {
withSQLConf(
SQLConf.SESSION_LOCAL_TIMEZONE.key -> tz,
- "spark.comet.expression.DateFormatClass.allowIncompatible" -> "true") {
+ "spark.comet.expression.DateFormatClass.allowIncompatible" -> "true",
+ CometConf.COMET_SCALA_UDF_CODEGEN_ENABLED.key -> "false") {
// Native to_char results may diverge from Spark for non-UTC timezones (the reason the
// JVM UDF is the default), so we only check that execution stays inside Comet. ORDER BY
// is omitted to keep the plan free of AQEShuffleRead.
val df = sql("SELECT c0, date_format(c0, 'yyyy-MM-dd') from tbl")
df.collect()
checkCometOperators(stripAQEPlan(df.queryExecution.executedPlan))
+ assertExpressionImpl(df.queryExecution.executedPlan, Seq("date_format"), Seq.empty)
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org