You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/09/29 01:01:55 UTC
(datafusion-comet) branch gh-readonly-queue/main/pr-4901-8c3eb7b2d84394f282cdbb9839e180b048a42ed1 created (now 61810e2839)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a change to branch gh-readonly-queue/main/pr-4901-8c3eb7b2d84394f282cdbb9839e180b048a42ed1
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
at 61810e2839 feat: support divide_dt_interval with codegen dispatch (#4901)
This branch includes the following new commits:
new 61810e2839 feat: support divide_dt_interval with codegen dispatch (#4901)
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org
(datafusion-comet) 01/01: feat: support divide_dt_interval with codegen dispatch (#4901)
Posted by gi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-4901-8c3eb7b2d84394f282cdbb9839e180b048a42ed1
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
commit 61810e2839d54241336be44de8765370e183a5e2
Author: Peter Lee <pe...@gmail.com>
AuthorDate: Mon Sep 28 20:26:36 2026 +0000
feat: support divide_dt_interval with codegen dispatch (#4901)
* feat: support divide_dt_interval
* revert accidently remove code
* address review: fix serde map ordering, add wide decimal divisor coverage
- Move DivideDTInterval after Days to keep the temporalExpressions map's
alphabetical prefix intact.
- Cover DECIMAL(38, 18) and DECIMAL(38, 0) divisors in the SQL fixture,
both as columns (DecimalVector slow path for precision > 18) and as
literals. No fallback: the codegen dispatcher supports all decimal
precisions.
Co-Authored-By: Claude Fable 5 <no...@anthropic.com>
---------
Co-authored-by: Claude Fable 5 <no...@anthropic.com>
---
docs/source/user-guide/latest/expressions.md | 2 +-
.../org/apache/comet/serde/QueryPlanSerde.scala | 1 +
.../scala/org/apache/comet/serde/datetime.scala | 4 +-
.../expressions/datetime/divide_dt_interval.sql | 75 ++++++++++++++++++++++
4 files changed, 80 insertions(+), 2 deletions(-)
diff --git a/docs/source/user-guide/latest/expressions.md b/docs/source/user-guide/latest/expressions.md
index 6db1db3cd6..28d4577e4e 100644
--- a/docs/source/user-guide/latest/expressions.md
+++ b/docs/source/user-guide/latest/expressions.md
@@ -425,7 +425,7 @@ to Spark ([#2837](https://github.com/apache/datafusion-comet/issues/2837)). Enab
| `*` | ✅ | Native | DayTime interval multiplication routes through the JVM codegen dispatcher; YearMonth and Calendar interval multiplication fall back |
| `+` | ✅ | Native | |
| `-` | ✅ | Native | |
-| `/` | ✅ | Native | |
+| `/` | ✅ | Native | DayTime interval division routes through the JVM codegen dispatcher; YearMonth and Calendar interval division fall back |
| `abs` | ✅ | Hybrid | Interval types route through the JVM codegen dispatcher; numeric types run natively |
| `acos` | ✅ | Native | |
| `acosh` | ✅ | Native | |
diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala
index 1e75f172b4..1e43162bc6 100644
--- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala
+++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala
@@ -287,6 +287,7 @@ object QueryPlanSerde extends Logging with CometExprShim with CometTypeShim {
classOf[DateFormatClass] -> CometDateFormat,
classOf[DateFromUnixDate] -> CometDateFromUnixDate,
classOf[Days] -> CometDays,
+ classOf[DivideDTInterval] -> CometDivideDTInterval,
classOf[Hours] -> CometHours,
classOf[DateSub] -> CometDateSub,
classOf[UnixDate] -> CometUnixDate,
diff --git a/spark/src/main/scala/org/apache/comet/serde/datetime.scala b/spark/src/main/scala/org/apache/comet/serde/datetime.scala
index 5106e0059a..415f10e868 100644
--- a/spark/src/main/scala/org/apache/comet/serde/datetime.scala
+++ b/spark/src/main/scala/org/apache/comet/serde/datetime.scala
@@ -21,7 +21,7 @@ package org.apache.comet.serde
import java.util.Locale
-import org.apache.spark.sql.catalyst.expressions.{AddMonths, Attribute, Cast, ConvertTimezone, DateAdd, DateDiff, DateFormatClass, DateFromUnixDate, DateSub, DayOfMonth, DayOfWeek, DayOfYear, Days, Expression, FromUTCTimestamp, GetDateField, GetTimestamp, Hour, Hours, LastDay, Literal, MakeDate, MakeDTInterval, MakeInterval, MakeTimestamp, MakeYMInterval, MicrosToTimestamp, MillisToTimestamp, Minute, Month, MonthsBetween, MultiplyDTInterval, NextDay, PreciseTimestampConversion, Quarter, [...]
+import org.apache.spark.sql.catalyst.expressions.{AddMonths, Attribute, Cast, ConvertTimezone, DateAdd, DateDiff, DateFormatClass, DateFromUnixDate, DateSub, DayOfMonth, DayOfWeek, DayOfYear, Days, DivideDTInterval, Expression, FromUTCTimestamp, GetDateField, GetTimestamp, Hour, Hours, LastDay, Literal, MakeDate, MakeDTInterval, MakeInterval, MakeTimestamp, MakeYMInterval, MicrosToTimestamp, MillisToTimestamp, Minute, Month, MonthsBetween, MultiplyDTInterval, NextDay, PreciseTimestampCon [...]
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types.{CalendarIntervalType, DataType, DateType, DoubleType, FloatType, IntegerType, LongType, StringType, TimestampNTZType, TimestampType}
import org.apache.spark.unsafe.types.UTF8String
@@ -894,6 +894,8 @@ object CometMakeYMInterval extends CometCodegenDispatch[MakeYMInterval]
object CometMakeDTInterval extends CometCodegenDispatch[MakeDTInterval]
+object CometDivideDTInterval extends CometCodegenDispatch[DivideDTInterval]
+
object CometMakeInterval extends CometExpressionSerde[MakeInterval] with CodegenDispatchFallback {
private val incompatReason =
"The native implementation converts seconds to `Float64`, which can lose microsecond" +
diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/divide_dt_interval.sql b/spark/src/test/resources/sql-tests/expressions/datetime/divide_dt_interval.sql
new file mode 100644
index 0000000000..01bfdbb80d
--- /dev/null
+++ b/spark/src/test/resources/sql-tests/expressions/datetime/divide_dt_interval.sql
@@ -0,0 +1,75 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied. See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- Routes divide_dt_interval through the codegen dispatcher; produces DayTimeIntervalType.
+-- Config: spark.comet.exec.scalaUDF.codegen.enabled=true
+
+statement
+CREATE TABLE test_divide_dt_interval(days int, hours int, minutes int, seconds decimal(18,6), b tinyint, s smallint, i int, l long, f float, d double, dec decimal(10,2), dec_wide decimal(38,18), dec_big decimal(38,0)) USING parquet
+
+statement
+INSERT INTO test_divide_dt_interval VALUES
+ (1, 2, 3, 4.500000, CAST(2 AS TINYINT), CAST(3 AS SMALLINT), 2, CAST(3 AS BIGINT), CAST(1.5 AS FLOAT), CAST(2.5 AS DOUBLE), CAST(2.50 AS DECIMAL(10, 2)), CAST(2.5 AS DECIMAL(38, 18)), CAST(3 AS DECIMAL(38, 0))),
+ (-1, 0, 30, 15.250000, CAST(-2 AS TINYINT), CAST(-3 AS SMALLINT), -2, CAST(-3 AS BIGINT), CAST(-1.5 AS FLOAT), CAST(-2.5 AS DOUBLE), CAST(-2.50 AS DECIMAL(10, 2)), CAST(-2.5 AS DECIMAL(38, 18)), CAST(-3 AS DECIMAL(38, 0))),
+ (0, 0, 0, 0.000001, CAST(2 AS TINYINT), CAST(2 AS SMALLINT), 2, CAST(2 AS BIGINT), CAST(2.0 AS FLOAT), CAST(2.0 AS DOUBLE), CAST(2.00 AS DECIMAL(10, 2)), CAST(2.0 AS DECIMAL(38, 18)), CAST(2 AS DECIMAL(38, 0))),
+ (2, -6, 0, 0.000000, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL)
+
+query
+SELECT
+ make_dt_interval(days, hours, minutes, seconds) / b,
+ make_dt_interval(days, hours, minutes, seconds) / s,
+ make_dt_interval(days, hours, minutes, seconds) / i,
+ make_dt_interval(days, hours, minutes, seconds) / l,
+ make_dt_interval(days, hours, minutes, seconds) / f,
+ make_dt_interval(days, hours, minutes, seconds) / d,
+ make_dt_interval(days, hours, minutes, seconds) / dec
+FROM test_divide_dt_interval
+
+-- Wide decimal divisors: DECIMAL(38, 18) reads through the DecimalVector slow path
+-- (precision > 18 does not fit an unscaled long) and DECIMAL(38, 0) covers scale 0.
+query
+SELECT
+ make_dt_interval(days, hours, minutes, seconds) / dec_wide,
+ make_dt_interval(days, hours, minutes, seconds) / dec_big
+FROM test_divide_dt_interval
+
+-- literal interval input
+query
+SELECT INTERVAL '1 02:03:04.500000' DAY TO SECOND / i FROM test_divide_dt_interval
+
+-- literal divisors, including half-up rounding to the nearest microsecond.
+query
+SELECT
+ make_dt_interval(1, 2, 3, 4.5) / 2,
+ INTERVAL '0.000001' SECOND / 2,
+ INTERVAL '0.000001' SECOND / CAST(2.00 AS DECIMAL(10, 2)),
+ INTERVAL '1 02:03:04.500000' DAY TO SECOND / CAST(2.5 AS DECIMAL(38, 18)),
+ INTERVAL '1 02:03:04.500000' DAY TO SECOND / CAST(3 AS DECIMAL(38, 0)),
+ make_dt_interval(-1, 0, 30, 15.25) / 1.5D
+
+-- null interval input
+query
+SELECT make_dt_interval(NULL, hours, minutes, seconds) / 2
+FROM test_divide_dt_interval
+
+-- Division by zero fails regardless of ANSI mode.
+query expect_error(zero)
+SELECT make_dt_interval(1) / 0
+
+-- This interval is Long.MinValue microseconds, which cannot be divided by -1.
+query expect_error(overflow)
+SELECT make_dt_interval(-106751991, -4, 0, -54.775808) / -1
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org