You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/07/17 19:39:04 UTC
(datafusion) branch main updated: fix: handle null date and timestamp format arguments (#23641)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion.git
The following commit(s) were added to refs/heads/main by this push:
new 8a37a07d5e fix: handle null date and timestamp format arguments (#23641)
8a37a07d5e is described below
commit 8a37a07d5ef1316663e13daa39f8d2de015f4168
Author: linfeng <33...@users.noreply.github.com>
AuthorDate: Sat Jul 18 03:24:18 2026 +0800
fix: handle null date and timestamp format arguments (#23641)
## Which issue does this PR close?
<!--
We generally require a GitHub issue to be filed for all bug fixes and
enhancements and this helps us generate change logs for our releases.
You can link an issue to this PR using the GitHub syntax. For example
`Closes #123` indicates that this PR will close issue #123.
-->
- Closes #23640.
## Rationale for this change
<!--
Why are you proposing this change? If this is already explained clearly
in the issue then this section is not needed.
Explaining clearly why changes are proposed helps reviewers understand
your changes and offer better suggestions for fixes.
-->
The formatted `to_date` and `to_timestamp*` functions can produce
internal errors when the input string is NULL or when all format
arguments are NULL.
A NULL value in a format column is also treated as an empty string
instead of being skipped.
`to_unixtime` uses the same timestamp parsing path, so it is affected as
well.
## What changes are included in this PR?
<!--
There is no need to duplicate the description in the issue here but it
is sometimes worth providing a summary of the individual changes in this
PR.
-->
- Return NULL when the scalar input string is NULL.
- Return NULL when all format arguments are NULL.
- Skip NULL formats in array rows.
- Keep returning a parsing error when all non-NULL formats fail.
## Are these changes tested?
Yes, SLTs were added for `to_date`, `to_timestamp`, and `to_unixtime`.
<!--
We typically require tests for all PRs in order to:
1. Prevent the code from being accidentally broken by subsequent changes
2. Serve as another way to document the expected behavior of the code
If tests are not included in your PR, please explain why (for example,
are they covered by existing tests)?
-->
## Are there any user-facing changes?
<!--
If there are user-facing changes then we may require documentation to be
updated before approving the PR.
-->
<!--
If there are any breaking changes to public APIs, please add the `api
change` label.
-->
Yes, only the bug fix described above. No API changes.
---
datafusion/functions/src/datetime/common.rs | 34 +++++++++++++++-------
.../sqllogictest/test_files/datetime/dates.slt | 21 +++++++++++++
.../test_files/datetime/timestamps.slt | 29 ++++++++++++++++++
3 files changed, 73 insertions(+), 11 deletions(-)
diff --git a/datafusion/functions/src/datetime/common.rs b/datafusion/functions/src/datetime/common.rs
index 2db64beafa..9a7f94bd59 100644
--- a/datafusion/functions/src/datetime/common.rs
+++ b/datafusion/functions/src/datetime/common.rs
@@ -32,7 +32,7 @@ use chrono::{DateTime, TimeZone, Utc};
use datafusion_common::cast::as_generic_string_array;
use datafusion_common::{
DataFusionError, Result, ScalarValue, exec_datafusion_err, exec_err,
- internal_datafusion_err, unwrap_or_internal_err,
+ internal_datafusion_err,
};
use datafusion_expr::ColumnarValue;
@@ -353,9 +353,9 @@ where
// if the first argument is a scalar utf8 all arguments are expected to be scalar utf8
ColumnarValue::Scalar(scalar) => match scalar.try_as_str() {
Some(a) => {
- let a = a.as_ref();
- // ASK: Why do we trust `a` to be non-null at this point?
- let a = unwrap_or_internal_err!(a);
+ let Some(a) = a.as_ref() else {
+ return Ok(ColumnarValue::Scalar(scalar_value(dt, None)?));
+ };
let mut ret = None;
@@ -384,7 +384,10 @@ where
}
}
- unwrap_or_internal_err!(ret)
+ match ret {
+ Some(ret) => ret,
+ None => Ok(ColumnarValue::Scalar(scalar_value(dt, None)?)),
+ }
}
other => {
exec_err!("Unsupported data type {other:?} for function {name}")
@@ -483,12 +486,21 @@ where
if let Some(x) = x {
for arg in args {
let v = match arg {
- ColumnarValue::Array(a) => match a.data_type() {
- DataType::Utf8View => Ok(a.as_string_view().value(pos)),
- DataType::LargeUtf8 => Ok(a.as_string::<i64>().value(pos)),
- DataType::Utf8 => Ok(a.as_string::<i32>().value(pos)),
- other => exec_err!("Unexpected type encountered '{other}'"),
- },
+ ColumnarValue::Array(a) => {
+ if a.is_null(pos) {
+ continue;
+ }
+ match a.data_type() {
+ DataType::Utf8View => Ok(a.as_string_view().value(pos)),
+ DataType::LargeUtf8 => {
+ Ok(a.as_string::<i64>().value(pos))
+ }
+ DataType::Utf8 => Ok(a.as_string::<i32>().value(pos)),
+ other => {
+ exec_err!("Unexpected type encountered '{other}'")
+ }
+ }
+ }
ColumnarValue::Scalar(s) => match s.try_as_str() {
Some(Some(v)) => Ok(v),
Some(None) => continue, // null string
diff --git a/datafusion/sqllogictest/test_files/datetime/dates.slt b/datafusion/sqllogictest/test_files/datetime/dates.slt
index d2a7360b12..68d87eceed 100644
--- a/datafusion/sqllogictest/test_files/datetime/dates.slt
+++ b/datafusion/sqllogictest/test_files/datetime/dates.slt
@@ -298,6 +298,27 @@ SELECT to_date('2020-09-08 12/00/00+00:00', '%c', '%+')
query error DataFusion error: Execution error: Error parsing timestamp from '2020\-09\-08 12/00/00\+00:00' using format '%q': trailing input
SELECT to_date('2020-09-08 12/00/00+00:00', '%q')
+# NULL string scalar inputs and all-NULL scalar formats return NULL
+query DD
+SELECT
+ to_date(NULL::VARCHAR, '%Y-%m-%d'),
+ to_date('2020-09-08', NULL::VARCHAR, NULL::VARCHAR)
+----
+NULL NULL
+
+# NULL array formats are skipped for each row; rows with no usable format return NULL
+query ID
+SELECT id, to_date(value, format1, format2)
+FROM (
+ VALUES
+ (1, '2020-09-08', NULL::VARCHAR, '%Y-%m-%d'),
+ (2, '2020-09-08', NULL::VARCHAR, NULL::VARCHAR)
+) AS t(id, value, format1, format2)
+ORDER BY id
+----
+1 2020-09-08
+2 NULL
+
statement ok
create table ts_utf8_data(ts varchar(100), format varchar(100)) as values
('2020-09-08 12/00/00+00:00', '%Y-%m-%d %H/%M/%S%#z'),
diff --git a/datafusion/sqllogictest/test_files/datetime/timestamps.slt b/datafusion/sqllogictest/test_files/datetime/timestamps.slt
index 8ba095ef93..9ac00e72b4 100644
--- a/datafusion/sqllogictest/test_files/datetime/timestamps.slt
+++ b/datafusion/sqllogictest/test_files/datetime/timestamps.slt
@@ -518,6 +518,35 @@ SELECT COUNT(*) FROM ts_data_secs where ts > to_timestamp_seconds('2020-09-08 12
----
2
+# NULL string scalar inputs and all-NULL scalar formats return NULL
+query PP
+SELECT
+ to_timestamp(NULL::VARCHAR, '%Y-%m-%d'),
+ to_timestamp('2020-09-08', NULL::VARCHAR, NULL::VARCHAR)
+----
+NULL NULL
+
+# NULL array formats are skipped for each row; rows with no usable format return NULL
+query IP
+SELECT id, to_timestamp(value, format1, format2)
+FROM (
+ VALUES
+ (1, '2020-09-08', NULL::VARCHAR, '%Y-%m-%d'),
+ (2, '2020-09-08', NULL::VARCHAR, NULL::VARCHAR)
+) AS t(id, value, format1, format2)
+ORDER BY id
+----
+1 2020-09-08T00:00:00
+2 NULL
+
+# to_unixtime uses the same formatted string parsing path
+query II
+SELECT
+ to_unixtime(NULL::VARCHAR, '%Y-%m-%d'),
+ to_unixtime('2020-09-08', NULL::VARCHAR, NULL::VARCHAR)
+----
+NULL NULL
+
# to_timestamp float inputs
query PPP
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org