You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2026/05/01 11:10:31 UTC
(doris) branch branch-4.1 updated: branch-4.1: [Opt](parse) Add fast path for canonical format datetime parse #62757 (#62976)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new dd87fadc349 branch-4.1: [Opt](parse) Add fast path for canonical format datetime parse #62757 (#62976)
dd87fadc349 is described below
commit dd87fadc349e702fb229d1cd4ce3c4c4fef3b3db
Author: github-actions[bot] <41...@users.noreply.github.com>
AuthorDate: Fri May 1 19:10:24 2026 +0800
branch-4.1: [Opt](parse) Add fast path for canonical format datetime parse #62757 (#62976)
Cherry-picked from #62757
Co-authored-by: zclllyybb <zh...@selectdb.com>
---
.../core/data_type_serde/datelike_serde_common.hpp | 93 ++++++++++
.../cast/cast_to_date_or_datetime_impl.hpp | 38 ++++
.../function/cast/cast_to_datetimev2_impl.hpp | 21 +++
be/src/exprs/function/cast/cast_to_datev2_impl.hpp | 16 ++
be/test/core/value/vdatetime_value_test.cpp | 191 +++++++++++++++++++++
be/test/runtime/timestamptz_value_test.cpp | 2 +-
6 files changed, 360 insertions(+), 1 deletion(-)
diff --git a/be/src/core/data_type_serde/datelike_serde_common.hpp b/be/src/core/data_type_serde/datelike_serde_common.hpp
index eef56e19be3..c58a90ed62b 100644
--- a/be/src/core/data_type_serde/datelike_serde_common.hpp
+++ b/be/src/core/data_type_serde/datelike_serde_common.hpp
@@ -17,13 +17,106 @@
#pragma once
+#include <cstddef>
#include <cstdint>
+#include <type_traits>
#include "common/config.h"
#include "core/value/vdatetime_value.h"
namespace doris {
+template <typename T>
+inline constexpr bool can_fast_parse_fixed_canonical_time = false;
+
+template <>
+inline constexpr bool can_fast_parse_fixed_canonical_time<VecDateTimeValue> = true;
+
+template <>
+inline constexpr bool can_fast_parse_fixed_canonical_time<DateV2Value<DateTimeV2ValueType>> = true;
+
+template <>
+inline constexpr bool can_fast_parse_fixed_canonical_time<DateV2Value<DateV2ValueType>> = true;
+
+enum class DatelikeFastParseResult : uint8_t {
+ FAIL,
+ // A canonical YYYY-MM-DD prefix matched. Callers still need the full parser when
+ // there is remaining suffix that the fast path did not consume.
+ DATE_ONLY,
+ // A canonical YYYY-MM-DD HH:MM:SS prefix matched.
+ DATE_TIME,
+};
+
+inline PURE bool is_fixed_two_digit_ascii(const char* ptr) {
+ return static_cast<unsigned>(ptr[0] - '0') < 10 && static_cast<unsigned>(ptr[1] - '0') < 10;
+}
+
+inline PURE bool is_fixed_four_digit_ascii(const char* ptr) {
+ return is_fixed_two_digit_ascii(ptr) && is_fixed_two_digit_ascii(ptr + 2);
+}
+
+inline PURE uint32_t parse_fixed_two_digit_ascii(const char* ptr) {
+ return (ptr[0] - '0') * 10 + (ptr[1] - '0');
+}
+
+inline PURE uint32_t parse_fixed_four_digit_ascii(const char* ptr) {
+ return parse_fixed_two_digit_ascii(ptr) * 100 + parse_fixed_two_digit_ascii(ptr + 2);
+}
+
+template <typename T>
+inline DatelikeFastParseResult try_parse_fixed_canonical_datelike_prefix(const char* ptr,
+ size_t size, T& res) {
+ if (size < 10 || ptr[4] != '-' || ptr[7] != '-' || !is_fixed_four_digit_ascii(ptr) ||
+ !is_fixed_two_digit_ascii(ptr + 5) || !is_fixed_two_digit_ascii(ptr + 8)) {
+ return DatelikeFastParseResult::FAIL;
+ }
+
+ const uint32_t year = parse_fixed_four_digit_ascii(ptr);
+ const uint32_t month = parse_fixed_two_digit_ascii(ptr + 5);
+ const uint32_t day = parse_fixed_two_digit_ascii(ptr + 8);
+ if (!try_convert_set_zero_date(res, year, month, day)) {
+ if (!res.template set_time_unit<TimeUnit::YEAR>(year) ||
+ !res.template set_time_unit<TimeUnit::MONTH>(month) ||
+ !res.template set_time_unit<TimeUnit::DAY>(day)) {
+ return DatelikeFastParseResult::FAIL;
+ }
+ }
+
+ if (size == 10) {
+ return DatelikeFastParseResult::DATE_ONLY;
+ }
+
+ if constexpr (!can_fast_parse_fixed_canonical_time<T>) {
+ return DatelikeFastParseResult::DATE_ONLY;
+ }
+
+ if (size < 19 || (ptr[10] != ' ' && ptr[10] != 'T') || ptr[13] != ':' || ptr[16] != ':' ||
+ !is_fixed_two_digit_ascii(ptr + 11) || !is_fixed_two_digit_ascii(ptr + 14) ||
+ !is_fixed_two_digit_ascii(ptr + 17)) {
+ return DatelikeFastParseResult::DATE_ONLY;
+ }
+
+ const uint32_t hour = parse_fixed_two_digit_ascii(ptr + 11);
+ const uint32_t minute = parse_fixed_two_digit_ascii(ptr + 14);
+ const uint32_t second = parse_fixed_two_digit_ascii(ptr + 17);
+ if constexpr (std::is_same_v<T, DateV2Value<DateV2ValueType>>) {
+ // DATEV2 ignores the time fields, but validating a canonical HH:MM:SS prefix here
+ // still lets strict callers continue from the suffix without reparsing the prefix.
+ if (!res.template test_time_unit<TimeUnit::HOUR>(hour) ||
+ !res.template test_time_unit<TimeUnit::MINUTE>(minute) ||
+ !res.template test_time_unit<TimeUnit::SECOND>(second)) {
+ return DatelikeFastParseResult::FAIL;
+ }
+ } else {
+ if (!res.template set_time_unit<TimeUnit::HOUR>(hour) ||
+ !res.template set_time_unit<TimeUnit::MINUTE>(minute) ||
+ !res.template set_time_unit<TimeUnit::SECOND>(second)) {
+ return DatelikeFastParseResult::FAIL;
+ }
+ }
+ return DatelikeFastParseResult::DATE_TIME;
+}
+
inline PURE uint32_t complete_4digit_year(uint32_t year) {
if (year < 70) {
return year + 2000; // 00-69 -> 2000-2069
diff --git a/be/src/exprs/function/cast/cast_to_date_or_datetime_impl.hpp b/be/src/exprs/function/cast/cast_to_date_or_datetime_impl.hpp
index 0e17efc968d..931c733e60c 100644
--- a/be/src/exprs/function/cast/cast_to_date_or_datetime_impl.hpp
+++ b/be/src/exprs/function/cast/cast_to_date_or_datetime_impl.hpp
@@ -356,6 +356,44 @@ inline bool CastToDateOrDatetime::from_string_strict_mode(const StringRef& str,
uint32_t part[4];
bool has_second = false;
+ if (auto fast_parse_result = try_parse_fixed_canonical_datelike_prefix(ptr, str.size, res);
+ fast_parse_result != DatelikeFastParseResult::FAIL) {
+ if constexpr (!IsDatetime) {
+ if (fast_parse_result == DatelikeFastParseResult::DATE_ONLY) {
+ if (str.size == 10) {
+ cast_to_type<TargetType>(res);
+ return true;
+ }
+ } else {
+ has_second = true;
+ if (str.size == 19) {
+ cast_to_type<TargetType>(res);
+ return true;
+ }
+ ptr += 19;
+ goto FRAC;
+ }
+ } else {
+ if (fast_parse_result == DatelikeFastParseResult::DATE_ONLY) {
+ res.template unchecked_set_time_unit<TimeUnit::HOUR>(0);
+ res.template unchecked_set_time_unit<TimeUnit::MINUTE>(0);
+ res.template unchecked_set_time_unit<TimeUnit::SECOND>(0);
+ if (str.size == 10) {
+ cast_to_type<TargetType>(res);
+ return true;
+ }
+ } else {
+ has_second = true;
+ if (str.size == 19) {
+ cast_to_type<TargetType>(res);
+ return true;
+ }
+ ptr += 19;
+ goto FRAC;
+ }
+ }
+ }
+
// special `date` and `time` part format: 14-length digits string. parse it as YYYYMMDDHHMMSS
if (ptr + 13 < end && is_digit_range(ptr, ptr + 14)) {
// if the string is all digits, treat it as a date in YYYYMMDD format.
diff --git a/be/src/exprs/function/cast/cast_to_datetimev2_impl.hpp b/be/src/exprs/function/cast/cast_to_datetimev2_impl.hpp
index ced869c52bd..9ba5d42dd94 100644
--- a/be/src/exprs/function/cast/cast_to_datetimev2_impl.hpp
+++ b/be/src/exprs/function/cast/cast_to_datetimev2_impl.hpp
@@ -361,6 +361,27 @@ inline bool CastToDatetimeV2::from_string_strict_mode_internal(
uint32_t part[4];
bool has_second = false;
+ if (auto fast_parse_result = try_parse_fixed_canonical_datelike_prefix(ptr, str.size, res);
+ fast_parse_result != DatelikeFastParseResult::FAIL) {
+ if (fast_parse_result == DatelikeFastParseResult::DATE_ONLY) {
+ res.unchecked_set_time_unit<TimeUnit::HOUR>(0);
+ res.unchecked_set_time_unit<TimeUnit::MINUTE>(0);
+ res.unchecked_set_time_unit<TimeUnit::SECOND>(0);
+ res.unchecked_set_time_unit<TimeUnit::MICROSECOND>(0);
+ if (str.size == 10) {
+ goto POST_PROCESS;
+ }
+ } else {
+ has_second = true;
+ if (str.size == 19) {
+ res.unchecked_set_time_unit<TimeUnit::MICROSECOND>(0);
+ goto POST_PROCESS;
+ }
+ ptr += 19;
+ goto FRAC;
+ }
+ }
+
// special `date` and `time` part format: 14-length digits string. parse it as YYYYMMDDHHMMSS
if (ptr + 13 < end && is_digit_range(ptr, ptr + 14)) {
// if the string is all digits, treat it as a date in YYYYMMDD format.
diff --git a/be/src/exprs/function/cast/cast_to_datev2_impl.hpp b/be/src/exprs/function/cast/cast_to_datev2_impl.hpp
index 33bbb7dc5d7..e292bbdd071 100644
--- a/be/src/exprs/function/cast/cast_to_datev2_impl.hpp
+++ b/be/src/exprs/function/cast/cast_to_datev2_impl.hpp
@@ -279,6 +279,22 @@ inline bool CastToDateV2::from_string_strict_mode(const StringRef& str,
uint32_t part[4];
bool has_second = false;
+ if (auto fast_parse_result = try_parse_fixed_canonical_datelike_prefix(ptr, str.size, res);
+ fast_parse_result != DatelikeFastParseResult::FAIL) {
+ if (fast_parse_result == DatelikeFastParseResult::DATE_ONLY) {
+ if (str.size == 10) {
+ return true;
+ }
+ } else {
+ has_second = true;
+ if (str.size == 19) {
+ return true;
+ }
+ ptr += 19;
+ goto FRAC;
+ }
+ }
+
// special `date` and `time` part format: 14-length digits string. parse it as YYYYMMDDHHMMSS
if (ptr + 13 < end && is_digit_range(ptr, ptr + 14)) {
// if the string is all digits, treat it as a date in YYYYMMDD format.
diff --git a/be/test/core/value/vdatetime_value_test.cpp b/be/test/core/value/vdatetime_value_test.cpp
index ffca52ef04e..8cb701d97d5 100644
--- a/be/test/core/value/vdatetime_value_test.cpp
+++ b/be/test/core/value/vdatetime_value_test.cpp
@@ -23,6 +23,7 @@
#include <string>
#include "common/exception.h"
+#include "core/data_type_serde/datelike_serde_common.hpp"
#include "core/types.h"
#include "exprs/function/cast/cast_to_datetimev2_impl.hpp"
#include "gtest/gtest_pred_impl.h"
@@ -163,6 +164,196 @@ TEST(VDateTimeValueTest, datetime_v2_from_uint64_test) {
}
}
+TEST(VDateTimeValueTest, datelike_fast_parse_ascii_helpers) {
+ EXPECT_TRUE(is_fixed_two_digit_ascii("00"));
+ EXPECT_TRUE(is_fixed_two_digit_ascii("42"));
+ EXPECT_FALSE(is_fixed_two_digit_ascii("4x"));
+ EXPECT_FALSE(is_fixed_two_digit_ascii("x2"));
+
+ EXPECT_TRUE(is_fixed_four_digit_ascii("2024"));
+ EXPECT_TRUE(is_fixed_four_digit_ascii("0000"));
+ EXPECT_FALSE(is_fixed_four_digit_ascii("202x"));
+ EXPECT_FALSE(is_fixed_four_digit_ascii("20x4"));
+
+ EXPECT_EQ(parse_fixed_two_digit_ascii("00"), 0);
+ EXPECT_EQ(parse_fixed_two_digit_ascii("59"), 59);
+ EXPECT_EQ(parse_fixed_four_digit_ascii("0000"), 0);
+ EXPECT_EQ(parse_fixed_four_digit_ascii("2024"), 2024);
+}
+
+TEST(VDateTimeValueTest, datelike_fast_parse_supported_types) {
+ EXPECT_TRUE(can_fast_parse_fixed_canonical_time<VecDateTimeValue>);
+ EXPECT_TRUE(can_fast_parse_fixed_canonical_time<DateV2Value<DateTimeV2ValueType>>);
+ EXPECT_TRUE(can_fast_parse_fixed_canonical_time<DateV2Value<DateV2ValueType>>);
+ EXPECT_FALSE(can_fast_parse_fixed_canonical_time<int>);
+}
+
+TEST(VDateTimeValueTest, datetime_v2_try_parse_fixed_canonical_datelike_prefix) {
+ {
+ DateV2Value<DateTimeV2ValueType> dt;
+ EXPECT_EQ(try_parse_fixed_canonical_datelike_prefix("2024-01-02", 10, dt),
+ DatelikeFastParseResult::DATE_ONLY);
+ EXPECT_EQ(dt.to_string(), "2024-01-02 00:00:00");
+ EXPECT_EQ(dt.microsecond(), 0);
+ }
+
+ {
+ DateV2Value<DateTimeV2ValueType> dt;
+ EXPECT_EQ(try_parse_fixed_canonical_datelike_prefix("2024-01-02 03:04:05", 19, dt),
+ DatelikeFastParseResult::DATE_TIME);
+ EXPECT_EQ(dt.to_string(), "2024-01-02 03:04:05");
+ EXPECT_EQ(dt.microsecond(), 0);
+ }
+
+ {
+ DateV2Value<DateTimeV2ValueType> dt;
+ EXPECT_EQ(try_parse_fixed_canonical_datelike_prefix("2024-01-02T03:04:05", 19, dt),
+ DatelikeFastParseResult::DATE_TIME);
+ EXPECT_EQ(dt.to_string(), "2024-01-02 03:04:05");
+ }
+
+ {
+ DateV2Value<DateTimeV2ValueType> dt;
+ EXPECT_EQ(try_parse_fixed_canonical_datelike_prefix("2024-01-02 03:04:05.123456", 26, dt),
+ DatelikeFastParseResult::DATE_TIME);
+ EXPECT_EQ(dt.to_string(6), "2024-01-02 03:04:05.000000");
+ }
+}
+
+TEST(VDateTimeValueTest, date_v2_try_parse_fixed_canonical_datelike_prefix) {
+ {
+ DateV2Value<DateV2ValueType> dt;
+ EXPECT_EQ(try_parse_fixed_canonical_datelike_prefix("2024-01-02", 10, dt),
+ DatelikeFastParseResult::DATE_ONLY);
+ EXPECT_EQ(dt.to_string(), "2024-01-02");
+ }
+
+ {
+ DateV2Value<DateV2ValueType> dt;
+ EXPECT_EQ(try_parse_fixed_canonical_datelike_prefix("2024-01-02 03:04:05", 19, dt),
+ DatelikeFastParseResult::DATE_TIME);
+ EXPECT_EQ(dt.to_string(), "2024-01-02");
+ EXPECT_EQ(dt.hour(), 0);
+ EXPECT_EQ(dt.minute(), 0);
+ EXPECT_EQ(dt.second(), 0);
+ }
+
+ {
+ DateV2Value<DateV2ValueType> dt;
+ EXPECT_EQ(try_parse_fixed_canonical_datelike_prefix("2024-01-02 03:04:05 +08:00", 26, dt),
+ DatelikeFastParseResult::DATE_TIME);
+ EXPECT_EQ(dt.to_string(), "2024-01-02");
+ }
+}
+
+TEST(VDateTimeValueTest, datetime_try_parse_fixed_canonical_datelike_prefix) {
+ {
+ VecDateTimeValue dt;
+ dt.set_type(TIME_DATETIME);
+ EXPECT_EQ(try_parse_fixed_canonical_datelike_prefix("2024-01-02", 10, dt),
+ DatelikeFastParseResult::DATE_ONLY);
+ char buf[64];
+ dt.to_string(buf);
+ EXPECT_EQ(std::string(buf), "2024-01-02 00:00:00");
+ }
+
+ {
+ VecDateTimeValue dt;
+ dt.set_type(TIME_DATETIME);
+ EXPECT_EQ(try_parse_fixed_canonical_datelike_prefix("2024-01-02 03:04:05", 19, dt),
+ DatelikeFastParseResult::DATE_TIME);
+ char buf[64];
+ dt.to_string(buf);
+ EXPECT_EQ(std::string(buf), "2024-01-02 03:04:05");
+ }
+
+ {
+ VecDateTimeValue dt;
+ dt.set_type(TIME_DATE);
+ EXPECT_EQ(try_parse_fixed_canonical_datelike_prefix("2024-01-02 03:04:05", 19, dt),
+ DatelikeFastParseResult::DATE_TIME);
+ char buf[64];
+ dt.to_string(buf);
+ EXPECT_EQ(std::string(buf), "2024-01-02");
+ EXPECT_EQ(dt.hour(), 3);
+ EXPECT_EQ(dt.minute(), 4);
+ EXPECT_EQ(dt.second(), 5);
+ }
+}
+
+TEST(VDateTimeValueTest, datelike_try_parse_fixed_canonical_datelike_prefix_failures) {
+ {
+ DateV2Value<DateTimeV2ValueType> dt;
+ EXPECT_EQ(try_parse_fixed_canonical_datelike_prefix("2024-1-02", 9, dt),
+ DatelikeFastParseResult::FAIL);
+ }
+
+ {
+ DateV2Value<DateTimeV2ValueType> dt;
+ EXPECT_EQ(try_parse_fixed_canonical_datelike_prefix("2024/01/02", 10, dt),
+ DatelikeFastParseResult::FAIL);
+ }
+
+ {
+ DateV2Value<DateTimeV2ValueType> dt;
+ EXPECT_EQ(try_parse_fixed_canonical_datelike_prefix("2024-0x-02", 10, dt),
+ DatelikeFastParseResult::FAIL);
+ }
+
+ {
+ DateV2Value<DateTimeV2ValueType> dt;
+ EXPECT_EQ(try_parse_fixed_canonical_datelike_prefix("2024-13-02", 10, dt),
+ DatelikeFastParseResult::FAIL);
+ }
+
+ {
+ DateV2Value<DateTimeV2ValueType> dt;
+ EXPECT_EQ(try_parse_fixed_canonical_datelike_prefix("2024-01-02 25:04:05", 19, dt),
+ DatelikeFastParseResult::FAIL);
+ }
+
+ {
+ DateV2Value<DateTimeV2ValueType> dt;
+ EXPECT_EQ(try_parse_fixed_canonical_datelike_prefix("2024-01-02 03:04", 16, dt),
+ DatelikeFastParseResult::DATE_ONLY);
+ EXPECT_EQ(dt.to_string(), "2024-01-02 00:00:00");
+ }
+
+ {
+ DateV2Value<DateTimeV2ValueType> dt;
+ EXPECT_EQ(try_parse_fixed_canonical_datelike_prefix("2024-01-02 3:04:05", 18, dt),
+ DatelikeFastParseResult::DATE_ONLY);
+ EXPECT_EQ(dt.to_string(), "2024-01-02 00:00:00");
+ }
+}
+
+TEST(VDateTimeValueTest, datelike_try_parse_fixed_canonical_datelike_prefix_zero_date) {
+ const bool old_allow_zero_date = config::allow_zero_date;
+
+ config::allow_zero_date = true;
+ {
+ DateV2Value<DateV2ValueType> dt;
+ EXPECT_EQ(try_parse_fixed_canonical_datelike_prefix("0000-00-00", 10, dt),
+ DatelikeFastParseResult::DATE_ONLY);
+ EXPECT_EQ(dt.to_string(), "0000-01-01");
+ }
+ {
+ DateV2Value<DateTimeV2ValueType> dt;
+ EXPECT_EQ(try_parse_fixed_canonical_datelike_prefix("0000-00-00 11:22:33", 19, dt),
+ DatelikeFastParseResult::DATE_TIME);
+ EXPECT_EQ(dt.to_string(), "0000-01-01 11:22:33");
+ }
+
+ config::allow_zero_date = false;
+ {
+ DateV2Value<DateV2ValueType> dt;
+ EXPECT_EQ(try_parse_fixed_canonical_datelike_prefix("0000-00-00", 10, dt),
+ DatelikeFastParseResult::FAIL);
+ }
+
+ config::allow_zero_date = old_allow_zero_date;
+}
+
TEST(VDateTimeValueTest, date_v2_from_date_format_str_test) {
uint16_t year = 2022;
uint8_t month = 5;
diff --git a/be/test/runtime/timestamptz_value_test.cpp b/be/test/runtime/timestamptz_value_test.cpp
index 3ae98870a1f..ad35e8681c3 100644
--- a/be/test/runtime/timestamptz_value_test.cpp
+++ b/be/test/runtime/timestamptz_value_test.cpp
@@ -184,4 +184,4 @@ TEST(TimeStampTzValueTest, to_datetime) {
}
}
-} // namespace doris
\ No newline at end of file
+} // namespace doris
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org