You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2026/10/15 01:17:27 UTC
(doris) branch branch-4.2 updated: [feature](timestamp_ns) Add end-to-end TIMESTAMP_NS support (#66761) (#68754)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.2 by this push:
new 07a9ddc7a1d [feature](timestamp_ns) Add end-to-end TIMESTAMP_NS support (#66761) (#68754)
07a9ddc7a1d is described below
commit 07a9ddc7a1df913ddcf9d76cba7afe2210518e4e
Author: TengJianPing <te...@selectdb.com>
AuthorDate: Thu Oct 15 09:17:15 2026 +0800
[feature](timestamp_ns) Add end-to-end TIMESTAMP_NS support (#66761) (#68754)
### What problem does this PR solve?
Issue Number: close #xxx
Related PR: #xxx
Problem Summary:
### Release note
None
### Check List (For Author)
- Test <!-- At least one of them must be included. -->
- [ ] Regression test
- [ ] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason <!-- Add your reason? -->
- Behavior changed:
- [ ] No.
- [ ] Yes. <!-- Explain the behavior change -->
- Does this need documentation?
- [ ] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---
be/src/agent/be_exec_version_manager.cpp | 4 +-
be/src/agent/be_exec_version_manager.h | 1 +
be/src/core/call_on_type_index.h | 5 +
be/src/core/column/column_array.cpp | 6 +
be/src/core/column/column_array.h | 2 +-
be/src/core/column/column_vector.cpp | 1 +
be/src/core/column/column_vector.h | 8 +-
be/src/core/column/columns_common.cpp | 2 +
.../variant_v2/column_variant_v2_typed_column.cpp | 1 +
.../variant_v2/column_variant_v2_typed_column.h | 2 +
be/src/core/data_type/convert_field_to_type.cpp | 6 +
be/src/core/data_type/data_type.cpp | 2 +
be/src/core/data_type/data_type_factory.cpp | 10 +
be/src/core/data_type/data_type_number_base.cpp | 11 +-
be/src/core/data_type/data_type_number_base.h | 3 +-
be/src/core/data_type/data_type_time.h | 4 +-
be/src/core/data_type/data_type_timestamp_ns.cpp | 48 +
.../core/data_type/data_type_timestamp_ns.h} | 37 +-
be/src/core/data_type/define_primitive_type.h | 3 +-
be/src/core/data_type/primitive_type.cpp | 9 +
be/src/core/data_type/primitive_type.h | 15 +
be/src/core/data_type/storage_field_type.cpp | 4 +
.../data_type_serde/data_type_number_serde.cpp | 38 +-
.../core/data_type_serde/data_type_number_serde.h | 20 +-
be/src/core/data_type_serde/data_type_serde.cpp | 2 +
.../data_type_timestamp_ns_serde.cpp | 505 ++++
.../data_type_serde/data_type_timestamp_ns_serde.h | 102 +
be/src/core/field.cpp | 23 +
be/src/core/type_limit.h | 8 +
be/src/core/value/time_value.h | 6 +-
be/src/core/value/timestamp_ns_value.cpp | 98 +
be/src/core/value/timestamp_ns_value.h | 324 +++
be/src/core/value/vdatetime_value.cpp | 25 +-
be/src/core/value/vdatetime_value.h | 4 +-
be/src/exec/common/field_visitors.h | 3 +
be/src/exec/common/hash_table/hash.h | 12 +
be/src/exec/common/join_utils.h | 38 +-
be/src/exec/common/variant_util.cpp | 4 +-
be/src/exec/operator/hashjoin_build_sink.cpp | 197 +-
be/src/exec/operator/hashjoin_build_sink.h | 1 +
.../operator/join/process_hash_table_probe_impl.h | 44 +-
be/src/exec/operator/scan_operator.cpp | 22 +-
be/src/exec/operator/scan_operator.h | 4 +
be/src/exec/pipeline/dependency.h | 3 +-
.../exec/runtime_filter/runtime_filter_wrapper.cpp | 4 +
be/src/exec/runtime_filter/utils.h | 3 +
be/src/exec/sink/vrow_distribution.h | 1 +
.../aggregate_function_approx_count_distinct.cpp | 2 +-
.../aggregate_function_approx_count_distinct.h | 5 +-
.../exprs/aggregate/aggregate_function_array_agg.h | 5 +-
.../exprs/aggregate/aggregate_function_collect.h | 2 +-
.../aggregate/aggregate_function_histogram.cpp | 12 +-
.../exprs/aggregate/aggregate_function_map_v2.cpp | 3 +-
.../aggregate/aggregate_function_min_max_by.cpp | 4 +
.../aggregate/aggregate_function_min_max_by.h | 6 +
.../aggregate/aggregate_function_min_max_impl.h | 4 +
.../aggregate_function_sequence_match.cpp | 3 +
.../aggregate/aggregate_function_topn_array.cpp | 10 +-
.../aggregate/aggregate_function_topn_weighted.cpp | 10 +-
be/src/exprs/aggregate/aggregate_function_uniq.cpp | 13 +-
.../aggregate/aggregate_function_window_funnel.cpp | 3 +
.../aggregate/aggregate_function_window_funnel.h | 32 +-
.../aggregate_function_window_funnel_v2.cpp | 3 +
.../aggregate_function_window_funnel_v2.h | 57 +-
be/src/exprs/create_predicate_function.h | 1 +
.../function/array/function_array_aggregation.cpp | 5 +-
be/src/exprs/function/array/function_array_index.h | 6 +-
.../exprs/function/array/function_array_range.cpp | 34 +-
.../exprs/function/array/function_array_remove.h | 5 +
.../function/cast/cast_to_basic_number_common.h | 8 +-
be/src/exprs/function/cast/cast_to_date.h | 69 +
.../function/cast/cast_to_datetimev2_impl.hpp | 227 +-
be/src/exprs/function/cast/cast_to_float.h | 1 +
be/src/exprs/function/cast/cast_to_int.h | 5 +-
be/src/exprs/function/cast/cast_to_string.h | 13 +
be/src/exprs/function/cast/cast_to_timestamp_ns.h | 403 +++
be/src/exprs/function/cast/cast_to_timestamptz.h | 48 +
be/src/exprs/function/cast/cast_wrapper_decls.h | 5 +-
be/src/exprs/function/cast/function_cast.cpp | 5 +-
be/src/exprs/function/cast/function_cast_date.cpp | 4 +-
...estamptz.cpp => function_cast_timestamp_ns.cpp} | 42 +-
.../function/cast/function_cast_timestamptz.cpp | 3 +-
.../cast/variant_v2/cast_variant_to_scalar.cpp | 81 +-
be/src/exprs/function/date_format_type.h | 3 +-
be/src/exprs/function/date_time_transforms.h | 230 +-
be/src/exprs/function/dictionary.h | 8 +-
be/src/exprs/function/function.cpp | 7 +
be/src/exprs/function/function.h | 4 +-
be/src/exprs/function/function_convert_tz.cpp | 66 +-
.../function_date_or_datetime_computation.cpp | 144 +-
.../function_date_or_datetime_computation.h | 482 +++-
.../function_date_or_datetime_to_something.h | 9 +-
.../function_date_or_datetime_to_string.cpp | 46 +-
.../function/function_datetime_floor_ceil.cpp | 154 +-
.../function_datetime_string_to_string.cpp | 8 +
.../function/function_datetime_string_to_string.h | 1 +
be/src/exprs/function/function_java_udf.cpp | 10 +-
be/src/exprs/function/function_map.cpp | 12 +-
.../function/function_other_types_to_date.cpp | 155 +-
be/src/exprs/function/functions_comparison.h | 114 +-
be/src/exprs/function/least_greast.cpp | 17 +-
be/src/exprs/function/time_of_function.cpp | 19 +
be/src/exprs/function/to_time_function.cpp | 59 +
be/src/exprs/function_context.cpp | 1 +
be/src/exprs/function_context.h | 5 +
.../exprs/lambda_function/varray_sort_function.cpp | 3 +-
be/src/exprs/vcase_expr.h | 3 +
be/src/exprs/vcast_expr.cpp | 3 +
be/src/exprs/vcast_expr.h | 10 +-
be/src/exprs/vcondition_expr.cpp | 5 +
be/src/exprs/vdirect_in_predicate.h | 1 +
be/src/exprs/vectorized_fn_call.cpp | 2 +
be/src/exprs/vexpr.cpp | 9 +
be/src/exprs/vexpr.h | 9 +
be/src/exprs/vexpr_context.h | 6 +
be/src/exprs/vin_predicate.cpp | 1 +
be/src/exprs/vtopn_pred.cpp | 2 +
be/src/format/arrow/arrow_row_batch.cpp | 4 +
be/src/format/jni/jni_data_bridge.cpp | 5 +
be/src/format/jni/jni_data_bridge.h | 4 +-
.../information_schema/schema_columns_scanner.cpp | 14 +-
be/src/runtime/fold_constant_executor.cpp | 16 +-
be/src/runtime/runtime_predicate.cpp | 3 +-
be/src/storage/delete/delete_handler.cpp | 2 +
.../bloom_filter/bloom_filter_index_writer.cpp | 1 +
be/src/storage/index/index_writer.cpp | 1 +
.../index/inverted/inverted_index_writer.cpp | 1 +
.../storage/index/inverted/inverted_index_writer.h | 2 +-
be/src/storage/index/snii/bkd/bkd_index_block.cpp | 4 +-
be/src/storage/index/zone_map/zone_map_index.cpp | 1 +
be/src/storage/iterator/olap_data_convertor.cpp | 3 +
be/src/storage/key_coder.cpp | 1 +
be/src/storage/key_coder.h | 1 +
be/src/storage/olap_common.h | 2 +
be/src/storage/olap_scan_common.h | 9 +-
be/src/storage/partial_update_info.cpp | 28 +
be/src/storage/predicate/predicate_creator.cpp | 4 +
.../predicate/predicate_creator_comparison.cpp | 4 +
.../predicate/predicate_creator_in_list_in.cpp | 4 +
.../predicate/predicate_creator_in_list_not_in.cpp | 4 +
be/src/storage/segment/encoding_info.cpp | 5 +
.../segment/variant/v2/variant_path_builder.cpp | 43 +-
.../segment/variant/v2/variant_storage_cell.cpp | 13 +
be/src/storage/tablet/tablet_meta.cpp | 3 +
be/src/storage/tablet/tablet_schema.cpp | 16 +
be/src/storage/tablet/tablet_schema.h | 8 +
be/src/storage/tablet_info.cpp | 6 +
be/src/storage/types.h | 10 +-
be/src/storage/utils.cpp | 26 +-
be/src/storage/utils.h | 3 +-
be/src/tools/meta_tool.cpp | 10 +
be/src/udf/python/python_server.py | 10 +
be/src/util/raw_value.h | 3 +-
be/test/core/column/common_column_test.h | 5 +-
be/test/core/data_type/common_data_type_test.h | 1 +
.../core/data_type/data_type_datetime_v2_test.cpp | 5 +-
.../core/data_type/data_type_timestamp_ns_test.cpp | 685 +++++
be/test/core/data_type/storage_field_type_test.cpp | 4 +-
.../data_type_serde/data_type_serde_arrow_test.cpp | 181 ++
.../data_type_serde_datelike_batch_test.cpp | 78 +-
.../data_type_serde_decoded_values_test.cpp | 2 +-
be/test/core/jsonb/convert_field_to_type_test.cpp | 20 +
be/test/exec/common/schema_util_test.cpp | 9 +
be/test/exec/operator/asof_join_test.cpp | 50 +
.../operator/scan_normalize_predicate_test.cpp | 89 +
.../runtime_filter/runtime_filter_wrapper_test.cpp | 2 +
.../sink/tablet_sink_hash_partitioner_test.cpp | 109 +
.../aggregate/agg_approx_count_distinct_test.cpp | 64 +
be/test/exprs/aggregate/agg_min_max_test.cpp | 37 +
.../exprs/aggregate/vec_sequence_match_test.cpp | 10 +
be/test/exprs/aggregate/vec_window_funnel_test.cpp | 50 +
.../exprs/aggregate/vec_window_funnel_v2_test.cpp | 23 +
be/test/exprs/function/cast/cast_test.h | 2 +
.../exprs/function/cast/cast_timestamp_ns_test.cpp | 292 +++
.../function/cast/cast_to_timestamptz_test.cpp | 100 +
.../function/cast/cast_variant_v2_from_test.cpp | 44 +
...ion_comparison_evaluate_inverted_index_test.cpp | 51 +
be/test/exprs/function/function_test_util.cpp | 21 +-
be/test/exprs/function/function_test_util.h | 9 +
be/test/exprs/function/function_time_test.cpp | 29 +
.../exprs/function/function_timestamp_ns_test.cpp | 652 +++++
.../function/simple_function_factory_test.cpp | 108 +
be/test/exprs/vcondition_expr_test.cpp | 56 +-
be/test/format_v2/jni/jni_table_reader_test.cpp | 38 +
be/test/runtime/fold_constant_executor_test.cpp | 87 +
be/test/runtime/runtime_predicate_test.cpp | 23 +-
be/test/storage/delete/delete_handler_test.cpp | 7 +
.../index/snii/snii_bkd_adapter_types_test.cpp | 5 +-
be/test/storage/key_coder_test.cpp | 1 +
be/test/storage/partial_update_info_test.cpp | 84 +
.../predicate/block_column_predicate_test.cpp | 53 +-
.../storage/predicate/predicate_creator_test.cpp | 135 +
.../bloom_filter_index_reader_writer_test.cpp | 42 +
.../storage/segment/inverted_index_writer_test.cpp | 112 +-
.../segment/variant_assembler_legacy_test.cpp | 7 +
be/test/storage/segment/zone_map_index_test.cpp | 90 +
be/test/storage/tablet/tablet_schema_test.cpp | 19 +
.../variant/variant_column_writer_reader_test.cpp | 104 +
be/test/testutil/datetime_ut_util.h | 2 +-
.../doris/common/jni/utils/JavaUdfDataType.java | 6 +
.../doris/common/jni/utils/TypeNativeBytes.java | 18 +
.../apache/doris/common/jni/vec/ColumnType.java | 5 +
.../apache/doris/common/jni/vec/VectorColumn.java | 55 +
.../apache/doris/common/jni/JniScannerTest.java | 89 +
.../java/org/apache/doris/udf/BaseExecutor.java | 6 +-
.../org/apache/doris/catalog/PrimitiveType.java | 16 +-
.../java/org/apache/doris/catalog/ScalarType.java | 26 +-
.../main/java/org/apache/doris/catalog/Type.java | 27 +-
.../main/java/org/apache/doris/common/Config.java | 4 +-
.../antlr4/org/apache/doris/nereids/DorisLexer.g4 | 1 +
.../antlr4/org/apache/doris/nereids/DorisParser.g4 | 2 +
.../java/org/apache/doris/analysis/CastExpr.java | 18 +-
.../java/org/apache/doris/analysis/ColumnDef.java | 14 +-
.../org/apache/doris/analysis/DateLiteral.java | 5 +-
.../apache/doris/analysis/DateLiteralUtils.java | 372 +++
.../org/apache/doris/analysis/LiteralExpr.java | 9 +-
.../apache/doris/analysis/PartitionExprUtil.java | 75 +-
.../apache/doris/analysis/TimeStampNsLiteral.java | 400 +++
.../org/apache/doris/catalog/AggregateType.java | 2 +
.../doris/catalog/BuiltinScalarFunctions.java | 8 +
.../main/java/org/apache/doris/catalog/Column.java | 15 +-
.../java/org/apache/doris/catalog/ColumnType.java | 9 +
.../org/apache/doris/catalog/PartitionInfo.java | 3 +-
.../org/apache/doris/catalog/PartitionKey.java | 33 +-
.../doris/common/util/DynamicPartitionUtil.java | 1 +
.../common/util/FetchRemoteTabletSchemaUtil.java | 1 +
.../apache/doris/common/util/PropertyAnalyzer.java | 2 +-
.../java/org/apache/doris/common/util/Util.java | 1 +
.../doris/httpv2/util/StatementSubmitter.java | 3 +-
.../util/streamresponse/JsonStreamResponse.java | 3 +-
.../doris/mtmv/MTMVPartitionExprDateTrunc.java | 83 +
...MTMVRelatedPartitionDescSyncLimitGenerator.java | 5 +-
.../main/java/org/apache/doris/mtmv/MTMVUtil.java | 9 +
.../org/apache/doris/mysql/MysqlSerializer.java | 26 +-
.../glue/translator/ExpressionTranslator.java | 1 +
.../nereids/load/NereidsLoadScanProvider.java | 8 +-
.../nereids/load/NereidsStreamLoadPlanner.java | 8 +-
.../doris/nereids/parser/LogicalPlanBuilder.java | 4 +-
.../doris/nereids/rules/analysis/BindSink.java | 8 +-
.../rules/analysis/DatetimeFunctionBinder.java | 45 +-
.../nereids/rules/analysis/ExpressionAnalyzer.java | 21 +-
.../mv/AbstractMaterializedViewRule.java | 2 +-
.../nereids/rules/expression/check/CheckCast.java | 26 +
.../expression/rules/DateFunctionRewrite.java | 28 +
.../expression/rules/FoldConstantRuleOnBE.java | 33 +-
.../expression/rules/FoldConstantRuleOnFE.java | 12 +-
.../rules/PredicateRewriteForPartitionPrune.java | 121 +-
.../rules/expression/rules/SimplifyCastRule.java | 2 +-
.../rules/SimplifyComparisonPredicate.java | 100 +-
.../expression/rules/SimplifyInPredicate.java | 7 +-
.../PushDownVirtualColumnsIntoOlapScan.java | 2 +
.../doris/nereids/stats/FilterEstimation.java | 36 +-
.../doris/nereids/trees/expressions/Cast.java | 111 +-
.../trees/expressions/TimestampArithmetic.java | 3 +-
.../functions/ComputeSignatureHelper.java | 42 +-
.../functions/DateTimeWithPrecision.java | 16 +-
.../expressions/functions/SearchSignature.java | 46 +-
.../SupportsMixedTimeStampNsDateTime.java | 19 +-
.../expressions/functions/agg/SequenceCount.java | 3 +
.../functions/agg/SequenceFunction.java | 3 +-
.../expressions/functions/agg/SequenceMatch.java | 3 +
.../expressions/functions/agg/TopNWeighted.java | 8 +
.../expressions/functions/agg/WindowFunnel.java | 6 +-
.../expressions/functions/agg/WindowFunnelV2.java | 6 +-
.../functions/executable/DateTimeAcquire.java | 66 +-
.../functions/executable/DateTimeArithmetic.java | 289 ++-
.../executable/DateTimeExtractAndTransform.java | 481 +++-
.../functions/executable/StringArithmetic.java | 15 +
.../functions/executable/TimeRoundSeries.java | 353 ++-
.../expressions/functions/scalar/AddTime.java | 3 +
.../expressions/functions/scalar/ArrayRange.java | 5 +
.../functions/scalar/ArrayRangeDayUnit.java | 3 +
.../functions/scalar/ArrayRangeHourUnit.java | 3 +
.../functions/scalar/ArrayRangeMinuteUnit.java | 3 +
.../functions/scalar/ArrayRangeMonthUnit.java | 3 +
.../functions/scalar/ArrayRangeQuarterUnit.java | 3 +
.../functions/scalar/ArrayRangeSecondUnit.java | 3 +
.../functions/scalar/ArrayRangeWeekUnit.java | 3 +
.../functions/scalar/ArrayRangeYearUnit.java | 3 +
.../expressions/functions/scalar/Century.java | 4 +-
.../expressions/functions/scalar/Coalesce.java | 7 +-
.../expressions/functions/scalar/ConvertTz.java | 5 +-
.../trees/expressions/functions/scalar/Date.java | 4 +-
.../expressions/functions/scalar/DateDiff.java | 11 +-
.../expressions/functions/scalar/DateFormat.java | 3 +
.../trees/expressions/functions/scalar/DateV2.java | 4 +-
.../expressions/functions/scalar/DayCeil.java | 8 +
.../expressions/functions/scalar/DayFloor.java | 8 +
.../expressions/functions/scalar/DayHour.java | 4 +-
.../expressions/functions/scalar/DayHourAdd.java | 3 +
.../expressions/functions/scalar/DayHourSub.java | 3 +
.../functions/scalar/DayMicrosecond.java | 4 +-
.../functions/scalar/DayMicrosecondAdd.java | 6 +
.../functions/scalar/DayMicrosecondSub.java | 6 +
.../expressions/functions/scalar/DayMinute.java | 4 +-
.../expressions/functions/scalar/DayMinuteAdd.java | 3 +
.../expressions/functions/scalar/DayMinuteSub.java | 3 +
.../expressions/functions/scalar/DayName.java | 4 +-
.../expressions/functions/scalar/DayOfMonth.java | 4 +-
.../expressions/functions/scalar/DayOfWeek.java | 4 +-
.../expressions/functions/scalar/DayOfYear.java | 4 +-
.../expressions/functions/scalar/DaySecond.java | 4 +-
.../expressions/functions/scalar/DaySecondAdd.java | 3 +
.../expressions/functions/scalar/DaySecondSub.java | 3 +
.../expressions/functions/scalar/DaysAdd.java | 3 +
.../expressions/functions/scalar/DaysDiff.java | 9 +-
.../expressions/functions/scalar/DaysSub.java | 2 +
.../trees/expressions/functions/scalar/Field.java | 2 +
.../expressions/functions/scalar/FromUnixtime.java | 61 +-
.../functions/scalar/GreatestLeast.java | 7 +-
.../trees/expressions/functions/scalar/Hour.java | 2 +
.../expressions/functions/scalar/HourCeil.java | 8 +
.../expressions/functions/scalar/HourFloor.java | 8 +
.../functions/scalar/HourMicrosecond.java | 4 +-
.../functions/scalar/HourMicrosecondAdd.java | 6 +
.../functions/scalar/HourMicrosecondSub.java | 6 +
.../expressions/functions/scalar/HourMinute.java | 4 +-
.../functions/scalar/HourMinuteAdd.java | 3 +
.../functions/scalar/HourMinuteSub.java | 3 +
.../expressions/functions/scalar/HourSecond.java | 4 +-
.../functions/scalar/HourSecondAdd.java | 3 +
.../functions/scalar/HourSecondSub.java | 3 +
.../expressions/functions/scalar/HoursAdd.java | 4 +-
.../expressions/functions/scalar/HoursDiff.java | 10 +-
.../expressions/functions/scalar/HoursSub.java | 4 +-
.../trees/expressions/functions/scalar/If.java | 2 +-
.../expressions/functions/scalar/LastDay.java | 4 +-
.../functions/scalar/MicroSecondTimestamp.java | 4 +-
.../functions/scalar/MicroSecondsAdd.java | 5 +
.../functions/scalar/MicroSecondsDiff.java | 10 +-
.../functions/scalar/MicroSecondsSub.java | 5 +
.../expressions/functions/scalar/Microsecond.java | 13 +
.../functions/scalar/MilliSecondTimestamp.java | 4 +-
.../functions/scalar/MilliSecondsAdd.java | 5 +
.../functions/scalar/MilliSecondsDiff.java | 10 +-
.../functions/scalar/MilliSecondsSub.java | 5 +
.../trees/expressions/functions/scalar/Minute.java | 11 +
.../expressions/functions/scalar/MinuteCeil.java | 8 +
.../expressions/functions/scalar/MinuteFloor.java | 8 +
.../functions/scalar/MinuteMicrosecond.java | 4 +-
.../functions/scalar/MinuteMicrosecondAdd.java | 6 +
.../functions/scalar/MinuteMicrosecondSub.java | 6 +
.../expressions/functions/scalar/MinuteSecond.java | 4 +-
.../functions/scalar/MinuteSecondAdd.java | 3 +
.../functions/scalar/MinuteSecondSub.java | 3 +
.../expressions/functions/scalar/MinutesAdd.java | 4 +-
.../expressions/functions/scalar/MinutesDiff.java | 10 +-
.../expressions/functions/scalar/MinutesSub.java | 4 +-
.../trees/expressions/functions/scalar/Month.java | 4 +-
.../expressions/functions/scalar/MonthCeil.java | 8 +
.../expressions/functions/scalar/MonthFloor.java | 8 +
.../expressions/functions/scalar/MonthName.java | 4 +-
.../expressions/functions/scalar/MonthsAdd.java | 3 +
.../functions/scalar/MonthsBetween.java | 5 +
.../expressions/functions/scalar/MonthsDiff.java | 10 +-
.../expressions/functions/scalar/MonthsSub.java | 3 +
.../{MinutesAdd.java => NanoSecondsAdd.java} | 31 +-
.../{WeeksDiff.java => NanoSecondsDiff.java} | 45 +-
.../{MinutesAdd.java => NanoSecondsSub.java} | 31 +-
.../scalar/{Microsecond.java => Nanosecond.java} | 50 +-
.../expressions/functions/scalar/NextDay.java | 2 +
.../trees/expressions/functions/scalar/NullIf.java | 13 +-
.../trees/expressions/functions/scalar/Nvl.java | 2 +-
.../expressions/functions/scalar/PreviousDay.java | 2 +
.../expressions/functions/scalar/Quarter.java | 2 +
.../expressions/functions/scalar/QuarterCeil.java | 8 +
.../expressions/functions/scalar/QuarterFloor.java | 8 +
.../expressions/functions/scalar/QuartersAdd.java | 3 +
.../expressions/functions/scalar/QuartersDiff.java | 10 +-
.../expressions/functions/scalar/QuartersSub.java | 3 +
.../trees/expressions/functions/scalar/Second.java | 12 +
.../expressions/functions/scalar/SecondCeil.java | 8 +
.../expressions/functions/scalar/SecondFloor.java | 8 +
.../functions/scalar/SecondMicrosecond.java | 4 +-
.../functions/scalar/SecondMicrosecondAdd.java | 6 +
.../functions/scalar/SecondMicrosecondSub.java | 6 +
.../functions/scalar/SecondTimestamp.java | 4 +-
.../expressions/functions/scalar/SecondsAdd.java | 4 +-
.../expressions/functions/scalar/SecondsDiff.java | 10 +-
.../expressions/functions/scalar/SecondsSub.java | 4 +-
.../expressions/functions/scalar/SubTime.java | 3 +
.../trees/expressions/functions/scalar/Time.java | 4 +-
.../expressions/functions/scalar/TimeDiff.java | 17 +-
.../expressions/functions/scalar/TimeFormat.java | 5 +-
.../expressions/functions/scalar/TimeToSec.java | 2 +
.../expressions/functions/scalar/Timestamp.java | 5 +-
.../trees/expressions/functions/scalar/ToDate.java | 4 +-
.../expressions/functions/scalar/ToDateV2.java | 4 +-
.../trees/expressions/functions/scalar/ToDays.java | 4 +-
.../expressions/functions/scalar/ToIso8601.java | 2 +
.../trees/expressions/functions/scalar/ToJson.java | 2 +
.../expressions/functions/scalar/ToMonday.java | 4 +-
.../expressions/functions/scalar/ToSeconds.java | 4 +-
.../functions/scalar/UnixTimestamp.java | 46 +-
.../expressions/functions/scalar/UtcTimestamp.java | 18 +-
.../trees/expressions/functions/scalar/Week.java | 5 +-
.../expressions/functions/scalar/WeekCeil.java | 8 +
.../expressions/functions/scalar/WeekFloor.java | 8 +
.../expressions/functions/scalar/WeekOfYear.java | 4 +-
.../expressions/functions/scalar/Weekday.java | 4 +-
.../expressions/functions/scalar/WeeksAdd.java | 3 +
.../expressions/functions/scalar/WeeksDiff.java | 10 +-
.../expressions/functions/scalar/WeeksSub.java | 3 +
.../trees/expressions/functions/scalar/Year.java | 4 +-
.../expressions/functions/scalar/YearCeil.java | 8 +
.../expressions/functions/scalar/YearFloor.java | 8 +
.../expressions/functions/scalar/YearMonth.java | 4 +-
.../expressions/functions/scalar/YearMonthAdd.java | 3 +
.../expressions/functions/scalar/YearMonthSub.java | 3 +
.../expressions/functions/scalar/YearOfWeek.java | 2 +
.../expressions/functions/scalar/YearWeek.java | 5 +-
.../expressions/functions/scalar/YearsAdd.java | 3 +
.../expressions/functions/scalar/YearsDiff.java | 10 +-
.../expressions/functions/scalar/YearsSub.java | 3 +
.../functions/window/RequireTrivialTypes.java | 2 +
.../trees/expressions/literal/DateLiteral.java | 65 +-
.../trees/expressions/literal/DateTimeLiteral.java | 32 +
.../expressions/literal/DateTimeV2Literal.java | 26 +-
.../nereids/trees/expressions/literal/Literal.java | 18 +-
.../trees/expressions/literal/NumericLiteral.java | 12 +-
.../expressions/literal/StringLikeLiteral.java | 12 +-
.../expressions/literal/TimeStampNsLiteral.java | 355 +++
.../trees/expressions/literal/TimeV2Literal.java | 4 +
.../expressions/literal/TimestampTzLiteral.java | 12 +-
.../expressions/visitor/ExpressionVisitor.java | 5 +
.../expressions/visitor/ScalarFunctionVisitor.java | 20 +
.../plans/commands/CreateFunctionCommand.java | 3 +
.../plans/commands/info/ColumnDefinition.java | 2 +-
.../trees/plans/commands/info/DefaultValue.java | 39 +-
.../trees/plans/logical/LogicalRecursiveUnion.java | 6 +-
.../trees/plans/logical/LogicalSetOperation.java | 6 +-
.../nereids/trees/plans/logical/LogicalUnion.java | 4 +-
.../org/apache/doris/nereids/types/DataType.java | 43 +-
.../doris/nereids/types/TimeStampNsType.java | 99 +
.../doris/nereids/types/coercion/DateLikeType.java | 4 +
.../doris/nereids/util/DateTimeFormatterUtils.java | 93 +-
.../org/apache/doris/nereids/util/DateUtils.java | 114 +
.../org/apache/doris/nereids/util/JoinUtils.java | 6 +
.../doris/nereids/util/PredicateInferUtils.java | 6 +
.../doris/nereids/util/TypeCoercionUtils.java | 758 +++++-
.../org/apache/doris/persist/gson/GsonUtils.java | 2 +
.../apache/doris/planner/GroupCommitPlanner.java | 20 +-
.../java/org/apache/doris/qe/ConnectContext.java | 13 +-
.../main/java/org/apache/doris/qe/Coordinator.java | 14 +-
.../org/apache/doris/qe/CoordinatorContext.java | 34 +-
.../java/org/apache/doris/qe/StmtExecutor.java | 5 +
.../org/apache/doris/qe/cache/PartitionRange.java | 1 +
.../service/arrowflight/FlightSqlSchemaHelper.java | 2 +
.../doris/statistics/util/StatisticsUtil.java | 6 +-
.../doris/analysis/CastExprHashPartitionTest.java | 46 +
.../analysis/CurrentTimestampDefaultValueTest.java | 86 +
.../doris/analysis/DateLiteralUtilsTest.java | 804 ++++++
.../java/org/apache/doris/catalog/ColumnTest.java | 35 +
.../apache/doris/catalog/CreateFunctionTest.java | 6 +
.../org/apache/doris/catalog/PartitionKeyTest.java | 107 +
.../org/apache/doris/catalog/TimestampNsUdf.java | 26 +-
.../java/org/apache/doris/catalog/TypeTest.java | 35 +
.../apache/doris/mtmv/MTMVPartitionUtilTest.java | 46 +
...TMVRelatedPartitionDescRollUpGeneratorTest.java | 55 +
...RelatedPartitionDescSyncLimitGeneratorTest.java | 10 +
.../java/org/apache/doris/mtmv/MTMVUtilTest.java | 11 +
.../doris/mysql/MysqlSerializerDateTimeTest.java | 72 +
.../nereids/load/NereidsLoadScanProviderTest.java | 29 +
.../rules/analysis/DatetimeFunctionBinderTest.java | 18 +
.../rules/analysis/ExpressionAnalyzerTest.java | 23 +
.../rules/analysis/UserVariableAnalysisTest.java | 14 +
.../nereids/rules/expression/FoldConstantTest.java | 221 +-
.../rules/expression/check/CheckCastTest.java | 65 +
.../expression/rules/DateFunctionRewriteTest.java | 84 +
.../PredicateRewriteForPartitionPruneTest.java | 139 +
.../expression/rules/SimplifyCastRuleTest.java | 5 +
.../rules/SimplifyComparisonPredicateTest.java | 143 ++
.../rules/rewrite/InferPredicateByReplaceTest.java | 19 +
.../PushDownVirtualColumnsIntoOlapScanTest.java | 12 +
.../doris/nereids/stats/FilterEstimationTest.java | 28 +
.../trees/expressions/CastMonotonicityTest.java | 41 +
.../doris/nereids/trees/expressions/CastTest.java | 51 +
.../nereids/trees/expressions/LiteralTest.java | 89 +
.../functions/ComputeSignatureHelperTest.java | 48 +
.../functions/executable/DateTimeAcquireTest.java | 110 +
.../DateTimeExtractAndTransformTest.java | 76 +
.../functions/executable/StringArithmeticTest.java | 11 +
.../functions/scalar/FromUnixtimeTest.java | 56 +
.../scalar/TimestampNsFunctionSignatureTest.java | 350 +++
.../scalar/TimestampNsMonotonicityTest.java | 99 +
.../trees/expressions/literal/DateLiteralTest.java | 50 +
.../expressions/literal/DateTimeLiteralTest.java | 161 +-
.../expressions/literal/IntegerLiteralTest.java | 11 +
.../literal/TimeStampNsLiteralTest.java | 258 ++
.../plans/commands/insert/InsertUtilsTest.java | 19 +
.../apache/doris/nereids/types/DataTypeTest.java | 82 +
.../apache/doris/nereids/util/JoinUtilsTest.java | 22 +
.../doris/nereids/util/TypeCoercionMatrixTest.java | 25 +
.../doris/nereids/util/TypeCoercionUtilsTest.java | 655 +++++
.../apache/doris/planner/ConstantExpressTest.java | 4 +
.../doris/planner/GroupCommitPlannerTest.java | 59 +
.../doris/planner/QueryCacheNormalizerTest.java | 61 +-
.../doris/qe/ConnectProcessorTimestampNsTest.java | 50 +
.../apache/doris/qe/CoordinatorContextTest.java | 72 +
.../java/org/apache/doris/qe/StmtExecutorTest.java | 33 +
.../FlightSqlSchemaHelperArrowTypeTest.java | 24 +
.../doris/statistics/util/StatisticsUtilTest.java | 29 +
gensrc/proto/olap_file.proto | 2 +
gensrc/proto/types.proto | 1 +
gensrc/thrift/Descriptors.thrift | 3 +
gensrc/thrift/Exprs.thrift | 2 +
gensrc/thrift/Types.thrift | 3 +-
.../data/arrow_flight_sql_p0/test_select.out | 16 +
.../timestamp_ns/test_cast_string_non_strict.out | 18 +
.../test_from_unixtime_decimal_paths.out | 5 +
.../test_nanosecond_datev2_datetimev2.out | 14 +
...tamp_ns_additional_function_argument_matrix.out | 583 +++++
.../timestamp_ns/test_timestamp_ns_agg_state.out | 3 +
.../test_timestamp_ns_binary_output.out | 16 +
.../timestamp_ns/test_timestamp_ns_cast.out | 77 +
.../test_timestamp_ns_complex_type.out | 13 +
.../test_timestamp_ns_count_distinct.out | 6 +
.../timestamp_ns/test_timestamp_ns_delete.out | 20 +
...est_timestamp_ns_expression_argument_matrix.out | 31 +
.../timestamp_ns/test_timestamp_ns_expressions.out | 105 +
.../test_timestamp_ns_function_argument_matrix.out | 110 +
.../timestamp_ns/test_timestamp_ns_functions.out | 214 ++
.../test_timestamp_ns_generated_column.out | 5 +
.../timestamp_ns/test_timestamp_ns_group_by.out | 6 +
.../timestamp_ns/test_timestamp_ns_index.out | 44 +
.../timestamp_ns/test_timestamp_ns_join.out | 79 +
.../timestamp_ns/test_timestamp_ns_jsonb.out | 17 +
.../timestamp_ns/test_timestamp_ns_literal.out | 49 +
.../test_timestamp_ns_mixed_datetime_coercion.out | 127 +
...est_timestamp_ns_mixed_datetime_expressions.out | 403 +++
.../timestamp_ns/test_timestamp_ns_mv.out | 18 +
.../test_timestamp_ns_named_zone_gap.out | 6 +
.../test_timestamp_ns_partition_bucket.out | 49 +
.../timestamp_ns/test_timestamp_ns_primary_key.out | 18 +
.../timestamp_ns/test_timestamp_ns_rf.out | 15 +
.../timestamp_ns/test_timestamp_ns_sort.out | 25 +
.../timestamp_ns/test_timestamp_ns_statistics.out | 8 +
.../test_timestamp_ns_storage_agg_key.out | 42 +
.../test_timestamp_ns_storage_dup_key.out | 80 +
.../test_timestamp_ns_storage_uniq_key.out | 46 +
.../timestamp_ns/test_timestamp_ns_stream_load.csv | 9 +
.../timestamp_ns/test_timestamp_ns_stream_load.out | 17 +
.../test_timestamptz_storage_dup_key.out | 2 +
.../test_timestamptz_storage_uniq_key.out | 1 +
.../data/javaudf_p0/test_javaudf_timestamp_ns.out | 36 +
.../data/mtmv_p0/test_mtmv_timestamp_ns_rollup.out | 6 +
.../data/nereids_function_p0/agg_function/agg.out | 2600 ++++++++++---------
.../array/test_array_timestamp_ns.csv | 11 +
.../data/nereids_function_p0/fn_test.dat | 26 +-
.../data/nereids_function_p0/gen_function/gen.out | 104 +-
.../nereids_function_p0/scalar_function/Array.out | 852 +++++++
.../nereids_function_p0/scalar_function/Array1.out | 2696 ++++++++++++++++++--
.../nereids_function_p0/scalar_function/Array2.out | 11 +
.../data/nereids_function_p0/scalar_function/C.out | 170 +-
.../data/nereids_function_p0/scalar_function/D.out | 896 ++++++-
.../data/nereids_function_p0/scalar_function/F.out | 129 +-
.../data/nereids_function_p0/scalar_function/G.out | 33 +-
.../data/nereids_function_p0/scalar_function/H.out | 414 +++
.../data/nereids_function_p0/scalar_function/L.out | 126 +-
.../data/nereids_function_p0/scalar_function/M.out | 857 +++++++
.../data/nereids_function_p0/scalar_function/N.out | 66 +-
.../data/nereids_function_p0/scalar_function/Q.out | 29 +
.../data/nereids_function_p0/scalar_function/S.out | 650 ++++-
.../data/nereids_function_p0/scalar_function/T.out | 268 +-
.../data/nereids_function_p0/scalar_function/U.out | 29 +
.../data/nereids_function_p0/scalar_function/W.out | 525 ++++
.../data/nereids_function_p0/scalar_function/Y.out | 472 ++++
.../scalar_function/nereids_scalar_fn_map.out | 290 +++
.../test_timestamp_arithmetic.out | 384 +++
.../data/nereids_p0/show/test_show_data_types.out | 1 +
.../pythonudf_p0/test_pythonudf_timestamp_ns.out | 37 +
.../data/types/test_show_data_types.out | 1 +
.../java/org/apache/doris/udf/TimestampNsTest.java | 58 +-
.../suites/arrow_flight_sql_p0/test_select.groovy | 62 +
.../test_cast_string_non_strict.groovy | 66 +
.../test_from_unixtime_decimal_paths.groovy | 50 +
.../test_nanosecond_datev2_datetimev2.groovy | 64 +
...p_ns_additional_function_argument_matrix.groovy | 388 +++
.../test_timestamp_ns_agg_state.groovy | 42 +
.../test_timestamp_ns_binary_output.groovy | 63 +
.../timestamp_ns/test_timestamp_ns_cast.groovy | 494 ++++
.../test_timestamp_ns_complex_type.groovy | 95 +
.../test_timestamp_ns_count_distinct.groovy | 52 +
.../timestamp_ns/test_timestamp_ns_delete.groovy | 54 +
..._timestamp_ns_expression_argument_matrix.groovy | 173 ++
.../test_timestamp_ns_expressions.groovy | 285 +++
...st_timestamp_ns_function_argument_matrix.groovy | 512 ++++
.../test_timestamp_ns_functions.groovy | 708 +++++
.../test_timestamp_ns_generated_column.groovy | 41 +
.../timestamp_ns/test_timestamp_ns_group_by.groovy | 59 +-
.../timestamp_ns/test_timestamp_ns_index.groovy | 270 ++
.../timestamp_ns/test_timestamp_ns_join.groovy | 314 +++
.../timestamp_ns/test_timestamp_ns_jsonb.groovy | 53 +
.../timestamp_ns/test_timestamp_ns_literal.groovy | 242 ++
...est_timestamp_ns_mixed_datetime_coercion.groovy | 148 ++
..._timestamp_ns_mixed_datetime_expressions.groovy | 597 +++++
.../timestamp_ns/test_timestamp_ns_mv.groovy | 147 ++
.../test_timestamp_ns_named_zone_gap.groovy | 47 +-
.../test_timestamp_ns_partition_bucket.groovy | 350 +++
.../test_timestamp_ns_primary_key.groovy | 127 +
.../timestamp_ns/test_timestamp_ns_rf.groovy | 103 +
.../timestamp_ns/test_timestamp_ns_sort.groovy | 45 +
.../test_timestamp_ns_statistics.groovy | 47 +
.../test_timestamp_ns_storage_agg_key.groovy | 84 +
.../test_timestamp_ns_storage_dup_key.groovy | 119 +
.../test_timestamp_ns_storage_uniq_key.groovy | 76 +
.../test_timestamp_ns_stream_load.groovy | 75 +
...urrent_timestamp_as_column_default_value.groovy | 8 +-
.../sql-manual/sql-functions/doc_date_error.groovy | 22 +-
.../javaudf_p0/test_javaudf_timestamp_ns.groovy | 115 +
.../mtmv_p0/test_mtmv_timestamp_ns_rollup.groovy | 77 +
.../nereids_function_p0/agg_function/agg.groovy | 98 +
.../nereids_function_p0/gen_function/gen.groovy | 2 +
.../suites/nereids_function_p0/load.groovy | 33 +-
.../scalar_function/Array.groovy | 68 +-
.../scalar_function/Array1.groovy | 46 +
.../scalar_function/Array2.groovy | 24 +-
.../nereids_function_p0/scalar_function/C.groovy | 14 +
.../nereids_function_p0/scalar_function/D.groovy | 54 +
.../nereids_function_p0/scalar_function/F.groovy | 2 +
.../nereids_function_p0/scalar_function/G.groovy | 2 +
.../nereids_function_p0/scalar_function/H.groovy | 34 +-
.../nereids_function_p0/scalar_function/L.groovy | 4 +
.../nereids_function_p0/scalar_function/M.groovy | 68 +-
.../nereids_function_p0/scalar_function/N.groovy | 14 +-
.../nereids_function_p0/scalar_function/Q.groovy | 2 +
.../nereids_function_p0/scalar_function/S.groovy | 32 +-
.../nereids_function_p0/scalar_function/T.groovy | 16 +
.../nereids_function_p0/scalar_function/U.groovy | 2 +
.../nereids_function_p0/scalar_function/W.groovy | 42 +-
.../nereids_function_p0/scalar_function/Y.groovy | 38 +-
.../scalar_function/nereids_scalar_fn_map.groovy | 22 +-
.../test_timestamp_arithmetic.sql | 32 +
.../test_add_sub_diff_ceil_floor.groovy | 10 +
.../test_list_partition_datatype.groovy | 3 +-
.../test_pythonudf_timestamp_ns.groovy | 143 ++
.../datetime_functions/test_date_function.groovy | 12 +-
637 files changed, 37210 insertions(+), 3058 deletions(-)
diff --git a/be/src/agent/be_exec_version_manager.cpp b/be/src/agent/be_exec_version_manager.cpp
index a2d3d05f384..7c5c2b0b08b 100644
--- a/be/src/agent/be_exec_version_manager.cpp
+++ b/be/src/agent/be_exec_version_manager.cpp
@@ -131,8 +131,10 @@ void BeExecVersionManager::check_function_compatibility(int current_be_exec_vers
// 13: start from Doris 4.1
// a. support strict ownership hash routing for external table sink writers.
// b. support Paimon default fixed-bucket routing in the external sink exchange.
+// 14: start from master
+// a. support TIMESTAMP_NS in Thrift descriptors and PBlock exchange.
-const int BeExecVersionManager::max_be_exec_version = 13;
+const int BeExecVersionManager::max_be_exec_version = SUPPORT_TIMESTAMP_NS_VERSION;
const int BeExecVersionManager::min_be_exec_version = 0;
std::map<std::string, std::set<int>> BeExecVersionManager::_function_change_map {};
std::set<std::string> BeExecVersionManager::_function_restrict_map;
diff --git a/be/src/agent/be_exec_version_manager.h b/be/src/agent/be_exec_version_manager.h
index 5d081efb258..99fedb2f9ea 100644
--- a/be/src/agent/be_exec_version_manager.h
+++ b/be/src/agent/be_exec_version_manager.h
@@ -35,6 +35,7 @@ constexpr inline int USE_CONST_SERDE =
constexpr inline int USE_NEW_FIXED_OBJECT_SERIALIZATION_VERSION = 10;
constexpr inline int SUPPORT_ICEBERG_MERGE_CARDINALITY_VERSION = 11;
constexpr inline int SUPPORT_EXTERNAL_TABLE_SINK_HASH_VERSION = 13;
+constexpr inline int SUPPORT_TIMESTAMP_NS_VERSION = 14;
class BeExecVersionManager {
public:
diff --git a/be/src/core/call_on_type_index.h b/be/src/core/call_on_type_index.h
index 79e8407d5ac..c03a7678b37 100644
--- a/be/src/core/call_on_type_index.h
+++ b/be/src/core/call_on_type_index.h
@@ -156,6 +156,7 @@ bool call_on_basic_types(PrimitiveType type_num1, PrimitiveType type_num2, F&& f
class DataTypeDate;
class DataTypeDateV2;
class DataTypeDateTimeV2;
+class DataTypeTimeStampNs;
class DataTypeDateTime;
class DataTypeIPv4;
class DataTypeIPv6;
@@ -202,6 +203,8 @@ bool call_on_index_and_data_type(PrimitiveType number, F&& f) {
return f(TypePair<DataTypeDateV2, T>());
case PrimitiveType::TYPE_DATETIMEV2:
return f(TypePair<DataTypeDateTimeV2, T>());
+ case PrimitiveType::TYPE_TIMESTAMP_NS:
+ return f(TypePair<DataTypeTimeStampNs, T>());
case PrimitiveType::TYPE_DATETIME:
return f(TypePair<DataTypeDateTime, T>());
case PrimitiveType::TYPE_TIMEV2:
@@ -355,6 +358,8 @@ bool dispatch_type_base(PrimitiveType number, F&& f) {
return f(DispatchDataType<TYPE_DATEV2>());
case PrimitiveType::TYPE_DATETIMEV2:
return f(DispatchDataType<TYPE_DATETIMEV2>());
+ case PrimitiveType::TYPE_TIMESTAMP_NS:
+ return f(DispatchDataType<TYPE_TIMESTAMP_NS>());
case PrimitiveType::TYPE_DATETIME:
return f(DispatchDataType<TYPE_DATETIME>());
case PrimitiveType::TYPE_TIMEV2:
diff --git a/be/src/core/column/column_array.cpp b/be/src/core/column/column_array.cpp
index c018313d749..5d64273c33e 100644
--- a/be/src/core/column/column_array.cpp
+++ b/be/src/core/column/column_array.cpp
@@ -874,6 +874,9 @@ ColumnArrayDataOffsets filter_return_new_dispatch(const Filter& filt, ssize_t re
return filter_number_return_new<TYPE_DATETIME>(filt, result_size_hint, data, offsets);
if (check_and_get_column<ColumnDateTimeV2>(data.get()))
return filter_number_return_new<TYPE_DATETIMEV2>(filt, result_size_hint, data, offsets);
+ if (check_and_get_column<ColumnTimeStampNs>(data.get())) {
+ return filter_number_return_new<TYPE_TIMESTAMP_NS>(filt, result_size_hint, data, offsets);
+ }
if (check_and_get_column<ColumnTimeStampTz>(data.get()))
return filter_number_return_new<TYPE_TIMESTAMPTZ>(filt, result_size_hint, data, offsets);
if (check_and_get_column<ColumnTimeV2>(data.get()))
@@ -942,6 +945,9 @@ size_t filter_inplace_dispatch(const Filter& filter, IColumn& src_data,
return filter_number_inplace<TYPE_DATETIME>(filter, src_data, src_offsets);
if (check_and_get_column<ColumnDateTimeV2>(&src_data))
return filter_number_inplace<TYPE_DATETIMEV2>(filter, src_data, src_offsets);
+ if (check_and_get_column<ColumnTimeStampNs>(&src_data)) {
+ return filter_number_inplace<TYPE_TIMESTAMP_NS>(filter, src_data, src_offsets);
+ }
if (check_and_get_column<ColumnTimeStampTz>(&src_data))
return filter_number_inplace<TYPE_TIMESTAMPTZ>(filter, src_data, src_offsets);
if (check_and_get_column<ColumnTimeV2>(&src_data))
diff --git a/be/src/core/column/column_array.h b/be/src/core/column/column_array.h
index 46008ac99c4..8ccafcace9b 100644
--- a/be/src/core/column/column_array.h
+++ b/be/src/core/column/column_array.h
@@ -47,7 +47,7 @@ class SipHash;
ColumnFloat64, ColumnDecimal32, ColumnDecimal64, ColumnDecimal128V3, \
ColumnDecimal128V2, ColumnDecimal256
#define ALL_COLUMNS_TIME \
- ColumnDate, ColumnDateTime, ColumnDateV2, ColumnDateTimeV2, ColumnTimeStampTz
+ ColumnDate, ColumnDateTime, ColumnDateV2, ColumnDateTimeV2, ColumnTimeStampNs, ColumnTimeStampTz
#define ALL_COLUMNS_NUMERIC ALL_COLUMNS_NUMBER, ALL_COLUMNS_TIME
#define ALL_COLUMNS_SIMPLE ALL_COLUMNS_NUMERIC, ColumnString, ColumnIPv4, ColumnIPv6
diff --git a/be/src/core/column/column_vector.cpp b/be/src/core/column/column_vector.cpp
index dabe4716356..c47237ba0f9 100644
--- a/be/src/core/column/column_vector.cpp
+++ b/be/src/core/column/column_vector.cpp
@@ -517,6 +517,7 @@ template class ColumnVector<TYPE_DATEV2>;
template class ColumnVector<TYPE_DATETIME>;
template class ColumnVector<TYPE_DATETIMEV2>;
template class ColumnVector<TYPE_TIME>;
+template class ColumnVector<TYPE_TIMESTAMP_NS>;
template class ColumnVector<TYPE_TIMEV2>;
template class ColumnVector<TYPE_TIMESTAMPTZ>;
template class ColumnVector<TYPE_UINT32>;
diff --git a/be/src/core/column/column_vector.h b/be/src/core/column/column_vector.h
index 5112ad82a42..2b06ef936ca 100644
--- a/be/src/core/column/column_vector.h
+++ b/be/src/core/column/column_vector.h
@@ -47,6 +47,7 @@
#include "core/string_ref.h"
#include "core/types.h"
#include "core/uint24.h"
+#include "core/value/timestamp_ns_value.h"
#include "core/value/vdatetime_value.h"
#include "util/unaligned.h"
@@ -71,7 +72,7 @@ template <PrimitiveType T>
class ColumnVector final : public COWHelper<IColumn, ColumnVector<T>> {
static_assert(is_int_or_bool(T) || is_ip(T) || is_date_type(T) || is_float_or_double(T) ||
T == TYPE_TIME || T == TYPE_TIMEV2 || T == TYPE_UINT32 || T == TYPE_UINT64 ||
- T == TYPE_TIMESTAMPTZ);
+ T == TYPE_TIMESTAMPTZ || is_timestamp_ns_type(T));
private:
using Self = ColumnVector;
@@ -121,7 +122,7 @@ public:
void insert_range_of_integer(value_type begin, value_type end) {
if constexpr (!is_float_or_double(T) && T != TYPE_TIME && T != TYPE_TIMEV2 &&
- T != TYPE_TIMESTAMPTZ && !is_date_type(T)) {
+ T != TYPE_TIMESTAMPTZ && !is_date_type(T) && !is_timestamp_ns_type(T)) {
auto old_size = data.size();
auto new_size = old_size + static_cast<size_t>(end - begin);
data.resize(new_size);
@@ -318,7 +319,7 @@ public:
}
Int64 get_int(size_t n) const override {
- if constexpr (is_date_type(T) || T == TYPE_TIMESTAMPTZ) {
+ if constexpr (is_date_type(T) || T == TYPE_TIMESTAMPTZ || is_timestamp_ns_type(T)) {
throw doris::Exception(ErrorCode::NOT_IMPLEMENTED_ERROR,
"Method get_int is not supported for " + get_name());
return 0;
@@ -434,6 +435,7 @@ using ColumnDate = ColumnVector<TYPE_DATE>;
using ColumnDateTime = ColumnVector<TYPE_DATETIME>;
using ColumnDateV2 = ColumnVector<TYPE_DATEV2>;
using ColumnDateTimeV2 = ColumnVector<TYPE_DATETIMEV2>;
+using ColumnTimeStampNs = ColumnVector<TYPE_TIMESTAMP_NS>;
using ColumnFloat32 = ColumnVector<TYPE_FLOAT>;
using ColumnFloat64 = ColumnVector<TYPE_DOUBLE>;
using ColumnIPv4 = ColumnVector<TYPE_IPV4>;
diff --git a/be/src/core/column/columns_common.cpp b/be/src/core/column/columns_common.cpp
index bfaee36a8e0..1129032dc07 100644
--- a/be/src/core/column/columns_common.cpp
+++ b/be/src/core/column/columns_common.cpp
@@ -359,6 +359,8 @@ INSTANTIATE(DateV2Value<DateV2ValueType>, IColumn::Offset)
INSTANTIATE(DateV2Value<DateV2ValueType>, ColumnArray::Offset64)
INSTANTIATE(DateV2Value<DateTimeV2ValueType>, IColumn::Offset)
INSTANTIATE(DateV2Value<DateTimeV2ValueType>, ColumnArray::Offset64)
+INSTANTIATE(TimeStampNsValue, IColumn::Offset)
+INSTANTIATE(TimeStampNsValue, ColumnArray::Offset64)
INSTANTIATE(TimestampTzValue, IColumn::Offset)
INSTANTIATE(TimestampTzValue, ColumnArray::Offset64)
INSTANTIATE(UInt32, IColumn::Offset)
diff --git a/be/src/core/column/variant_v2/column_variant_v2_typed_column.cpp b/be/src/core/column/variant_v2/column_variant_v2_typed_column.cpp
index 85fe0be0431..1fcf0b1d8d9 100644
--- a/be/src/core/column/variant_v2/column_variant_v2_typed_column.cpp
+++ b/be/src/core/column/variant_v2/column_variant_v2_typed_column.cpp
@@ -37,6 +37,7 @@ bool is_supported_variant_typed_identity(PrimitiveType type) {
case TYPE_DATEV2:
case TYPE_DATETIME:
case TYPE_DATETIMEV2:
+ case TYPE_TIMESTAMP_NS:
case TYPE_TIMESTAMPTZ:
case TYPE_CHAR:
case TYPE_VARCHAR:
diff --git a/be/src/core/column/variant_v2/column_variant_v2_typed_column.h b/be/src/core/column/variant_v2/column_variant_v2_typed_column.h
index f5aca49a746..a37057a00be 100644
--- a/be/src/core/column/variant_v2/column_variant_v2_typed_column.h
+++ b/be/src/core/column/variant_v2/column_variant_v2_typed_column.h
@@ -117,6 +117,8 @@ void with_variant_typed_scalar(const Column& column, size_t row, uint8_t scale,
} else if constexpr (Type == TYPE_DATETIMEV2) {
const int64_t value = variant_timestamp_micros(column.get_data()[row], row, "DATETIMEV2");
callback(VariantScalarRef::timestamp_micros(value, false));
+ } else if constexpr (Type == TYPE_TIMESTAMP_NS) {
+ callback(VariantScalarRef::timestamp_nanos(column.get_data()[row].epoch_nanos(), false));
} else if constexpr (Type == TYPE_TIMESTAMPTZ) {
const int64_t value = variant_timestamp_micros(column.get_data()[row], row, "TIMESTAMPTZ");
callback(VariantScalarRef::timestamp_micros(value, true));
diff --git a/be/src/core/data_type/convert_field_to_type.cpp b/be/src/core/data_type/convert_field_to_type.cpp
index 8b430361a70..3ac74453e26 100644
--- a/be/src/core/data_type/convert_field_to_type.cpp
+++ b/be/src/core/data_type/convert_field_to_type.cpp
@@ -59,6 +59,9 @@ void dispatch(F&& f, const Field& field) {
case PrimitiveType::TYPE_DATETIMEV2:
f(field.template get<TYPE_DATETIMEV2>());
return;
+ case PrimitiveType::TYPE_TIMESTAMP_NS:
+ f(field.template get<TYPE_TIMESTAMP_NS>());
+ return;
case PrimitiveType::TYPE_TIMESTAMPTZ:
f(field.template get<TYPE_TIMESTAMPTZ>());
return;
@@ -181,6 +184,9 @@ public:
void operator()(const DateV2Value<DateTimeV2ValueType>& x, JsonbWriter* writer) const {
writer->writeInt64(*(UInt64*)&x);
}
+ void operator()(const TimeStampNsValue& x, JsonbWriter* writer) const {
+ writer->writeInt64(x.epoch_nanos());
+ }
void operator()(const TimestampTzValue& x, JsonbWriter* writer) const {
writer->writeInt64(*(UInt64*)&x);
}
diff --git a/be/src/core/data_type/data_type.cpp b/be/src/core/data_type/data_type.cpp
index 36c3c38bdf0..609f6e189ab 100644
--- a/be/src/core/data_type/data_type.cpp
+++ b/be/src/core/data_type/data_type.cpp
@@ -127,6 +127,8 @@ PGenericType_TypeId IDataType::get_pdata_type(const IDataType* data_type) {
return PGenericType::VARIANT;
case PrimitiveType::TYPE_DATETIMEV2:
return PGenericType::DATETIMEV2;
+ case PrimitiveType::TYPE_TIMESTAMP_NS:
+ return PGenericType::TIMESTAMP_NS;
case PrimitiveType::TYPE_TIMESTAMPTZ:
return PGenericType::TIMESTAMPTZ;
case PrimitiveType::TYPE_BITMAP:
diff --git a/be/src/core/data_type/data_type_factory.cpp b/be/src/core/data_type/data_type_factory.cpp
index b4bc3380a64..495c01430e0 100644
--- a/be/src/core/data_type/data_type_factory.cpp
+++ b/be/src/core/data_type/data_type_factory.cpp
@@ -59,6 +59,7 @@
#include "core/data_type/data_type_string.h"
#include "core/data_type/data_type_struct.h"
#include "core/data_type/data_type_time.h"
+#include "core/data_type/data_type_timestamp_ns.h"
#include "core/data_type/data_type_timestamptz.h"
#include "core/data_type/data_type_varbinary.h"
#include "core/data_type/data_type_variant.h"
@@ -158,6 +159,9 @@ DataTypePtr DataTypeFactory::_create_primitive_data_type(const FieldType& type,
case FieldType::OLAP_FIELD_TYPE_DATETIMEV2:
result = create_datetimev2(scale);
break;
+ case FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS:
+ result = std::make_shared<DataTypeTimeStampNs>();
+ break;
case FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ:
result = std::make_shared<DataTypeTimeStampTz>(scale);
break;
@@ -264,6 +268,9 @@ DataTypePtr DataTypeFactory::create_data_type(const PColumnMeta& pcolumn) {
case PGenericType::DATETIMEV2:
nested = std::make_shared<DataTypeDateTimeV2>(pcolumn.decimal_param().scale());
break;
+ case PGenericType::TIMESTAMP_NS:
+ nested = std::make_shared<DataTypeTimeStampNs>();
+ break;
case PGenericType::DATETIME:
nested = std::make_shared<DataTypeDateTime>();
break;
@@ -442,6 +449,9 @@ DataTypePtr DataTypeFactory::create_data_type(const PrimitiveType primitive_type
case TYPE_DATETIMEV2:
nested = create_datetimev2(scale);
break;
+ case TYPE_TIMESTAMP_NS:
+ nested = std::make_shared<DataTypeTimeStampNs>();
+ break;
case TYPE_DATETIME:
nested = std::make_shared<DataTypeDateTime>();
break;
diff --git a/be/src/core/data_type/data_type_number_base.cpp b/be/src/core/data_type/data_type_number_base.cpp
index 1dd7a19591b..0b1fc19715d 100644
--- a/be/src/core/data_type/data_type_number_base.cpp
+++ b/be/src/core/data_type/data_type_number_base.cpp
@@ -63,7 +63,7 @@ std::string DataTypeNumberBase<T>::to_string(
return std::to_string(value);
} else if constexpr (T == TYPE_DATETIME || T == TYPE_DATE) {
return std::to_string(binary_cast<doris::VecDateTimeValue, Int64>(value));
- } else if constexpr (is_date_type(T) || T == TYPE_TIMESTAMPTZ) {
+ } else if constexpr (is_date_type(T) || T == TYPE_TIMESTAMPTZ || is_timestamp_ns_type(T)) {
return std::to_string(value.to_date_int_val());
} else if constexpr (std::numeric_limits<typename PrimitiveTypeTraits<T>::CppType>::is_iec559) {
return CastToString::from_number(value);
@@ -243,8 +243,12 @@ FieldWithDataType DataTypeNumberBase<T>::get_field_with_data_type(const IColumn&
assert_cast<const ColumnVector<T>&, TypeCheckOnRelease::DISABLE>(column);
Field field;
column_data.get(row_num, field);
- return FieldWithDataType {.field = std::move(field),
- .base_scalar_type_id = get_primitive_type()};
+ auto field_with_data_type = FieldWithDataType {.field = std::move(field),
+ .base_scalar_type_id = get_primitive_type()};
+ if constexpr (T == TYPE_TIMESTAMP_NS) {
+ field_with_data_type.scale = static_cast<int>(get_scale());
+ }
+ return field_with_data_type;
}
/// Explicit template instantiations - to avoid code bloat in headers.
@@ -260,6 +264,7 @@ template class DataTypeNumberBase<TYPE_DATE>;
template class DataTypeNumberBase<TYPE_DATEV2>;
template class DataTypeNumberBase<TYPE_DATETIME>;
template class DataTypeNumberBase<TYPE_DATETIMEV2>;
+template class DataTypeNumberBase<TYPE_TIMESTAMP_NS>;
template class DataTypeNumberBase<TYPE_IPV4>;
template class DataTypeNumberBase<TYPE_IPV6>;
template class DataTypeNumberBase<TYPE_TIME>;
diff --git a/be/src/core/data_type/data_type_number_base.h b/be/src/core/data_type/data_type_number_base.h
index eb8690c1516..5091c548cbe 100644
--- a/be/src/core/data_type/data_type_number_base.h
+++ b/be/src/core/data_type/data_type_number_base.h
@@ -49,7 +49,8 @@ class IColumn;
template <PrimitiveType T>
class DataTypeNumberBase : public IDataType {
static_assert(is_int_or_bool(T) || is_ip(T) || is_date_type(T) || is_float_or_double(T) ||
- T == TYPE_TIME || T == TYPE_TIMEV2 || T == TYPE_TIMESTAMPTZ);
+ T == TYPE_TIME || T == TYPE_TIMEV2 || T == TYPE_TIMESTAMPTZ ||
+ is_timestamp_ns_type(T));
public:
static constexpr bool is_parametric = false;
diff --git a/be/src/core/data_type/data_type_time.h b/be/src/core/data_type/data_type_time.h
index afcc97fe8e1..1a0e083a4ee 100644
--- a/be/src/core/data_type/data_type_time.h
+++ b/be/src/core/data_type/data_type_time.h
@@ -40,8 +40,10 @@ class IColumn;
class DataTypeTimeV2 final : public DataTypeNumberBase<PrimitiveType::TYPE_TIMEV2> {
public:
+ static constexpr UInt32 MAX_SCALE = 6;
+
DataTypeTimeV2(int scale = 0) : _scale(scale) {
- if (UNLIKELY(scale > 6)) {
+ if (UNLIKELY(scale > static_cast<int>(MAX_SCALE))) {
throw doris::Exception(ErrorCode::INTERNAL_ERROR, "Scale {} is out of bounds", scale);
}
if (scale == -1) {
diff --git a/be/src/core/data_type/data_type_timestamp_ns.cpp b/be/src/core/data_type/data_type_timestamp_ns.cpp
new file mode 100644
index 00000000000..5bcfa3aa383
--- /dev/null
+++ b/be/src/core/data_type/data_type_timestamp_ns.cpp
@@ -0,0 +1,48 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "core/data_type/data_type_timestamp_ns.h"
+
+#include <typeinfo>
+
+#include "common/exception.h"
+#include "core/data_type_serde/data_type_timestamp_ns_serde.h"
+#include "core/string_ref.h"
+
+namespace doris {
+
+DataTypeSerDeSPtr DataTypeTimeStampNs::get_serde(int nesting_level) const {
+ return std::make_shared<DataTypeTimeStampNsSerDe>(nesting_level);
+}
+
+Field DataTypeTimeStampNs::get_field(const TExprNode& node) const {
+ TimeStampNsValue value;
+ const StringRef string_value(node.date_literal.value.data(), node.date_literal.value.size());
+ const auto status = parse_timestamp_ns(string_value, &value);
+ if (!status.ok()) {
+ throw doris::Exception(doris::ErrorCode::INVALID_ARGUMENT,
+ "Invalid value: {} for type TimeStampNs: {}",
+ node.date_literal.value, status.to_string());
+ }
+ return Field::create_field<TYPE_TIMESTAMP_NS>(value);
+}
+
+bool DataTypeTimeStampNs::equals(const IDataType& rhs) const {
+ return typeid(rhs) == typeid(*this);
+}
+
+} // namespace doris
diff --git a/be/test/testutil/datetime_ut_util.h b/be/src/core/data_type/data_type_timestamp_ns.h
similarity index 54%
copy from be/test/testutil/datetime_ut_util.h
copy to be/src/core/data_type/data_type_timestamp_ns.h
index 9f0967ce1d7..984557b235a 100644
--- a/be/test/testutil/datetime_ut_util.h
+++ b/be/src/core/data_type/data_type_timestamp_ns.h
@@ -17,21 +17,24 @@
#pragma once
-#include "core/value/timestamptz_value.h"
-#include "core/value/vdatetime_value.h"
+#include <memory>
+#include <string>
+
+#include "core/data_type/data_type_number_base.h"
+#include "core/value/timestamp_ns_value.h"
+
namespace doris {
-inline auto make_datetime(int year, int month, int day, int hour, int minute, int second,
- int microsecond) {
- DateV2Value<DateTimeV2ValueType> dt;
- dt.unchecked_set_time(year, month, day, hour, minute, second, microsecond);
- return dt;
-}
-
-inline auto make_timestamptz(int year, int month, int day, int hour, int minute, int second,
- int microsecond) {
- DateV2Value<DateTimeV2ValueType> dt;
- dt.unchecked_set_time(year, month, day, hour, minute, second, microsecond);
- return TimestampTzValue(dt.to_date_int_val());
-}
-
-} // namespace doris
\ No newline at end of file
+
+class DataTypeTimeStampNs final : public DataTypeNumberBase<PrimitiveType::TYPE_TIMESTAMP_NS> {
+public:
+ const std::string get_family_name() const override { return "TimeStampNs"; }
+
+ bool equals(const IDataType& rhs) const override;
+
+ DataTypeSerDeSPtr get_serde(int nesting_level = 1) const override;
+
+ Field get_field(const TExprNode& node) const override;
+ UInt32 get_scale() const override { return TimeStampNsValue::FRACTIONAL_DIGITS; }
+};
+
+} // namespace doris
diff --git a/be/src/core/data_type/define_primitive_type.h b/be/src/core/data_type/define_primitive_type.h
index 852b68a96b5..6606242cf1d 100644
--- a/be/src/core/data_type/define_primitive_type.h
+++ b/be/src/core/data_type/define_primitive_type.h
@@ -73,7 +73,8 @@ enum PrimitiveType : PrimitiveNative {
TYPE_UINT64, /* 39, used as offset */
TYPE_FIXED_LENGTH_OBJECT, /* 40, represent fixed-length object on BE */
TYPE_VARBINARY, /* 41, varbinary */
- TYPE_TIMESTAMPTZ /* 42, timestamptz */
+ TYPE_TIMESTAMPTZ, /* 42, timestamptz */
+ TYPE_TIMESTAMP_NS /* 43, signed Int64 epoch nanoseconds */
};
} // namespace doris
diff --git a/be/src/core/data_type/primitive_type.cpp b/be/src/core/data_type/primitive_type.cpp
index 746fc655323..208e9e9f411 100644
--- a/be/src/core/data_type/primitive_type.cpp
+++ b/be/src/core/data_type/primitive_type.cpp
@@ -69,6 +69,9 @@ PrimitiveType thrift_to_type(TPrimitiveType::type ttype) {
case TPrimitiveType::DATETIMEV2:
return TYPE_DATETIMEV2;
+ case TPrimitiveType::TIMESTAMP_NS:
+ return TYPE_TIMESTAMP_NS;
+
case TPrimitiveType::TIMEV2:
return TYPE_TIMEV2;
@@ -191,6 +194,9 @@ TPrimitiveType::type to_thrift(PrimitiveType ptype) {
case TYPE_DATETIMEV2:
return TPrimitiveType::DATETIMEV2;
+ case TYPE_TIMESTAMP_NS:
+ return TPrimitiveType::TIMESTAMP_NS;
+
case TYPE_TIMEV2:
return TPrimitiveType::TIMEV2;
@@ -305,6 +311,9 @@ std::string type_to_string(PrimitiveType t) {
case TYPE_DATETIMEV2:
return "DATETIMEV2";
+ case TYPE_TIMESTAMP_NS:
+ return "TIMESTAMP_NS";
+
case TYPE_TIMEV2:
return "TIMEV2";
diff --git a/be/src/core/data_type/primitive_type.h b/be/src/core/data_type/primitive_type.h
index b46c1e11ba3..cba1fe9c1d5 100644
--- a/be/src/core/data_type/primitive_type.h
+++ b/be/src/core/data_type/primitive_type.h
@@ -36,6 +36,7 @@
#include "exec/common/template_helpers.hpp"
namespace doris {
+class TimeStampNsValue;
#include "common/compile_check_begin.h"
template <typename T>
class ColumnStr;
@@ -74,6 +75,7 @@ class DataTypeTimeV2;
class DataTypeDateTime;
class DataTypeDate;
class DataTypeDateTimeV2;
+class DataTypeTimeStampNs;
class DataTypeDateV2;
class DataTypeTimeStampTz;
template <PrimitiveType T>
@@ -111,6 +113,7 @@ using ColumnDateTime = ColumnVector<TYPE_DATETIME>;
using ColumnDateV2 = ColumnVector<TYPE_DATEV2>;
using ColumnTimeStampTz = ColumnVector<TYPE_TIMESTAMPTZ>;
using ColumnDateTimeV2 = ColumnVector<TYPE_DATETIMEV2>;
+using ColumnTimeStampNs = ColumnVector<TYPE_TIMESTAMP_NS>;
using ColumnFloat32 = ColumnVector<TYPE_FLOAT>;
using ColumnFloat64 = ColumnVector<TYPE_DOUBLE>;
using ColumnIPv4 = ColumnVector<TYPE_IPV4>;
@@ -140,6 +143,7 @@ constexpr bool is_enumeration_type(PrimitiveType type) {
case TYPE_STRING:
case TYPE_DATETIME:
case TYPE_DATETIMEV2:
+ case TYPE_TIMESTAMP_NS:
case TYPE_TIMESTAMPTZ:
case TYPE_TIMEV2:
case TYPE_DECIMALV2:
@@ -178,6 +182,10 @@ constexpr bool is_date_type(PrimitiveType type) {
type == TYPE_DATEV2;
}
+constexpr bool is_timestamp_ns_type(PrimitiveType type) {
+ return type == TYPE_TIMESTAMP_NS;
+}
+
constexpr bool is_time_type(PrimitiveType type) {
return type == TYPE_TIME || type == TYPE_TIMEV2;
}
@@ -373,6 +381,13 @@ struct PrimitiveTypeTraits<TYPE_DATETIMEV2> {
using ColumnType = ColumnDateTimeV2;
};
template <>
+struct PrimitiveTypeTraits<TYPE_TIMESTAMP_NS> {
+ using CppType = TimeStampNsValue;
+ using StorageFieldType = int64_t;
+ using DataType = DataTypeTimeStampNs;
+ using ColumnType = ColumnTimeStampNs;
+};
+template <>
struct PrimitiveTypeTraits<TYPE_DATEV2> {
using CppType = DateV2Value<DateV2ValueType>;
using StorageFieldType = uint32_t;
diff --git a/be/src/core/data_type/storage_field_type.cpp b/be/src/core/data_type/storage_field_type.cpp
index 269015401dc..ec10e25ee68 100644
--- a/be/src/core/data_type/storage_field_type.cpp
+++ b/be/src/core/data_type/storage_field_type.cpp
@@ -73,6 +73,8 @@ FieldType primitive_type_to_storage_field_type(PrimitiveType type) {
return FieldType::OLAP_FIELD_TYPE_DATEV2;
case PrimitiveType::TYPE_DATETIMEV2:
return FieldType::OLAP_FIELD_TYPE_DATETIMEV2;
+ case PrimitiveType::TYPE_TIMESTAMP_NS:
+ return FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS;
case PrimitiveType::TYPE_TIMEV2:
return FieldType::OLAP_FIELD_TYPE_TIMEV2;
case PrimitiveType::TYPE_DECIMAL32:
@@ -167,6 +169,8 @@ PrimitiveType storage_field_type_to_primitive_type(FieldType type) {
return PrimitiveType::TYPE_DATEV2;
case FieldType::OLAP_FIELD_TYPE_DATETIMEV2:
return PrimitiveType::TYPE_DATETIMEV2;
+ case FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS:
+ return PrimitiveType::TYPE_TIMESTAMP_NS;
case FieldType::OLAP_FIELD_TYPE_TIMEV2:
return PrimitiveType::TYPE_TIMEV2;
case FieldType::OLAP_FIELD_TYPE_DECIMAL32:
diff --git a/be/src/core/data_type_serde/data_type_number_serde.cpp b/be/src/core/data_type_serde/data_type_number_serde.cpp
index 8ea6c7bfe6b..5841abf4c68 100644
--- a/be/src/core/data_type_serde/data_type_number_serde.cpp
+++ b/be/src/core/data_type_serde/data_type_number_serde.cpp
@@ -734,6 +734,13 @@ Status DataTypeNumberSerDe<T>::write_column_to_arrow(const IColumn& column, cons
return Status::OK();
}
+template <>
+Status DataTypeNumberSerDe<TYPE_TIMESTAMP_NS>::write_column_to_arrow(
+ const IColumn& column, const NullMap* null_map, arrow::ArrayBuilder* array_builder,
+ int64_t start, int64_t end, const cctz::time_zone& ctz) const {
+ return Status::NotSupported("DataTypeNumberSerDe<TYPE_TIMESTAMP_NS>::write_column_to_arrow");
+}
+
template <PrimitiveType T>
Status DataTypeNumberSerDe<T>::read_column_from_decoded_values(
IColumn& column, const DecodedColumnView& view) const {
@@ -1115,8 +1122,8 @@ template <PrimitiveType T>
constexpr bool can_write_to_jsonb_from_number() {
return T == TYPE_BOOLEAN || T == TYPE_TINYINT || T == TYPE_SMALLINT || T == TYPE_INT ||
T == TYPE_BIGINT || T == TYPE_LARGEINT || T == TYPE_FLOAT || T == TYPE_DOUBLE ||
- T == TYPE_DATEV2 || T == TYPE_DATETIMEV2 || T == TYPE_TIMESTAMPTZ || T == TYPE_IPV4 ||
- T == TYPE_IPV6 || T == TYPE_TIMEV2;
+ T == TYPE_DATEV2 || T == TYPE_DATETIMEV2 || T == TYPE_TIMESTAMP_NS ||
+ T == TYPE_TIMESTAMPTZ || T == TYPE_IPV4 || T == TYPE_IPV6 || T == TYPE_TIMEV2;
}
template <PrimitiveType T>
@@ -1154,6 +1161,8 @@ bool write_to_jsonb_from_number(auto& data, JsonbWriter& writer, int scale) {
return jsonb_writer_string(writer, CastToString::from_datev2(data));
} else if constexpr (T == TYPE_DATETIMEV2) {
return jsonb_writer_string(writer, CastToString::from_datetimev2(data, scale));
+ } else if constexpr (T == TYPE_TIMESTAMP_NS) {
+ return jsonb_writer_string(writer, CastToString::from_timestamp_ns(data));
} else if constexpr (T == TYPE_TIMESTAMPTZ) {
return jsonb_writer_string(writer, CastToString::from_timestamptz(data, scale));
} else if constexpr (T == TYPE_IPV4) {
@@ -1440,6 +1449,8 @@ void DataTypeNumberSerDe<T>::read_one_cell_from_jsonb(IColumn& column,
col.insert_value(binary_cast<Int64, VecDateTimeValue>(arg->unpack<JsonbInt64Val>()->val()));
} else if constexpr (T == TYPE_BIGINT) {
col.insert_value(arg->unpack<JsonbInt64Val>()->val());
+ } else if constexpr (T == TYPE_TIMESTAMP_NS) {
+ col.insert_value(TimeStampNsValue(arg->unpack<JsonbInt64Val>()->val()));
} else if constexpr (T == TYPE_LARGEINT) {
col.insert_value(arg->unpack<JsonbInt128Val>()->val());
} else if constexpr (T == TYPE_FLOAT) {
@@ -1473,7 +1484,7 @@ void DataTypeNumberSerDe<T>::write_one_cell_to_jsonb(const IColumn& column,
int32_t val = *reinterpret_cast<const int32_t*>(data_ref.data);
result.writeInt32(val);
} else if constexpr (T == TYPE_BIGINT || T == TYPE_DATE || T == TYPE_DATETIME ||
- T == TYPE_DATETIMEV2 || T == TYPE_TIMESTAMPTZ) {
+ T == TYPE_DATETIMEV2 || T == TYPE_TIMESTAMP_NS || T == TYPE_TIMESTAMPTZ) {
int64_t val = *reinterpret_cast<const int64_t*>(data_ref.data);
result.writeInt64(val);
} else if constexpr (T == TYPE_LARGEINT) {
@@ -1714,6 +1725,10 @@ const uint8_t* DataTypeNumberSerDe<T>::deserialize_binary_to_column(const uint8_
col.insert_value(binary_cast<UInt64, DateV2Value<DateTimeV2ValueType>>(
unaligned_load<UInt64>(data)));
data += sizeof(UInt64);
+ } else if constexpr (T == TYPE_TIMESTAMP_NS) {
+ data += sizeof(uint8_t);
+ col.insert_value(TimeStampNsValue(unaligned_load<Int64>(data)));
+ data += sizeof(Int64);
} else {
throw doris::Exception(ErrorCode::NOT_IMPLEMENTED_ERROR,
"deserialize_binary_to_column with type '{}'", type_to_string(T));
@@ -1782,6 +1797,14 @@ const uint8_t* DataTypeNumberSerDe<T>::deserialize_binary_to_field(const uint8_t
info.scale = static_cast<int>(scale);
field = Field::create_field<T>(*(typename PrimitiveTypeTraits<T>::CppType*)&v);
data += sizeof(UInt64);
+ } else if constexpr (T == TYPE_TIMESTAMP_NS) {
+ const uint8_t scale = *data;
+ data += sizeof(uint8_t);
+ info.precision = -1;
+ info.scale = static_cast<int>(scale);
+ field = Field::create_field<TYPE_TIMESTAMP_NS>(
+ TimeStampNsValue(unaligned_load<Int64>(data)));
+ data += sizeof(Int64);
} else {
throw doris::Exception(ErrorCode::NOT_IMPLEMENTED_ERROR,
"deserialize_binary_to_column with type '{}'", type_to_string(T));
@@ -1800,6 +1823,8 @@ void value_to_string(const typename PrimitiveTypeTraits<T>::CppType value, Buffe
CastToString::push_datev2(value, bw);
} else if constexpr (T == TYPE_DATETIMEV2) {
CastToString::push_datetimev2(value, scale, bw);
+ } else if constexpr (T == TYPE_TIMESTAMP_NS) {
+ CastToString::push_timestamp_ns(value, bw);
} else if constexpr (T == TYPE_TIMESTAMPTZ) {
CastToString::push_timestamptz(value, scale, bw, options);
} else if constexpr (T == TYPE_TIME || T == TYPE_TIMEV2) {
@@ -1816,7 +1841,8 @@ template <PrimitiveType T>
void DataTypeNumberSerDe<T>::to_string(const IColumn& column, size_t row_num, BufferWritable& bw,
const FormatOptions& options) const {
auto& data = assert_cast<const ColumnType&, TypeCheckOnRelease::DISABLE>(column).get_data();
- if constexpr (is_timestamptz_type(T) || is_date_type(T) || is_time_type(T) || is_ip(T)) {
+ if constexpr (is_timestamptz_type(T) || is_date_type(T) || is_timestamp_ns_type(T) ||
+ is_time_type(T) || is_ip(T)) {
if (_nesting_level > 1) {
bw.write('"');
}
@@ -1843,7 +1869,8 @@ bool DataTypeNumberSerDe<T>::write_column_to_hive_text(const IColumn& column, Bu
int64_t row_idx,
const FormatOptions& options) const {
auto& data = assert_cast<const ColumnType&, TypeCheckOnRelease::DISABLE>(column).get_data();
- if constexpr (is_date_type(T) || is_timestamptz_type(T) || is_time_type(T) || is_ip(T)) {
+ if constexpr (is_date_type(T) || is_timestamptz_type(T) || is_timestamp_ns_type(T) ||
+ is_time_type(T) || is_ip(T)) {
if (_nesting_level > 1) {
bw.write('"');
}
@@ -1926,6 +1953,7 @@ template class DataTypeNumberSerDe<TYPE_DATE>;
template class DataTypeNumberSerDe<TYPE_DATEV2>;
template class DataTypeNumberSerDe<TYPE_DATETIME>;
template class DataTypeNumberSerDe<TYPE_DATETIMEV2>;
+template class DataTypeNumberSerDe<TYPE_TIMESTAMP_NS>;
template class DataTypeNumberSerDe<TYPE_IPV4>;
template class DataTypeNumberSerDe<TYPE_IPV6>;
template class DataTypeNumberSerDe<TYPE_TIME>;
diff --git a/be/src/core/data_type_serde/data_type_number_serde.h b/be/src/core/data_type_serde/data_type_number_serde.h
index 571888ff34b..9401b5bab6c 100644
--- a/be/src/core/data_type_serde/data_type_number_serde.h
+++ b/be/src/core/data_type_serde/data_type_number_serde.h
@@ -50,7 +50,8 @@ class Arena;
template <PrimitiveType T>
class DataTypeNumberSerDe : public DataTypeSerDe {
static_assert(is_int_or_bool(T) || is_ip(T) || is_date_type(T) || is_float_or_double(T) ||
- T == TYPE_TIME || T == TYPE_TIMEV2 || T == TYPE_TIMESTAMPTZ);
+ T == TYPE_TIME || T == TYPE_TIMEV2 || T == TYPE_TIMESTAMPTZ ||
+ is_timestamp_ns_type(T));
public:
using ColumnType = typename PrimitiveTypeTraits<T>::ColumnType;
@@ -250,6 +251,12 @@ Status DataTypeNumberSerDe<T>::read_column_from_pb(IColumn& column, const PValue
for (int i = 0; i < arg.int64_value_size(); ++i) {
data[old_column_size + i] = arg.int64_value(i);
}
+ } else if constexpr (T == TYPE_TIMESTAMP_NS) {
+ column.resize(old_column_size + arg.int64_value_size());
+ auto& data = reinterpret_cast<ColumnType&>(column).get_data();
+ for (int i = 0; i < arg.int64_value_size(); ++i) {
+ data[old_column_size + i] = TimeStampNsValue(arg.int64_value(i));
+ }
} else if constexpr (T == TYPE_FLOAT) {
column.resize(old_column_size + arg.float_value_size());
auto& data = reinterpret_cast<ColumnType&>(column).get_data();
@@ -335,6 +342,12 @@ Status DataTypeNumberSerDe<T>::write_column_to_pb(const IColumn& column, PValues
auto* values = result.mutable_int64_value();
values->Reserve(row_count);
values->Add(data.begin() + start, data.begin() + end);
+ } else if constexpr (T == TYPE_TIMESTAMP_NS) {
+ ptype->set_id(PGenericType::INT64);
+ auto* values = result.mutable_int64_value();
+ values->Reserve(row_count);
+ values->Add(reinterpret_cast<const int64_t*>(data.begin()) + start,
+ reinterpret_cast<const int64_t*>(data.begin()) + end);
} else if constexpr (T == TYPE_FLOAT) {
ptype->set_id(PGenericType::FLOAT);
auto* values = result.mutable_float_value();
@@ -351,5 +364,10 @@ Status DataTypeNumberSerDe<T>::write_column_to_pb(const IColumn& column, PValues
return Status::OK();
}
+template <>
+Status DataTypeNumberSerDe<TYPE_TIMESTAMP_NS>::write_column_to_arrow(
+ const IColumn& column, const NullMap* null_map, arrow::ArrayBuilder* array_builder,
+ int64_t start, int64_t end, const cctz::time_zone& ctz) const;
+
#include "common/compile_check_end.h"
} // namespace doris
diff --git a/be/src/core/data_type_serde/data_type_serde.cpp b/be/src/core/data_type_serde/data_type_serde.cpp
index b78e5615aa7..56d99d99940 100644
--- a/be/src/core/data_type_serde/data_type_serde.cpp
+++ b/be/src/core/data_type_serde/data_type_serde.cpp
@@ -296,6 +296,7 @@ const uint8_t* DataTypeSerDe::deserialize_binary_to_column(const uint8_t* data,
HANDLE_T_NUM_SERDE(OLAP_FIELD_TYPE_DATEV2, TYPE_DATEV2)
HANDLE_T_NUM_SERDE(OLAP_FIELD_TYPE_DATETIMEV2, TYPE_DATETIMEV2)
HANDLE_T_DEC_SERDE(OLAP_FIELD_TYPE_DECIMAL, TYPE_DECIMALV2)
+ HANDLE_T_NUM_SERDE(OLAP_FIELD_TYPE_TIMESTAMP_NS, TYPE_TIMESTAMP_NS)
HANDLE_T_DEC_SERDE(OLAP_FIELD_TYPE_DECIMAL32, TYPE_DECIMAL32)
HANDLE_T_DEC_SERDE(OLAP_FIELD_TYPE_DECIMAL64, TYPE_DECIMAL64)
HANDLE_T_DEC_SERDE(OLAP_FIELD_TYPE_DECIMAL128I, TYPE_DECIMAL128I)
@@ -361,6 +362,7 @@ const uint8_t* DataTypeSerDe::deserialize_binary_to_field(const uint8_t* data, F
HANDLE_T_NUM_SERDE(OLAP_FIELD_TYPE_DATEV2, TYPE_DATEV2)
HANDLE_T_NUM_SERDE(OLAP_FIELD_TYPE_DATETIMEV2, TYPE_DATETIMEV2)
HANDLE_T_DEC_SERDE(OLAP_FIELD_TYPE_DECIMAL, TYPE_DECIMALV2)
+ HANDLE_T_NUM_SERDE(OLAP_FIELD_TYPE_TIMESTAMP_NS, TYPE_TIMESTAMP_NS)
HANDLE_T_DEC_SERDE(OLAP_FIELD_TYPE_DECIMAL32, TYPE_DECIMAL32)
HANDLE_T_DEC_SERDE(OLAP_FIELD_TYPE_DECIMAL64, TYPE_DECIMAL64)
HANDLE_T_DEC_SERDE(OLAP_FIELD_TYPE_DECIMAL128I, TYPE_DECIMAL128I)
diff --git a/be/src/core/data_type_serde/data_type_timestamp_ns_serde.cpp b/be/src/core/data_type_serde/data_type_timestamp_ns_serde.cpp
new file mode 100644
index 00000000000..a7b47247e14
--- /dev/null
+++ b/be/src/core/data_type_serde/data_type_timestamp_ns_serde.cpp
@@ -0,0 +1,505 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "core/data_type_serde/data_type_timestamp_ns_serde.h"
+
+#include <arrow/builder.h>
+#include <cctz/time_zone.h>
+
+#include <string>
+
+#include "common/config.h"
+#include "common/exception.h"
+#include "core/assert_cast.h"
+#include "core/column/column_nullable.h"
+#include "core/column/column_string.h"
+#include "core/data_type_serde/arrow_validation.h"
+#include "core/value/vdatetime_value.h"
+#include "exprs/function/cast/cast_to_timestamp_ns.h"
+#include "util/mysql_row_buffer.h"
+#include "util/unaligned.h"
+
+namespace doris {
+
+namespace {
+
+Status get_nanos_per_arrow_timestamp_unit(arrow::TimeUnit::type unit, int64_t* nanos_per_unit) {
+ switch (unit) {
+ case arrow::TimeUnit::SECOND:
+ *nanos_per_unit = TimeStampNsValue::NANOS_PER_SECOND;
+ break;
+ case arrow::TimeUnit::MILLI:
+ *nanos_per_unit = TimeStampNsValue::NANOS_PER_MILLISECOND;
+ break;
+ case arrow::TimeUnit::MICRO:
+ *nanos_per_unit = TimeStampNsValue::NANOS_PER_MICROSECOND;
+ break;
+ case arrow::TimeUnit::NANO:
+ *nanos_per_unit = 1;
+ break;
+ default:
+ return Status::InvalidArgument("Unsupported Arrow timestamp unit: {}",
+ static_cast<int>(unit));
+ }
+ return Status::OK();
+}
+
+template <DatelikeParseMode ParseMode>
+Status parse_timestamp_ns_impl(StringRef str, TimeStampNsValue& value,
+ const cctz::time_zone* local_time_zone) {
+ constexpr bool IsStrict = is_datelike_parse_strict(ParseMode);
+ CastParameters params {.status = Status::OK(), .is_strict = IsStrict};
+ bool parsed = false;
+ if constexpr (IsStrict) {
+ parsed = CastToTimestampNs::from_string_strict_mode<ParseMode>(
+ str, value, local_time_zone, TimeStampNsValue::FRACTIONAL_DIGITS, params);
+ } else {
+ parsed = CastToTimestampNs::from_string_non_strict_mode(
+ str, value, local_time_zone, TimeStampNsValue::FRACTIONAL_DIGITS, params);
+ }
+ if (!parsed) {
+ if (!params.status.ok()) {
+ return Status::InvalidArgument("Invalid TIMESTAMP_NS value '{}': {}",
+ std::string(str.data, str.size),
+ params.status.to_string());
+ }
+ return Status::InvalidArgument("Invalid TIMESTAMP_NS value '{}'",
+ std::string(str.data, str.size));
+ }
+
+ return Status::OK();
+}
+
+Status parse_timestamp_ns_non_strict(StringRef str, TimeStampNsValue& value,
+ const cctz::time_zone* local_time_zone) {
+ return parse_timestamp_ns_impl<DatelikeParseMode::NON_STRICT>(str, value, local_time_zone);
+}
+
+} // namespace
+
+Status parse_timestamp_ns(StringRef str, TimeStampNsValue* value,
+ const cctz::time_zone* local_time_zone) {
+ return parse_timestamp_ns_impl<DatelikeParseMode::STRICT>(str, *value, local_time_zone);
+}
+
+Status DataTypeTimeStampNsSerDe::from_string_batch(const ColumnString& strings,
+ ColumnNullable& result,
+ const FormatOptions& options) const {
+ auto& data = assert_cast<ColumnTimeStampNs&>(result.get_nested_column()).get_data();
+ auto& null_map = result.get_null_map_column().get_data();
+ result.resize(strings.size());
+ for (size_t i = 0; i < strings.size(); ++i) {
+ TimeStampNsValue value;
+ const auto status =
+ parse_timestamp_ns_non_strict(strings.get_data_at(i), value, options.timezone);
+ null_map[i] = !status.ok();
+ data[i] = value;
+ }
+ return Status::OK();
+}
+
+Status DataTypeTimeStampNsSerDe::from_string_strict_mode_batch(
+ const ColumnString& strings, IColumn& result, const FormatOptions& options,
+ const NullMap::value_type* null_map) const {
+ auto& data = assert_cast<ColumnTimeStampNs&>(result).get_data();
+ result.resize(strings.size());
+ for (size_t i = 0; i < strings.size(); ++i) {
+ if (null_map != nullptr && null_map[i]) {
+ continue;
+ }
+ TimeStampNsValue value;
+ RETURN_IF_ERROR(parse_timestamp_ns(strings.get_data_at(i), &value, options.timezone));
+ data[i] = value;
+ }
+ return Status::OK();
+}
+
+Status DataTypeTimeStampNsSerDe::from_string(StringRef& str, IColumn& column,
+ const FormatOptions& options) const {
+ TimeStampNsValue value;
+ RETURN_IF_ERROR(parse_timestamp_ns_non_strict(str, value, options.timezone));
+ assert_cast<ColumnTimeStampNs&>(column).insert_value(value);
+ return Status::OK();
+}
+
+Status DataTypeTimeStampNsSerDe::from_string_strict_mode(StringRef& str, IColumn& column,
+ const FormatOptions& options) const {
+ TimeStampNsValue value;
+ RETURN_IF_ERROR(parse_timestamp_ns(str, &value, options.timezone));
+ assert_cast<ColumnTimeStampNs&>(column).insert_value(value);
+ return Status::OK();
+}
+
+template <typename IntDataType>
+Status DataTypeTimeStampNsSerDe::from_int_batch(const typename IntDataType::ColumnType& int_col,
+ ColumnNullable& target_col) const {
+ auto& col_data = assert_cast<ColumnTimeStampNs&>(target_col.get_nested_column());
+ auto& col_nullmap = target_col.get_null_map_column();
+ col_data.resize(int_col.size());
+ col_nullmap.resize(int_col.size());
+
+ CastParameters params {.status = Status::OK(), .is_strict = false};
+ for (size_t i = 0; i < int_col.size(); ++i) {
+ TimeStampNsValue value;
+ if (CastToTimestampNs::from_integer<DatelikeParseMode::NON_STRICT>(
+ int_col.get_element(i), value, params)) [[likely]] {
+ col_data.get_data()[i] = value;
+ col_nullmap.get_data()[i] = false;
+ } else {
+ col_data.get_data()[i] = TimeStampNsValue();
+ col_nullmap.get_data()[i] = true;
+ }
+ }
+ return Status::OK();
+}
+
+template <typename IntDataType>
+Status DataTypeTimeStampNsSerDe::from_int_strict_mode_batch(
+ const typename IntDataType::ColumnType& int_col, IColumn& target_col) const {
+ auto& col_data = assert_cast<ColumnTimeStampNs&>(target_col);
+ col_data.resize(int_col.size());
+
+ CastParameters params {.status = Status::OK(), .is_strict = true};
+ for (size_t i = 0; i < int_col.size(); ++i) {
+ TimeStampNsValue value;
+ CastToTimestampNs::from_integer<DatelikeParseMode::STRICT>(int_col.get_element(i), value,
+ params);
+ if (!params.status.ok()) [[unlikely]] {
+ params.status.prepend(
+ fmt::format("parse {} to timestamp_ns failed: ", int_col.get_element(i)));
+ return params.status;
+ }
+ col_data.get_data()[i] = value;
+ }
+ return Status::OK();
+}
+
+template <typename FloatDataType>
+Status DataTypeTimeStampNsSerDe::from_float_batch(
+ const typename FloatDataType::ColumnType& float_col, ColumnNullable& target_col) const {
+ auto& col_data = assert_cast<ColumnTimeStampNs&>(target_col.get_nested_column());
+ auto& col_nullmap = target_col.get_null_map_column();
+ col_data.resize(float_col.size());
+ col_nullmap.resize(float_col.size());
+
+ CastParameters params {.status = Status::OK(), .is_strict = false};
+ for (size_t i = 0; i < float_col.size(); ++i) {
+ TimeStampNsValue value;
+ if (CastToTimestampNs::from_float<DatelikeParseMode::NON_STRICT>(
+ float_col.get_data()[i], value, params)) [[likely]] {
+ col_data.get_data()[i] = value;
+ col_nullmap.get_data()[i] = false;
+ } else {
+ col_data.get_data()[i] = TimeStampNsValue();
+ col_nullmap.get_data()[i] = true;
+ }
+ }
+ return Status::OK();
+}
+
+template <typename FloatDataType>
+Status DataTypeTimeStampNsSerDe::from_float_strict_mode_batch(
+ const typename FloatDataType::ColumnType& float_col, IColumn& target_col) const {
+ auto& col_data = assert_cast<ColumnTimeStampNs&>(target_col);
+ col_data.resize(float_col.size());
+
+ CastParameters params {.status = Status::OK(), .is_strict = true};
+ for (size_t i = 0; i < float_col.size(); ++i) {
+ TimeStampNsValue value;
+ CastToTimestampNs::from_float<DatelikeParseMode::STRICT>(float_col.get_data()[i], value,
+ params);
+ if (!params.status.ok()) [[unlikely]] {
+ params.status.prepend(
+ fmt::format("parse {} to timestamp_ns failed: ", float_col.get_data()[i]));
+ return params.status;
+ }
+ col_data.get_data()[i] = value;
+ }
+ return Status::OK();
+}
+
+template <typename DecimalDataType>
+Status DataTypeTimeStampNsSerDe::from_decimal_batch(
+ const typename DecimalDataType::ColumnType& decimal_col, ColumnNullable& target_col) const {
+ auto& col_data = assert_cast<ColumnTimeStampNs&>(target_col.get_nested_column());
+ auto& col_nullmap = target_col.get_null_map_column();
+ col_data.resize(decimal_col.size());
+ col_nullmap.resize(decimal_col.size());
+
+ CastParameters params {.status = Status::OK(), .is_strict = false};
+ for (size_t i = 0; i < decimal_col.size(); ++i) {
+ TimeStampNsValue value;
+ if (CastToTimestampNs::from_decimal<DatelikeParseMode::NON_STRICT>(
+ decimal_col.get_intergral_part(i), decimal_col.get_fractional_part(i),
+ decimal_col.get_scale(), value, params)) [[likely]] {
+ col_data.get_data()[i] = value;
+ col_nullmap.get_data()[i] = false;
+ } else {
+ col_data.get_data()[i] = TimeStampNsValue();
+ col_nullmap.get_data()[i] = true;
+ }
+ }
+ return Status::OK();
+}
+
+template <typename DecimalDataType>
+Status DataTypeTimeStampNsSerDe::from_decimal_strict_mode_batch(
+ const typename DecimalDataType::ColumnType& decimal_col, IColumn& target_col) const {
+ auto& col_data = assert_cast<ColumnTimeStampNs&>(target_col);
+ col_data.resize(decimal_col.size());
+
+ CastParameters params {.status = Status::OK(), .is_strict = true};
+ for (size_t i = 0; i < decimal_col.size(); ++i) {
+ TimeStampNsValue value;
+ CastToTimestampNs::from_decimal<DatelikeParseMode::STRICT>(
+ decimal_col.get_intergral_part(i), decimal_col.get_fractional_part(i),
+ decimal_col.get_scale(), value, params);
+ if (!params.status.ok()) [[unlikely]] {
+ params.status.prepend(fmt::format(
+ "parse {}.{} to timestamp_ns failed: ", decimal_col.get_intergral_part(i),
+ decimal_col.get_fractional_part(i)));
+ return params.status;
+ }
+ col_data.get_data()[i] = value;
+ }
+ return Status::OK();
+}
+
+Status DataTypeTimeStampNsSerDe::serialize_column_to_json(const IColumn& column, int64_t start_idx,
+ int64_t end_idx, BufferWritable& bw,
+ FormatOptions& options) const {
+ SERIALIZE_COLUMN_TO_JSON();
+}
+
+Status DataTypeTimeStampNsSerDe::serialize_one_cell_to_json(const IColumn& column, int64_t row_num,
+ BufferWritable& bw,
+ FormatOptions& options) const {
+ auto [column_ptr, index] = check_column_const_set_readability(column, row_num);
+ if (_nesting_level > 1) {
+ bw.write('"');
+ }
+ const auto value =
+ assert_cast<const ColumnTimeStampNs&, TypeCheckOnRelease::DISABLE>(*column_ptr)
+ .get_element(index);
+ const std::string result = value.to_string();
+ bw.write(result.data(), result.size());
+ if (_nesting_level > 1) {
+ bw.write('"');
+ }
+ return Status::OK();
+}
+
+Status DataTypeTimeStampNsSerDe::deserialize_column_from_json_vector(
+ IColumn& column, std::vector<Slice>& slices, uint64_t* num_deserialized,
+ const FormatOptions& options) const {
+ DESERIALIZE_COLUMN_FROM_JSON_VECTOR();
+ return Status::OK();
+}
+
+Status DataTypeTimeStampNsSerDe::deserialize_one_cell_from_json(
+ IColumn& column, Slice& slice, const FormatOptions& options) const {
+ if (_nesting_level > 1) {
+ slice.trim_quote();
+ }
+ StringRef str(slice.data, slice.size);
+ return from_string(str, column, options);
+}
+
+Status DataTypeTimeStampNsSerDe::deserialize_column_from_jsonb(IColumn& column,
+ const JsonbValue* jsonb_value,
+ CastParameters& cast_params) const {
+ DORIS_CHECK(jsonb_value->isString());
+ return parse_column_from_jsonb_string(column, jsonb_value, cast_params);
+}
+
+Status DataTypeTimeStampNsSerDe::deserialize_column_from_jsonb_vector(
+ ColumnNullable& column_to, const ColumnString& column_from,
+ CastParameters& cast_params) const {
+ return DataTypeSerDe::deserialize_column_from_jsonb_vector(column_to, column_from, cast_params);
+}
+
+Status DataTypeTimeStampNsSerDe::write_column_to_arrow(const IColumn& column,
+ const NullMap* null_map,
+ arrow::ArrayBuilder* array_builder,
+ int64_t start, int64_t end,
+ const cctz::time_zone&) const {
+ const auto& data = assert_cast<const ColumnTimeStampNs&>(column).get_data();
+ auto& builder = assert_cast<arrow::TimestampBuilder&>(*array_builder);
+ const auto timestamp_type =
+ std::static_pointer_cast<arrow::TimestampType>(array_builder->type());
+ int64_t nanos_per_unit = 0;
+ RETURN_IF_ERROR(get_nanos_per_arrow_timestamp_unit(timestamp_type->unit(), &nanos_per_unit));
+
+ for (int64_t i = start; i < end; ++i) {
+ if (null_map != nullptr && (*null_map)[i]) {
+ RETURN_IF_ERROR(checkArrowStatus(builder.AppendNull(), column, builder));
+ continue;
+ }
+
+ const int64_t epoch_nanos = data[i].epoch_nanos();
+ // The default Arrow schema uses NANO. If a caller supplies a coarser timestamp schema,
+ // accept it only when conversion is exact instead of silently discarding nanoseconds.
+ if (epoch_nanos % nanos_per_unit != 0) {
+ return Status::InvalidArgument(
+ "TIMESTAMP_NS value {} cannot be represented exactly as Arrow timestamp unit "
+ "{}",
+ epoch_nanos, static_cast<int>(timestamp_type->unit()));
+ }
+ RETURN_IF_ERROR(
+ checkArrowStatus(builder.Append(epoch_nanos / nanos_per_unit), column, builder));
+ }
+ return Status::OK();
+}
+
+Status DataTypeTimeStampNsSerDe::read_column_from_arrow(IColumn& column,
+ const arrow::Array* arrow_array,
+ int64_t start, int64_t end,
+ const cctz::time_zone&) const {
+ if (config::enable_arrow_input_validation) {
+ check_arrow_array_range(*arrow_array, start, end);
+ }
+ if (arrow_array->type_id() != arrow::Type::TIMESTAMP) {
+ return Status::InvalidArgument("Cannot convert Arrow type {} to TIMESTAMP_NS",
+ arrow_array->type()->name());
+ }
+
+ const auto* timestamp_array = assert_cast<const arrow::TimestampArray*>(arrow_array);
+ if (config::enable_arrow_input_validation) {
+ check_arrow_fixed_width_buffer(*timestamp_array, sizeof(arrow::TimestampArray::value_type));
+ }
+ const auto timestamp_type = std::static_pointer_cast<arrow::TimestampType>(arrow_array->type());
+ int64_t nanos_per_unit = 0;
+ RETURN_IF_ERROR(get_nanos_per_arrow_timestamp_unit(timestamp_type->unit(), &nanos_per_unit));
+
+ auto& data = assert_cast<ColumnTimeStampNs&>(column).get_data();
+ const auto* raw_values = reinterpret_cast<const uint8_t*>(timestamp_array->raw_values());
+ for (int64_t i = start; i < end; ++i) {
+ // Nullable SerDe has already copied the validity bitmap. Avoid converting the unspecified
+ // payload of a null Arrow slot, which could otherwise produce a spurious overflow error.
+ if (timestamp_array->IsNull(i)) {
+ data.emplace_back();
+ continue;
+ }
+
+ const int64_t value = unaligned_load<int64_t>(raw_values + i * sizeof(int64_t));
+ int64_t epoch_nanos = 0;
+ if (__builtin_mul_overflow(value, nanos_per_unit, &epoch_nanos)) {
+ return Status::DataQualityError(
+ "Arrow timestamp {} in unit {} is outside the TIMESTAMP_NS range", value,
+ static_cast<int>(timestamp_type->unit()));
+ }
+ data.emplace_back(epoch_nanos);
+ }
+ return Status::OK();
+}
+
+Status DataTypeTimeStampNsSerDe::write_column_to_mysql_binary(const IColumn& column,
+ MysqlRowBinaryBuffer& row_buffer,
+ int64_t row_idx, bool col_const,
+ const FormatOptions& options) const {
+ const auto& data = assert_cast<const ColumnTimeStampNs&>(column).get_data();
+ const auto index = index_check_const(row_idx, col_const);
+ const auto value = TimeStampNsValue(data[index]).to_string();
+ if (row_buffer.push_string(value.data(), value.size()) != 0) {
+ return Status::InternalError("pack MySQL TIMESTAMP_NS buffer failed");
+ }
+ return Status::OK();
+}
+
+Status DataTypeTimeStampNsSerDe::write_column_to_orc(const std::string& timezone,
+ const IColumn& column, const NullMap* null_map,
+ orc::ColumnVectorBatch* orc_col_batch,
+ int64_t start, int64_t end, Arena& arena,
+ const FormatOptions& options) const {
+ return Status::NotSupported("DataTypeTimeStampNsSerDe::write_column_to_orc");
+}
+
+void DataTypeTimeStampNsSerDe::write_one_cell_to_binary(const IColumn& src_column,
+ ColumnString::Chars& chars,
+ int64_t row_num) const {
+ const auto type = static_cast<uint8_t>(FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS);
+ constexpr auto scale = static_cast<uint8_t>(9);
+ const auto value = assert_cast<const ColumnTimeStampNs&>(src_column).get_element(row_num);
+ const size_t old_size = chars.size();
+ chars.resize(old_size + sizeof(type) + sizeof(scale) + sizeof(value));
+ memcpy(chars.data() + old_size, &type, sizeof(type));
+ memcpy(chars.data() + old_size + sizeof(type), &scale, sizeof(scale));
+ memcpy(chars.data() + old_size + sizeof(type) + sizeof(scale), &value, sizeof(value));
+}
+
+std::string DataTypeTimeStampNsSerDe::to_olap_string(const Field& field) const {
+ return field.get<TYPE_TIMESTAMP_NS>().to_string();
+}
+
+Status DataTypeTimeStampNsSerDe::from_olap_string(const std::string& str, Field& field,
+ const FormatOptions& options) const {
+ TimeStampNsValue value;
+ RETURN_IF_ERROR(parse_timestamp_ns(StringRef(str.data(), str.size()), &value));
+ field = Field::create_field<TYPE_TIMESTAMP_NS>(value);
+ return Status::OK();
+}
+
+template Status DataTypeTimeStampNsSerDe::from_int_batch<DataTypeInt8>(
+ const DataTypeInt8::ColumnType& int_col, ColumnNullable& target_col) const;
+template Status DataTypeTimeStampNsSerDe::from_int_batch<DataTypeInt16>(
+ const DataTypeInt16::ColumnType& int_col, ColumnNullable& target_col) const;
+template Status DataTypeTimeStampNsSerDe::from_int_batch<DataTypeInt32>(
+ const DataTypeInt32::ColumnType& int_col, ColumnNullable& target_col) const;
+template Status DataTypeTimeStampNsSerDe::from_int_batch<DataTypeInt64>(
+ const DataTypeInt64::ColumnType& int_col, ColumnNullable& target_col) const;
+template Status DataTypeTimeStampNsSerDe::from_int_batch<DataTypeInt128>(
+ const DataTypeInt128::ColumnType& int_col, ColumnNullable& target_col) const;
+template Status DataTypeTimeStampNsSerDe::from_int_strict_mode_batch<DataTypeInt8>(
+ const DataTypeInt8::ColumnType& int_col, IColumn& target_col) const;
+template Status DataTypeTimeStampNsSerDe::from_int_strict_mode_batch<DataTypeInt16>(
+ const DataTypeInt16::ColumnType& int_col, IColumn& target_col) const;
+template Status DataTypeTimeStampNsSerDe::from_int_strict_mode_batch<DataTypeInt32>(
+ const DataTypeInt32::ColumnType& int_col, IColumn& target_col) const;
+template Status DataTypeTimeStampNsSerDe::from_int_strict_mode_batch<DataTypeInt64>(
+ const DataTypeInt64::ColumnType& int_col, IColumn& target_col) const;
+template Status DataTypeTimeStampNsSerDe::from_int_strict_mode_batch<DataTypeInt128>(
+ const DataTypeInt128::ColumnType& int_col, IColumn& target_col) const;
+template Status DataTypeTimeStampNsSerDe::from_float_batch<DataTypeFloat32>(
+ const DataTypeFloat32::ColumnType& float_col, ColumnNullable& target_col) const;
+template Status DataTypeTimeStampNsSerDe::from_float_batch<DataTypeFloat64>(
+ const DataTypeFloat64::ColumnType& float_col, ColumnNullable& target_col) const;
+template Status DataTypeTimeStampNsSerDe::from_float_strict_mode_batch<DataTypeFloat32>(
+ const DataTypeFloat32::ColumnType& float_col, IColumn& target_col) const;
+template Status DataTypeTimeStampNsSerDe::from_float_strict_mode_batch<DataTypeFloat64>(
+ const DataTypeFloat64::ColumnType& float_col, IColumn& target_col) const;
+template Status DataTypeTimeStampNsSerDe::from_decimal_batch<DataTypeDecimal32>(
+ const DataTypeDecimal32::ColumnType& decimal_col, ColumnNullable& target_col) const;
+template Status DataTypeTimeStampNsSerDe::from_decimal_batch<DataTypeDecimal64>(
+ const DataTypeDecimal64::ColumnType& decimal_col, ColumnNullable& target_col) const;
+template Status DataTypeTimeStampNsSerDe::from_decimal_batch<DataTypeDecimalV2>(
+ const DataTypeDecimalV2::ColumnType& decimal_col, ColumnNullable& target_col) const;
+template Status DataTypeTimeStampNsSerDe::from_decimal_batch<DataTypeDecimal128>(
+ const DataTypeDecimal128::ColumnType& decimal_col, ColumnNullable& target_col) const;
+template Status DataTypeTimeStampNsSerDe::from_decimal_batch<DataTypeDecimal256>(
+ const DataTypeDecimal256::ColumnType& decimal_col, ColumnNullable& target_col) const;
+template Status DataTypeTimeStampNsSerDe::from_decimal_strict_mode_batch<DataTypeDecimal32>(
+ const DataTypeDecimal32::ColumnType& decimal_col, IColumn& target_col) const;
+template Status DataTypeTimeStampNsSerDe::from_decimal_strict_mode_batch<DataTypeDecimal64>(
+ const DataTypeDecimal64::ColumnType& decimal_col, IColumn& target_col) const;
+template Status DataTypeTimeStampNsSerDe::from_decimal_strict_mode_batch<DataTypeDecimalV2>(
+ const DataTypeDecimalV2::ColumnType& decimal_col, IColumn& target_col) const;
+template Status DataTypeTimeStampNsSerDe::from_decimal_strict_mode_batch<DataTypeDecimal128>(
+ const DataTypeDecimal128::ColumnType& decimal_col, IColumn& target_col) const;
+template Status DataTypeTimeStampNsSerDe::from_decimal_strict_mode_batch<DataTypeDecimal256>(
+ const DataTypeDecimal256::ColumnType& decimal_col, IColumn& target_col) const;
+
+} // namespace doris
diff --git a/be/src/core/data_type_serde/data_type_timestamp_ns_serde.h b/be/src/core/data_type_serde/data_type_timestamp_ns_serde.h
new file mode 100644
index 00000000000..97eb42cb2e8
--- /dev/null
+++ b/be/src/core/data_type_serde/data_type_timestamp_ns_serde.h
@@ -0,0 +1,102 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include "core/data_type_serde/data_type_number_serde.h"
+#include "core/value/timestamp_ns_value.h"
+
+namespace doris {
+
+Status parse_timestamp_ns(StringRef str, TimeStampNsValue* value,
+ const cctz::time_zone* local_time_zone = nullptr);
+
+class DataTypeTimeStampNsSerDe final
+ : public DataTypeNumberSerDe<PrimitiveType::TYPE_TIMESTAMP_NS> {
+public:
+ explicit DataTypeTimeStampNsSerDe(int nesting_level = 1)
+ : DataTypeNumberSerDe<PrimitiveType::TYPE_TIMESTAMP_NS>(nesting_level) {}
+
+ Status from_string_batch(const ColumnString& str, ColumnNullable& column,
+ const FormatOptions& options) const override;
+ Status from_string_strict_mode_batch(
+ const ColumnString& str, IColumn& column, const FormatOptions& options,
+ const NullMap::value_type* null_map = nullptr) const override;
+ Status from_string(StringRef& str, IColumn& column,
+ const FormatOptions& options) const override;
+ Status from_string_strict_mode(StringRef& str, IColumn& column,
+ const FormatOptions& options) const override;
+
+ template <typename IntDataType>
+ Status from_int_batch(const IntDataType::ColumnType& int_col, ColumnNullable& target_col) const;
+ template <typename IntDataType>
+ Status from_int_strict_mode_batch(const IntDataType::ColumnType& int_col,
+ IColumn& target_col) const;
+
+ template <typename FloatDataType>
+ Status from_float_batch(const FloatDataType::ColumnType& float_col,
+ ColumnNullable& target_col) const;
+ template <typename FloatDataType>
+ Status from_float_strict_mode_batch(const FloatDataType::ColumnType& float_col,
+ IColumn& target_col) const;
+
+ template <typename DecimalDataType>
+ Status from_decimal_batch(const DecimalDataType::ColumnType& decimal_col,
+ ColumnNullable& target_col) const;
+ template <typename DecimalDataType>
+ Status from_decimal_strict_mode_batch(const DecimalDataType::ColumnType& decimal_col,
+ IColumn& target_col) const;
+
+ Status serialize_one_cell_to_json(const IColumn& column, int64_t row_num, BufferWritable& bw,
+ FormatOptions& options) const override;
+ Status serialize_column_to_json(const IColumn& column, int64_t start_idx, int64_t end_idx,
+ BufferWritable& bw, FormatOptions& options) const override;
+ Status deserialize_one_cell_from_json(IColumn& column, Slice& slice,
+ const FormatOptions& options) const override;
+ Status deserialize_column_from_json_vector(IColumn& column, std::vector<Slice>& slices,
+ uint64_t* num_deserialized,
+ const FormatOptions& options) const override;
+ Status deserialize_column_from_jsonb(IColumn& column, const JsonbValue* jsonb_value,
+ CastParameters& cast_params) const override;
+ Status deserialize_column_from_jsonb_vector(ColumnNullable& column_to,
+ const ColumnString& column_from,
+ CastParameters& cast_params) const override;
+
+ Status write_column_to_arrow(const IColumn& column, const NullMap* null_map,
+ arrow::ArrayBuilder* array_builder, int64_t start, int64_t end,
+ const cctz::time_zone& ctz) const override;
+ Status read_column_from_arrow(IColumn& column, const arrow::Array* arrow_array, int64_t start,
+ int64_t end, const cctz::time_zone& ctz) const override;
+ Status write_column_to_mysql_binary(const IColumn& column, MysqlRowBinaryBuffer& row_buffer,
+ int64_t row_idx, bool col_const,
+ const FormatOptions& options) const override;
+ Status write_column_to_orc(const std::string& timezone, const IColumn& column,
+ const NullMap* null_map, orc::ColumnVectorBatch* orc_col_batch,
+ int64_t start, int64_t end, Arena& arena,
+ const FormatOptions& options) const override;
+
+ void write_one_cell_to_binary(const IColumn& src_column, ColumnString::Chars& chars,
+ int64_t row_num) const override;
+ int get_scale() const override { return TimeStampNsValue::FRACTIONAL_DIGITS; }
+ std::string to_olap_string(const Field& field) const override;
+
+protected:
+ Status from_olap_string(const std::string& str, Field& field,
+ const FormatOptions& options) const override;
+};
+
+} // namespace doris
diff --git a/be/src/core/field.cpp b/be/src/core/field.cpp
index 6cef7b05ec2..54dd80fecbc 100644
--- a/be/src/core/field.cpp
+++ b/be/src/core/field.cpp
@@ -179,6 +179,9 @@ void Field::create(Field&& field) {
case PrimitiveType::TYPE_DATETIMEV2:
create_concrete<TYPE_DATETIMEV2>(std::move(field.template get<TYPE_DATETIMEV2>()));
return;
+ case PrimitiveType::TYPE_TIMESTAMP_NS:
+ create_concrete<TYPE_TIMESTAMP_NS>(std::move(field.template get<TYPE_TIMESTAMP_NS>()));
+ return;
case PrimitiveType::TYPE_DATEV2:
create_concrete<TYPE_DATEV2>(std::move(field.template get<TYPE_DATEV2>()));
return;
@@ -310,6 +313,9 @@ void Field::create(const Field& field) {
case PrimitiveType::TYPE_DATETIMEV2:
create_concrete<TYPE_DATETIMEV2>(field.template get<TYPE_DATETIMEV2>());
return;
+ case PrimitiveType::TYPE_TIMESTAMP_NS:
+ create_concrete<TYPE_TIMESTAMP_NS>(field.template get<TYPE_TIMESTAMP_NS>());
+ return;
case PrimitiveType::TYPE_DATEV2:
create_concrete<TYPE_DATEV2>(field.template get<TYPE_DATEV2>());
return;
@@ -471,6 +477,9 @@ void Field::assign(Field&& field) {
case PrimitiveType::TYPE_DATETIMEV2:
assign_concrete<TYPE_DATETIMEV2>(std::move(field.template get<TYPE_DATETIMEV2>()));
return;
+ case PrimitiveType::TYPE_TIMESTAMP_NS:
+ assign_concrete<TYPE_TIMESTAMP_NS>(std::move(field.template get<TYPE_TIMESTAMP_NS>()));
+ return;
case PrimitiveType::TYPE_DATETIME:
assign_concrete<TYPE_DATETIME>(std::move(field.template get<TYPE_DATETIME>()));
return;
@@ -580,6 +589,9 @@ void Field::assign(const Field& field) {
case PrimitiveType::TYPE_DATETIMEV2:
assign_concrete<TYPE_DATETIMEV2>(field.template get<TYPE_DATETIMEV2>());
return;
+ case PrimitiveType::TYPE_TIMESTAMP_NS:
+ assign_concrete<TYPE_TIMESTAMP_NS>(field.template get<TYPE_TIMESTAMP_NS>());
+ return;
case PrimitiveType::TYPE_DATETIME:
assign_concrete<TYPE_DATETIME>(field.template get<TYPE_DATETIME>());
return;
@@ -777,6 +789,9 @@ std::strong_ordering Field::operator<=>(const Field& rhs) const {
case PrimitiveType::TYPE_DATETIMEV2:
return get<PrimitiveType::TYPE_DATETIMEV2>().to_date_int_val() <=>
rhs.get<PrimitiveType::TYPE_DATETIMEV2>().to_date_int_val();
+ case PrimitiveType::TYPE_TIMESTAMP_NS:
+ return get<PrimitiveType::TYPE_TIMESTAMP_NS>() <=>
+ rhs.get<PrimitiveType::TYPE_TIMESTAMP_NS>();
case PrimitiveType::TYPE_DATEV2:
return get<PrimitiveType::TYPE_DATEV2>().to_date_int_val() <=>
rhs.get<PrimitiveType::TYPE_DATEV2>().to_date_int_val();
@@ -896,6 +911,7 @@ std::string_view Field::as_string_view() const {
// MATCH_PRIMITIVE_TYPE(TYPE_QUANTILE_STATE);
MATCH_PRIMITIVE_TYPE(TYPE_DATEV2);
MATCH_PRIMITIVE_TYPE(TYPE_DATETIMEV2);
+ MATCH_PRIMITIVE_TYPE(TYPE_TIMESTAMP_NS);
MATCH_PRIMITIVE_TYPE(TYPE_TIMEV2);
MATCH_PRIMITIVE_TYPE(TYPE_DECIMAL32);
MATCH_PRIMITIVE_TYPE(TYPE_DECIMAL64);
@@ -953,6 +969,8 @@ std::string Field::to_debug_string(int scale) const {
return CastToString::from_datev2(get<TYPE_DATEV2>());
case PrimitiveType::TYPE_DATETIMEV2:
return CastToString::from_datetimev2(get<TYPE_DATETIMEV2>(), scale);
+ case PrimitiveType::TYPE_TIMESTAMP_NS:
+ return get<TYPE_TIMESTAMP_NS>().to_string();
case PrimitiveType::TYPE_TIMESTAMPTZ:
return CastToString::from_timestamptz(get<TYPE_TIMESTAMPTZ>(), scale);
case PrimitiveType::TYPE_DECIMALV2:
@@ -997,6 +1015,8 @@ std::string Field::to_debug_string(int scale) const {
typename PrimitiveTypeTraits<TYPE_DATEV2>::CppType && rhs); \
template void Field::FUNC_NAME<TYPE_DATETIMEV2>( \
typename PrimitiveTypeTraits<TYPE_DATETIMEV2>::CppType && rhs); \
+ template void Field::FUNC_NAME<TYPE_TIMESTAMP_NS>( \
+ typename PrimitiveTypeTraits<TYPE_TIMESTAMP_NS>::CppType && rhs); \
template void Field::FUNC_NAME<TYPE_DECIMAL32>( \
typename PrimitiveTypeTraits<TYPE_DECIMAL32>::CppType && rhs); \
template void Field::FUNC_NAME<TYPE_DECIMAL64>( \
@@ -1047,6 +1067,8 @@ std::string Field::to_debug_string(int scale) const {
const typename PrimitiveTypeTraits<TYPE_DATEV2>::CppType& rhs); \
template void Field::FUNC_NAME<TYPE_DATETIMEV2>( \
const typename PrimitiveTypeTraits<TYPE_DATETIMEV2>::CppType& rhs); \
+ template void Field::FUNC_NAME<TYPE_TIMESTAMP_NS>( \
+ const typename PrimitiveTypeTraits<TYPE_TIMESTAMP_NS>::CppType& rhs); \
template void Field::FUNC_NAME<TYPE_TIMESTAMPTZ>( \
const typename PrimitiveTypeTraits<TYPE_TIMESTAMPTZ>::CppType& rhs); \
template void Field::FUNC_NAME<TYPE_TIMESTAMPTZ>( \
@@ -1145,6 +1167,7 @@ DECLARE_FUNCTION(TYPE_DATE)
DECLARE_FUNCTION(TYPE_DATETIME)
DECLARE_FUNCTION(TYPE_DATEV2)
DECLARE_FUNCTION(TYPE_DATETIMEV2)
+DECLARE_FUNCTION(TYPE_TIMESTAMP_NS)
DECLARE_FUNCTION(TYPE_TIMESTAMPTZ)
DECLARE_FUNCTION(TYPE_DECIMAL32)
DECLARE_FUNCTION(TYPE_DECIMAL64)
diff --git a/be/src/core/type_limit.h b/be/src/core/type_limit.h
index 824433a0319..b26ca545202 100644
--- a/be/src/core/type_limit.h
+++ b/be/src/core/type_limit.h
@@ -24,6 +24,8 @@
namespace doris {
+class TimeStampNsValue;
+
template <typename T>
struct type_limit {
static constexpr T min() { return std::numeric_limits<T>::lowest(); }
@@ -118,6 +120,12 @@ struct type_limit<DateV2Value<DateTimeV2ValueType>> {
}
};
+template <>
+struct type_limit<TimeStampNsValue> {
+ static TimeStampNsValue min();
+ static TimeStampNsValue max();
+};
+
template <>
struct type_limit<TimestampTzValue> {
static TimestampTzValue min() {
diff --git a/be/src/core/value/time_value.h b/be/src/core/value/time_value.h
index ce72e92d635..b3260edb878 100644
--- a/be/src/core/value/time_value.h
+++ b/be/src/core/value/time_value.h
@@ -153,7 +153,8 @@ public:
static bool valid(double time) { return time <= MAX_TIME && time >= -MAX_TIME; }
static bool to_format_string_conservative(const char* format, size_t len, char* to,
- size_t max_valid_length, TimeType time) {
+ size_t max_valid_length, TimeType time,
+ int nanosecond = -1) {
// If time is negative, we here only add a '-' to the begining of res
// This behavior is consistent with MySQL
if (time < 0) {
@@ -164,7 +165,8 @@ public:
return DatetimeValueUtil::to_format_string_without_check<true>(
format, len, to, max_valid_length, 0, 0, 0, TimeValue::hour(time),
- TimeValue::minute(time), TimeValue::second(time), TimeValue::microsecond(time));
+ TimeValue::minute(time), TimeValue::second(time), TimeValue::microsecond(time),
+ nanosecond);
}
};
} // namespace doris
diff --git a/be/src/core/value/timestamp_ns_value.cpp b/be/src/core/value/timestamp_ns_value.cpp
new file mode 100644
index 00000000000..e8d73367ae5
--- /dev/null
+++ b/be/src/core/value/timestamp_ns_value.cpp
@@ -0,0 +1,98 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "core/value/timestamp_ns_value.h"
+
+#include "core/type_limit.h"
+#include "exec/common/int_exp.h"
+
+namespace doris {
+
+TimeStampNsValue type_limit<TimeStampNsValue>::min() {
+ return TimeStampNsValue(std::numeric_limits<int64_t>::min());
+}
+
+TimeStampNsValue type_limit<TimeStampNsValue>::max() {
+ return TimeStampNsValue(std::numeric_limits<int64_t>::max());
+}
+
+DateV2Value<DateV2ValueType> TimeStampNsValue::to_date() const {
+ DateV2Value<DateV2ValueType> value;
+ [[maybe_unused]] const bool valid = value.get_date_from_daynr(static_cast<uint64_t>(daynr()));
+ DCHECK(valid);
+ return value;
+}
+
+DateV2Value<DateTimeV2ValueType> TimeStampNsValue::to_datetime() const {
+ int64_t days = _epoch_nanos / NANOS_PER_DAY;
+ int64_t nanos_of_day = _epoch_nanos % NANOS_PER_DAY;
+ if (nanos_of_day < 0) {
+ nanos_of_day += NANOS_PER_DAY;
+ --days;
+ }
+
+ DateV2Value<DateTimeV2ValueType> value;
+ [[maybe_unused]] const bool valid =
+ value.get_date_from_daynr(static_cast<uint64_t>(UNIX_EPOCH_DAYNR + days));
+ DCHECK(valid);
+
+ int64_t seconds = nanos_of_day / NANOS_PER_SECOND;
+ const auto hour = static_cast<uint8_t>(seconds / 3600);
+ seconds %= 3600;
+ const auto minute = static_cast<uint8_t>(seconds / 60);
+ const auto second = static_cast<uint16_t>(seconds % 60);
+ const auto microsecond =
+ static_cast<uint32_t>(nanos_of_day % NANOS_PER_SECOND / NANOS_PER_MICROSECOND);
+ value.unchecked_set_time(value.year(), value.month(), value.day(), hour, minute, second,
+ microsecond);
+ return value;
+}
+
+bool TimeStampNsValue::from_datetime(const DateV2Value<DateTimeV2ValueType>& value,
+ uint16_t nanosecond_remainder) {
+ DORIS_CHECK_LE(nanosecond_remainder, 999);
+ const int64_t seconds =
+ (value.daynr() - UNIX_EPOCH_DAYNR) * SECONDS_PER_DAY + value.time_part_to_seconds();
+ // The civil adapter and remainder form one exact epoch-nanosecond value. Int128 is required:
+ // valid boundary dates overflow Int64 during the intermediate seconds-to-nanoseconds product.
+ const __int128 epoch_nanos = static_cast<__int128>(seconds) * NANOS_PER_SECOND +
+ static_cast<__int128>(value.microsecond()) * 1000 +
+ nanosecond_remainder;
+ if (epoch_nanos < std::numeric_limits<int64_t>::min() ||
+ epoch_nanos > std::numeric_limits<int64_t>::max()) {
+ return false;
+ }
+ _epoch_nanos = static_cast<int64_t>(epoch_nanos);
+ return true;
+}
+
+int32_t TimeStampNsValue::to_buffer(char* buffer) const {
+ const auto value = to_datetime();
+ const int32_t base_length = value.to_buffer(buffer, 0);
+ buffer[base_length] = '.';
+ uint32_t nanos = nanosecond();
+ // Consume the fraction from most significant to least significant digit. Deliberately do not
+ // round here: type normalization happens while parsing/casting, before a value is stored.
+ for (int i = 0; i < FRACTIONAL_DIGITS; ++i) {
+ const auto divisor = static_cast<uint32_t>(int_exp10(FRACTIONAL_DIGITS - 1 - i));
+ buffer[base_length + 1 + i] = static_cast<char>('0' + nanos / divisor);
+ nanos %= divisor;
+ }
+ return base_length + 1 + FRACTIONAL_DIGITS;
+}
+
+} // namespace doris
diff --git a/be/src/core/value/timestamp_ns_value.h b/be/src/core/value/timestamp_ns_value.h
new file mode 100644
index 00000000000..8544b2b5875
--- /dev/null
+++ b/be/src/core/value/timestamp_ns_value.h
@@ -0,0 +1,324 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <compare>
+#include <cstdint>
+#include <functional>
+#include <string>
+#include <type_traits>
+
+#include "core/value/vdatetime_value.h"
+
+namespace doris {
+
+// TIMESTAMP_NS is represented by a signed Int64 count of nanoseconds from the Unix epoch. The
+// complete Int64 domain is valid, giving the exact range
+// [1677-09-21 00:12:43.145224192, 2262-04-11 23:47:16.854775807]. The SQL type is timezone-naive:
+// UTC below is only a deterministic bridge between the integer and civil calendar fields, and
+// must not apply the session time zone.
+//
+// Keep this class an Int64-sized trivially-copyable value. Columns, storage encodings, hashing,
+// and SIMD paths rely on being able to move its bytes exactly like an Int64.
+class TimeStampNsValue {
+public:
+ static constexpr int FRACTIONAL_DIGITS = 9;
+ static constexpr int64_t NANOS_PER_SECOND = 1000000000;
+ static constexpr int64_t NANOS_PER_MILLISECOND = 1000000;
+ static constexpr int64_t NANOS_PER_MICROSECOND = 1000;
+ static constexpr int64_t SECONDS_PER_DAY = 86400;
+ static constexpr int64_t NANOS_PER_DAY = SECONDS_PER_DAY * NANOS_PER_SECOND;
+ static constexpr int64_t UNIX_EPOCH_DAYNR = 719528;
+
+ constexpr TimeStampNsValue() = default;
+ explicit constexpr TimeStampNsValue(int64_t epoch_nanos) : _epoch_nanos(epoch_nanos) {}
+
+ // This is the raw physical key, not the packed civil layout used by DATETIMEV2.
+ constexpr int64_t to_date_int_val() const { return _epoch_nanos; }
+ constexpr int64_t epoch_nanos() const { return _epoch_nanos; }
+
+ // Split epoch nanoseconds using floor division so the fractional component is always
+ // non-negative. For example, -1ns becomes second -1 plus nanosecond 999,999,999 rather than
+ // second 0 plus a negative fraction.
+ int64_t epoch_seconds() const {
+ int64_t seconds = _epoch_nanos / NANOS_PER_SECOND;
+ if (_epoch_nanos % NANOS_PER_SECOND < 0) {
+ --seconds;
+ }
+ return seconds;
+ }
+
+ uint32_t nanosecond() const {
+ int64_t nanos = _epoch_nanos % NANOS_PER_SECOND;
+ if (nanos < 0) {
+ nanos += NANOS_PER_SECOND;
+ }
+ return static_cast<uint32_t>(nanos);
+ }
+
+ uint32_t microsecond() const { return nanosecond() / 1000; }
+ uint16_t nanosecond_remainder() const { return static_cast<uint16_t>(nanosecond() % 1000); }
+
+ // DATEV2 and DATETIMEV2 are used only as civil-calendar adapters. DATETIMEV2 carries the first
+ // six fractional digits; callers must preserve nanosecond_remainder() separately when the
+ // operation should not discard the final three digits.
+ DateV2Value<DateV2ValueType> to_date() const;
+ DateV2Value<DateTimeV2ValueType> to_datetime() const;
+ bool from_datetime(const DateV2Value<DateTimeV2ValueType>& value,
+ uint16_t nanosecond_remainder = 0);
+
+ // Calendar-only fields use the packed DATEV2 value as an adapter. Epoch-derived fields stay on
+ // the Int64 representation so extraction and difference functions do not pay for a
+ // civil-calendar conversion.
+ uint16_t year() const { return to_date().year(); }
+ uint8_t month() const { return to_date().month(); }
+ uint8_t day() const { return to_date().day(); }
+ uint8_t hour() const { return static_cast<uint8_t>(time_part_to_seconds() / 3600); }
+ uint8_t minute() const { return static_cast<uint8_t>(time_part_to_seconds() / 60 % 60); }
+ uint8_t second() const { return static_cast<uint8_t>(time_part_to_seconds() % 60); }
+ int quarter() const { return (month() - 1) / 3 + 1; }
+ int64_t daynr() const { return UNIX_EPOCH_DAYNR + epoch_days(); }
+ uint16_t year_of_week() const { return to_date().year_of_week(); }
+ uint8_t week(uint8_t mode) const { return to_date().week(mode); }
+ uint32_t year_week(uint8_t mode) const { return to_date().year_week(mode); }
+ int day_of_year() const { return to_date().day_of_year(); }
+ int day_of_week() const { return (weekday() + 1) % 7 + 1; }
+ uint8_t weekday() const { return calc_weekday(daynr(), false); }
+ int64_t time_part_to_nanosecond() const {
+ int64_t nanos = _epoch_nanos % NANOS_PER_DAY;
+ if (nanos < 0) {
+ nanos += NANOS_PER_DAY;
+ }
+ return nanos;
+ }
+ int64_t time_part_to_seconds() const { return time_part_to_nanosecond() / NANOS_PER_SECOND; }
+ int64_t time_part_to_microsecond() const {
+ return time_part_to_nanosecond() / NANOS_PER_MICROSECOND;
+ }
+
+ // This intentionally compares integral civil seconds and ignores fractional seconds. It is
+ // the contract used by sequence_match's second-based pattern conditions.
+ template <typename RHS>
+ int64_t datetime_diff_in_seconds(const RHS& rhs) const {
+ if constexpr (std::is_same_v<std::remove_cvref_t<RHS>, TimeStampNsValue>) {
+ return epoch_seconds() - rhs.epoch_seconds();
+ } else {
+ return (daynr() - rhs.daynr()) * SECOND_PER_HOUR * HOUR_PER_DAY +
+ time_part_to_seconds() - rhs.time_part_to_seconds();
+ }
+ }
+
+ template <typename RHS>
+ int64_t datetime_diff_in_microseconds(const RHS& rhs) const {
+ if constexpr (std::is_same_v<std::remove_cvref_t<RHS>, TimeStampNsValue>) {
+ return static_cast<int64_t>((static_cast<__int128>(_epoch_nanos) - rhs._epoch_nanos) /
+ NANOS_PER_MICROSECOND);
+ }
+ return (daynr() - rhs.daynr()) * HOUR_PER_DAY * SECOND_PER_HOUR * MS_PER_SECOND +
+ time_part_diff_in_ms(rhs);
+ }
+
+ const char* day_name_with_locale(const char* const* day_names) const {
+ return to_date().day_name_with_locale(day_names);
+ }
+ const char* month_name_with_locale(const char* const* month_names) const {
+ return to_date().month_name_with_locale(month_names);
+ }
+ bool to_format_string_conservative(const char* format, size_t len, char* to,
+ size_t max_valid_length) const {
+ return to_datetime().to_format_string_conservative(format, len, to, max_valid_length,
+ nanosecond());
+ }
+
+ bool is_valid_date() const { return true; }
+
+ // Calendar arithmetic preserves all nine fractional digits. The packed adapter carries the
+ // leading microseconds and the explicit remainder carries the final three nanoseconds.
+ template <TimeUnit unit>
+ bool date_add_interval(const TimeInterval& interval) {
+ auto value = to_datetime();
+ const uint16_t remainder = nanosecond_remainder();
+ if (!value.template date_add_interval<unit>(interval)) {
+ return false;
+ }
+ return from_datetime(value, remainder);
+ }
+
+ // Truncation deliberately discards every field below unit. Conversion back detects the two
+ // civil boundary days whose midnight lies outside the signed epoch-nanosecond range.
+ template <TimeUnit unit>
+ bool datetime_trunc() {
+ auto value = to_datetime();
+ if (!value.template datetime_trunc<unit>()) {
+ return false;
+ }
+ return from_datetime(value);
+ }
+
+ // TIMESTAMP_NS has no configurable scale. Formatting always emits all nine fractional digits,
+ // including trailing zeros, so every output path exposes the same fixed-width representation.
+ int32_t to_buffer(char* buffer) const;
+ char* to_string(char* buffer) const {
+ const int32_t length = to_buffer(buffer);
+ buffer[length] = '\0';
+ return buffer + length + 1;
+ }
+ std::string to_string() const {
+ char buffer[40];
+ const int32_t length = to_buffer(buffer);
+ return {buffer, static_cast<size_t>(length)};
+ }
+
+ auto operator<=>(const TimeStampNsValue&) const = default;
+
+private:
+ template <typename RHS>
+ int64_t time_part_diff_in_ms(const RHS& rhs) const {
+ return time_part_to_microsecond() - rhs.time_part_to_microsecond();
+ }
+
+ int64_t epoch_days() const {
+ int64_t days = _epoch_nanos / NANOS_PER_DAY;
+ if (_epoch_nanos % NANOS_PER_DAY < 0) {
+ --days;
+ }
+ return days;
+ }
+
+ int64_t _epoch_nanos = 0;
+};
+
+static_assert(sizeof(TimeStampNsValue) == sizeof(int64_t));
+static_assert(std::is_trivially_copyable_v<TimeStampNsValue>);
+
+inline __int128 datetime_diff_in_nanoseconds(const DateV2Value<DateTimeV2ValueType>& datetime1,
+ uint32_t nanosecond1,
+ const DateV2Value<DateTimeV2ValueType>& datetime2,
+ uint32_t nanosecond2) {
+ const __int128 day_difference = datetime2.daynr() - datetime1.daynr();
+ const int64_t second_difference =
+ datetime2.time_part_to_seconds() - datetime1.time_part_to_seconds();
+ return (day_difference * HOUR_PER_DAY * SECOND_PER_HOUR + second_difference) *
+ TimeStampNsValue::NANOS_PER_SECOND +
+ static_cast<int64_t>(nanosecond2) - nanosecond1;
+}
+
+// Whole-unit differences follow DATETIMEV2 semantics: calendar units compare civil fields while
+// elapsed units use the exact civil nanosecond difference and truncate toward zero.
+template <TimeUnit UNIT>
+int64_t datetime_diff_with_nanoseconds(const DateV2Value<DateTimeV2ValueType>& datetime1,
+ uint32_t nanosecond1,
+ const DateV2Value<DateTimeV2ValueType>& datetime2,
+ uint32_t nanosecond2) {
+ if constexpr (UNIT == YEAR || UNIT == QUARTER || UNIT == MONTH || UNIT == WEEK || UNIT == DAY) {
+ const auto time_key = [](const DateV2Value<DateTimeV2ValueType>& value, uint32_t nanosecond,
+ bool include_month) {
+ int64_t result = include_month ? value.month() : 0;
+ result = result * 32 + value.day();
+ result = result * 24 + value.hour();
+ result = result * 60 + value.minute();
+ result = result * 60 + value.second();
+ return result * TimeStampNsValue::NANOS_PER_SECOND + nanosecond;
+ };
+ if constexpr (UNIT == YEAR) {
+ int year = datetime2.year() - datetime1.year();
+ const int64_t remainder1 = time_key(datetime1, nanosecond1, true);
+ const int64_t remainder2 = time_key(datetime2, nanosecond2, true);
+ if (year > 0) {
+ year -= remainder2 < remainder1;
+ } else if (year < 0) {
+ year += remainder2 > remainder1;
+ }
+ return year;
+ } else if constexpr (UNIT == QUARTER || UNIT == MONTH) {
+ int month = (datetime2.year() - datetime1.year()) * 12 +
+ (datetime2.month() - datetime1.month());
+ const int64_t remainder1 = time_key(datetime1, nanosecond1, false);
+ const int64_t remainder2 = time_key(datetime2, nanosecond2, false);
+ if (month > 0) {
+ month -= remainder2 < remainder1;
+ } else if (month < 0) {
+ month += remainder2 > remainder1;
+ }
+ return UNIT == QUARTER ? month / 3 : month;
+ } else {
+ int64_t day = datetime2.daynr() - datetime1.daynr();
+ const int64_t time1 =
+ datetime1.time_part_to_seconds() * TimeStampNsValue::NANOS_PER_SECOND +
+ nanosecond1;
+ const int64_t time2 =
+ datetime2.time_part_to_seconds() * TimeStampNsValue::NANOS_PER_SECOND +
+ nanosecond2;
+ if (day > 0) {
+ day -= time2 < time1;
+ } else if (day < 0) {
+ day += time2 > time1;
+ }
+ return UNIT == WEEK ? day / 7 : day;
+ }
+ } else {
+ static_assert(UNIT == HOUR || UNIT == MINUTE || UNIT == SECOND || UNIT == MILLISECOND ||
+ UNIT == MICROSECOND,
+ "Unsupported TimeUnit for TIMESTAMP_NS datetime_diff");
+ constexpr int64_t divisor = [] {
+ if constexpr (UNIT == HOUR) {
+ return 3600 * TimeStampNsValue::NANOS_PER_SECOND;
+ } else if constexpr (UNIT == MINUTE) {
+ return 60 * TimeStampNsValue::NANOS_PER_SECOND;
+ } else if constexpr (UNIT == SECOND) {
+ return TimeStampNsValue::NANOS_PER_SECOND;
+ } else if constexpr (UNIT == MILLISECOND) {
+ return TimeStampNsValue::NANOS_PER_MILLISECOND;
+ } else {
+ return TimeStampNsValue::NANOS_PER_MICROSECOND;
+ }
+ }();
+ return static_cast<int64_t>(
+ datetime_diff_in_nanoseconds(datetime1, nanosecond1, datetime2, nanosecond2) /
+ divisor);
+ }
+}
+
+template <TimeUnit UNIT>
+int64_t datetime_diff(const TimeStampNsValue& ts_value1, const TimeStampNsValue& ts_value2) {
+ return datetime_diff_with_nanoseconds<UNIT>(ts_value1.to_datetime(), ts_value1.nanosecond(),
+ ts_value2.to_datetime(), ts_value2.nanosecond());
+}
+
+template <TimeUnit UNIT>
+int64_t datetime_diff(const TimeStampNsValue& ts_value1,
+ const DateV2Value<DateTimeV2ValueType>& ts_value2) {
+ return datetime_diff_with_nanoseconds<UNIT>(ts_value1.to_datetime(), ts_value1.nanosecond(),
+ ts_value2, ts_value2.microsecond() * 1000);
+}
+
+template <TimeUnit UNIT>
+int64_t datetime_diff(const DateV2Value<DateTimeV2ValueType>& ts_value1,
+ const TimeStampNsValue& ts_value2) {
+ return datetime_diff_with_nanoseconds<UNIT>(ts_value1, ts_value1.microsecond() * 1000,
+ ts_value2.to_datetime(), ts_value2.nanosecond());
+}
+
+} // namespace doris
+
+template <>
+struct std::hash<doris::TimeStampNsValue> {
+ size_t operator()(const doris::TimeStampNsValue& value) const {
+ return std::hash<int64_t> {}(value.epoch_nanos());
+ }
+};
diff --git a/be/src/core/value/vdatetime_value.cpp b/be/src/core/value/vdatetime_value.cpp
index 2bceb41b809..f02378c8ad5 100644
--- a/be/src/core/value/vdatetime_value.cpp
+++ b/be/src/core/value/vdatetime_value.cpp
@@ -2932,14 +2932,14 @@ void DateV2Value<T>::set_microsecond(uint64_t microsecond) {
template <typename T>
bool DateV2Value<T>::to_format_string_conservative(const char* format, size_t len, char* to,
- size_t max_valid_length) const {
+ size_t max_valid_length, int nanosecond) const {
if (is_invalid(year(), month(), day(), hour(), minute(), second(), microsecond())) {
return false;
}
return DatetimeValueUtil::to_format_string_without_check<false>(
format, len, to, max_valid_length, this->year(), this->month(), this->day(),
- this->hour(), this->minute(), this->second(), this->microsecond());
+ this->hour(), this->minute(), this->second(), this->microsecond(), nanosecond);
}
template <typename T>
@@ -3116,7 +3116,8 @@ template <bool only_time>
bool DatetimeValueUtil::to_format_string_without_check(const char* format, size_t len, char* to,
size_t max_valid_length, int16_t year,
int8_t month, int8_t day, int hour,
- int minute, int second, int ms) {
+ int minute, int second, int microsecond,
+ int nanosecond) {
char* const begin = to; // to check written bytes
char buf[64];
char* pos = nullptr;
@@ -3239,9 +3240,19 @@ bool DatetimeValueUtil::to_format_string_without_check(const char* format, size_
break;
case 'f':
// Microseconds (000000..999999)
- pos = int_to_str(ms, cursor);
+ pos = int_to_str(microsecond, cursor);
to = append_with_prefix(cursor, pos - cursor, '0', 6, to);
break;
+ case 'n':
+ if (nanosecond < 0) {
+ // Preserve the established behavior for types without nanosecond precision:
+ // unknown format specifiers are emitted literally without the percent sign.
+ *to++ = ch;
+ } else {
+ pos = int_to_str(nanosecond, cursor);
+ to = append_with_prefix(cursor, pos - cursor, '0', 9, to);
+ }
+ break;
case 'j':
// Day of year (001..366)
if constexpr (only_time) {
@@ -3698,10 +3709,12 @@ template bool DateV2Value<DateTimeV2ValueType>::datetime_trunc<TimeUnit::WEEK>()
template bool DatetimeValueUtil::to_format_string_without_check<false>(const char*, size_t, char*,
size_t, int16_t, int8_t,
- int8_t, int, int, int, int);
+ int8_t, int, int, int, int,
+ int);
template bool DatetimeValueUtil::to_format_string_without_check<true>(const char*, size_t, char*,
size_t, int16_t, int8_t,
- int8_t, int, int, int, int);
+ int8_t, int, int, int, int,
+ int);
#include "common/compile_check_avoid_end.h"
} // namespace doris
diff --git a/be/src/core/value/vdatetime_value.h b/be/src/core/value/vdatetime_value.h
index 02fefdf3abf..a94fff3b671 100644
--- a/be/src/core/value/vdatetime_value.h
+++ b/be/src/core/value/vdatetime_value.h
@@ -910,7 +910,7 @@ public:
// so need a little big buffer and its length as max_valid_length to make sure store valid data.
// to make sure of this. make the buffer size = <data_need_length> + SAFE_FORMAT_STRING_MARGIN. and pass this size as max_valid_length
bool to_format_string_conservative(const char* format, size_t len, char* to,
- size_t max_valid_length) const;
+ size_t max_valid_length, int nanosecond = -1) const;
bool from_date_format_str(const char* format, size_t format_len, const char* value,
size_t value_len) {
@@ -1866,7 +1866,7 @@ public:
static bool to_format_string_without_check(const char* format, size_t len, char* to,
size_t max_valid_length, int16_t year, int8_t month,
int8_t day, int hour, int minute, int second,
- int ms);
+ int microsecond, int nanosecond = -1);
private:
static uint8_t week(int16_t year, int8_t month, int8_t day, uint8_t mode);
diff --git a/be/src/exec/common/field_visitors.h b/be/src/exec/common/field_visitors.h
index 8b14b6bbd17..25ca811dd88 100644
--- a/be/src/exec/common/field_visitors.h
+++ b/be/src/exec/common/field_visitors.h
@@ -47,6 +47,9 @@ typename std::decay_t<Visitor>::ResultType apply_visitor(Visitor&& visitor, F&&
case PrimitiveType::TYPE_DATETIMEV2:
return visitor.template apply<PrimitiveType::TYPE_DATETIMEV2>(
field.template get<TYPE_DATETIMEV2>());
+ case PrimitiveType::TYPE_TIMESTAMP_NS:
+ return visitor.template apply<PrimitiveType::TYPE_TIMESTAMP_NS>(
+ field.template get<TYPE_TIMESTAMP_NS>());
case PrimitiveType::TYPE_TIMESTAMPTZ:
return visitor.template apply<PrimitiveType::TYPE_TIMESTAMPTZ>(
field.template get<TYPE_TIMESTAMPTZ>());
diff --git a/be/src/exec/common/hash_table/hash.h b/be/src/exec/common/hash_table/hash.h
index e8487bf4c8d..ea598fd7fc3 100644
--- a/be/src/exec/common/hash_table/hash.h
+++ b/be/src/exec/common/hash_table/hash.h
@@ -26,6 +26,7 @@
#include "core/string_ref.h"
#include "core/types.h"
#include "core/uint128.h"
+#include "core/value/timestamp_ns_value.h"
#include "parallel_hashmap/phmap_utils.h"
// Here is an empirical value.
@@ -111,6 +112,11 @@ struct DefaultHash<doris::DateV2Value<doris::DateV2ValueType>> {
}
};
+template <>
+struct DefaultHash<doris::TimeStampNsValue> {
+ size_t operator()(doris::TimeStampNsValue key) const { return int_hash64(key.epoch_nanos()); }
+};
+
template <>
struct DefaultHash<doris::TimestampTzValue> {
size_t operator()(doris::TimestampTzValue key) const {
@@ -168,6 +174,11 @@ inline size_t hash_crc32(doris::DateV2Value<doris::DateV2ValueType> u) {
return hash_crc32(u.to_date_int_val());
}
+template <>
+inline size_t hash_crc32(doris::TimeStampNsValue u) {
+ return hash_crc32(u.epoch_nanos());
+}
+
template <>
inline size_t hash_crc32(doris::TimestampTzValue u) {
return hash_crc32(u.to_date_int_val());
@@ -196,6 +207,7 @@ DEFINE_HASH(doris::Float64)
DEFINE_HASH(doris::VecDateTimeValue)
DEFINE_HASH(doris::DateV2Value<doris::DateTimeV2ValueType>)
DEFINE_HASH(doris::DateV2Value<doris::DateV2ValueType>)
+DEFINE_HASH(doris::TimeStampNsValue)
DEFINE_HASH(doris::TimestampTzValue)
DEFINE_HASH(unsigned __int128)
diff --git a/be/src/exec/common/join_utils.h b/be/src/exec/common/join_utils.h
index 5b48c4e140e..2a155176434 100644
--- a/be/src/exec/common/join_utils.h
+++ b/be/src/exec/common/join_utils.h
@@ -27,8 +27,33 @@
namespace doris {
+using AsofMixedDateTimeKey = unsigned __int128;
+
+template <typename ColumnType>
+struct AsofColumnIntType {
+ using type = typename ColumnType::value_type::underlying_value;
+};
+
+template <>
+struct AsofColumnIntType<ColumnTimeStampNs> {
+ using type = int64_t;
+};
+
+// DATETIMEV2 stores a packed civil value whose lowest field is microseconds. For a mixed
+// TIMESTAMP_NS/DATETIMEV2 ASOF comparison, append the final three nanosecond digits to that same
+// packed representation. The 128-bit intermediate also covers DATETIMEV2's full calendar range.
+template <typename DateValue>
+AsofMixedDateTimeKey asof_mixed_datetime_key(const DateValue& value) {
+ if constexpr (requires { value.nanosecond_remainder(); }) {
+ return static_cast<AsofMixedDateTimeKey>(value.to_datetime().to_date_int_val()) * 1000 +
+ value.nanosecond_remainder();
+ } else {
+ return static_cast<AsofMixedDateTimeKey>(value.to_date_int_val()) * 1000;
+ }
+}
+
// Devirtualize compare_at for ASOF JOIN supported column types.
-// ASOF JOIN only supports DateV2, DateTimeV2, and TimestampTZ.
+// ASOF JOIN only supports DateV2, DateTimeV2, TimestampNs, and TimestampTZ.
// Dispatches to the concrete ColumnVector<T> once so that all compare_at
// calls inside `func` are direct (non-virtual) calls.
// `func` receives a single argument: a const pointer to the concrete column
@@ -39,6 +64,8 @@ decltype(auto) asof_column_dispatch(const IColumn* col, Func&& func) {
return std::forward<Func>(func)(c_dv2);
} else if (const auto* c_dtv2 = check_and_get_column<ColumnDateTimeV2>(col)) {
return std::forward<Func>(func)(c_dtv2);
+ } else if (const auto* c_tsns = check_and_get_column<ColumnTimeStampNs>(col)) {
+ return std::forward<Func>(func)(c_tsns);
} else if (const auto* c_tstz = check_and_get_column<ColumnTimeStampTz>(col)) {
return std::forward<Func>(func)(c_tstz);
} else {
@@ -234,7 +261,8 @@ inline void try_convert_to_direct_mapping(
// ASOF JOIN index with inline values for cache-friendly branchless binary search.
// IntType is the integer representation of the ASOF column value:
-// uint32_t for DateV2, uint64_t for DateTimeV2 and TimestampTZ.
+// uint32_t for DateV2, uint64_t for DateTimeV2 and TimestampTZ,
+// int64_t for TimestampNs and AsofMixedDateTimeKey for mixed TimestampNs/DateTimeV2.
// Rows are sorted by asof_value during build, then materialized into SoA arrays
// so probe-side binary search only touches the ASOF values hot path.
template <typename IntType>
@@ -324,7 +352,9 @@ struct AsofIndexGroup {
// Type-erased container for all ASOF index groups.
// DateV2 -> uint32_t, DateTimeV2/TimestampTZ -> uint64_t.
-using AsofIndexVariant = std::variant<std::monostate, std::vector<AsofIndexGroup<uint32_t>>,
- std::vector<AsofIndexGroup<uint64_t>>>;
+using AsofIndexVariant =
+ std::variant<std::monostate, std::vector<AsofIndexGroup<uint32_t>>,
+ std::vector<AsofIndexGroup<uint64_t>>, std::vector<AsofIndexGroup<int64_t>>,
+ std::vector<AsofIndexGroup<AsofMixedDateTimeKey>>>;
} // namespace doris
diff --git a/be/src/exec/common/variant_util.cpp b/be/src/exec/common/variant_util.cpp
index a1ef298c08e..bdd4ad2f062 100644
--- a/be/src/exec/common/variant_util.cpp
+++ b/be/src/exec/common/variant_util.cpp
@@ -377,7 +377,8 @@ void get_column_by_type(const DataTypePtr& data_type, const std::string& name, T
PrimitiveType type = data_type->get_primitive_type();
if (is_int_or_bool(type) || is_string_type(type) || is_float_or_double(type) || is_ip(type) ||
- is_date_or_datetime(type) || type == PrimitiveType::TYPE_DATEV2) {
+ is_date_or_datetime(type) || type == PrimitiveType::TYPE_DATEV2 ||
+ type == PrimitiveType::TYPE_TIMESTAMP_NS) {
column.set_length(cast_set<int32_t>(data_type->get_size_of_value_in_memory()));
return;
}
@@ -568,6 +569,7 @@ bool is_bf_supported_by_fe_for_variant_subcolumn(FieldType type) {
case FieldType::OLAP_FIELD_TYPE_DATETIME:
case FieldType::OLAP_FIELD_TYPE_DATEV2:
case FieldType::OLAP_FIELD_TYPE_DATETIMEV2:
+ case FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS:
case FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ:
case FieldType::OLAP_FIELD_TYPE_DECIMAL:
case FieldType::OLAP_FIELD_TYPE_DECIMAL32:
diff --git a/be/src/exec/operator/hashjoin_build_sink.cpp b/be/src/exec/operator/hashjoin_build_sink.cpp
index cbf42346f33..048fc2b4c76 100644
--- a/be/src/exec/operator/hashjoin_build_sink.cpp
+++ b/be/src/exec/operator/hashjoin_build_sink.cpp
@@ -404,106 +404,113 @@ Status HashJoinBuildSinkLocalState::build_asof_index(Block& block) {
// Initialize reverse mapping (all build rows, including NULL ASOF values)
_shared_state->asof_build_row_to_bucket.resize(build_rows + 1, 0);
- // Dispatch on ASOF column type to create typed AsofIndexGroups with inline values.
- // Sub-group by actual key equality within each hash bucket (hash collisions),
- // extract integer representation of ASOF values, then sort by inline values.
- asof_column_dispatch(build_col_nested.get(), [&](const auto* typed_col) {
- using ColType = std::remove_const_t<std::remove_pointer_t<decltype(typed_col)>>;
-
- if constexpr (std::is_same_v<ColType, IColumn>) {
- throw Exception(ErrorCode::INTERNAL_ERROR,
- "Unsupported ASOF column type for inline optimization");
- } else {
- using IntType = typename ColType::value_type::underlying_value;
- const auto& col_data = typed_col->get_data();
-
- auto& groups = _shared_state->asof_index_groups
- .emplace<std::vector<AsofIndexGroup<IntType>>>();
-
- std::visit(
- Overload {
- [&](std::monostate&) {},
- [&](auto&& hash_table_ctx) {
- auto* hash_table = hash_table_ctx.hash_table.get();
- DORIS_CHECK(hash_table);
- const auto* build_keys = hash_table->get_build_keys();
- using KeyType = std::remove_const_t<
- std::remove_pointer_t<decltype(build_keys)>>;
- uint32_t next_group_id = 0;
-
- // Group rows by equality key within each hash bucket,
- // then sort each group by ASOF value only (pure integer compare).
- // This avoids the previous approach of sorting by (build_key, asof_value)
- // which required memcmp per comparison.
- //
- // For each bucket: walk the chain, find-or-create a group for
- // each distinct build_key, insert rows into the group with their
- // inline ASOF value. After all buckets are processed, sort each group.
-
- // Map from build_key -> group_id, reused across buckets.
- // Within a single hash bucket, the number of distinct keys is
- // typically very small (hash collisions are rare), so a flat
- // scan is efficient.
- struct KeyGroupEntry {
- KeyType key;
- uint32_t group_id;
- };
- std::vector<KeyGroupEntry> bucket_key_groups;
-
- {
- SCOPED_TIMER(_asof_index_group_timer);
- for (uint32_t bucket = 0; bucket <= bucket_size; ++bucket) {
- uint32_t row_idx = first_array[bucket];
- if (row_idx == 0) {
- continue;
- }
+ auto build_typed_groups = [&](const auto& col_data, auto key_getter) {
+ using IntType = std::remove_cvref_t<decltype(key_getter(col_data[0]))>;
+ auto& groups =
+ _shared_state->asof_index_groups.emplace<std::vector<AsofIndexGroup<IntType>>>();
- // For each row in this bucket's chain, find-or-create its group
- bucket_key_groups.clear();
- while (row_idx != 0) {
- DORIS_CHECK(row_idx <= build_rows);
- const auto& key = build_keys[row_idx];
-
- // Linear scan to find existing group for this key.
- // Bucket chains are short (avg ~1-2 distinct keys per bucket),
- // so this is faster than a hash map.
- uint32_t group_id = UINT32_MAX;
- for (const auto& entry : bucket_key_groups) {
- if (entry.key == key) {
- group_id = entry.group_id;
- break;
- }
- }
- if (group_id == UINT32_MAX) {
- group_id = next_group_id++;
- DORIS_CHECK(group_id == groups.size());
- groups.emplace_back();
- bucket_key_groups.push_back({key, group_id});
- }
+ std::visit(
+ Overload {
+ [&](std::monostate&) {},
+ [&](auto&& hash_table_ctx) {
+ auto* hash_table = hash_table_ctx.hash_table.get();
+ DORIS_CHECK(hash_table);
+ const auto* build_keys = hash_table->get_build_keys();
+ using KeyType = std::remove_const_t<
+ std::remove_pointer_t<decltype(build_keys)>>;
+ uint32_t next_group_id = 0;
+
+ // Group rows by equality key within each hash bucket,
+ // then sort each group by ASOF value only (pure integer compare).
+ // This avoids the previous approach of sorting by (build_key, asof_value)
+ // which required memcmp per comparison.
+ //
+ // For each bucket: walk the chain, find-or-create a group for
+ // each distinct build_key, insert rows into the group with their
+ // inline ASOF value. After all buckets are processed, sort each group.
+
+ // Map from build_key -> group_id, reused across buckets.
+ // Within a single hash bucket, the number of distinct keys is
+ // typically very small (hash collisions are rare), so a flat
+ // scan is efficient.
+ struct KeyGroupEntry {
+ KeyType key;
+ uint32_t group_id;
+ };
+ std::vector<KeyGroupEntry> bucket_key_groups;
+
+ {
+ SCOPED_TIMER(_asof_index_group_timer);
+ for (uint32_t bucket = 0; bucket <= bucket_size; ++bucket) {
+ uint32_t row_idx = first_array[bucket];
+ if (row_idx == 0) {
+ continue;
+ }
- _shared_state->asof_build_row_to_bucket[row_idx] =
- group_id;
- if (!(nullable_col &&
- nullable_col->is_null_at(row_idx))) {
- groups[group_id].add_row(
- col_data[row_idx].to_date_int_val(),
- row_idx);
+ // For each row in this bucket's chain, find-or-create its group
+ bucket_key_groups.clear();
+ while (row_idx != 0) {
+ DORIS_CHECK(row_idx <= build_rows);
+ const auto& key = build_keys[row_idx];
+
+ // Linear scan to find existing group for this key.
+ // Bucket chains are short (avg ~1-2 distinct keys per bucket),
+ // so this is faster than a hash map.
+ uint32_t group_id = UINT32_MAX;
+ for (const auto& entry : bucket_key_groups) {
+ if (entry.key == key) {
+ group_id = entry.group_id;
+ break;
}
+ }
+ if (group_id == UINT32_MAX) {
+ group_id = next_group_id++;
+ DORIS_CHECK(group_id == groups.size());
+ groups.emplace_back();
+ bucket_key_groups.push_back({key, group_id});
+ }
- row_idx = next_array[row_idx];
+ _shared_state->asof_build_row_to_bucket[row_idx] = group_id;
+ if (!(nullable_col && nullable_col->is_null_at(row_idx))) {
+ groups[group_id].add_row(key_getter(col_data[row_idx]),
+ row_idx);
}
+ row_idx = next_array[row_idx];
}
}
+ }
- // Sort each group by ASOF value only (pure integer comparison).
- {
- SCOPED_TIMER(_asof_index_sort_timer);
- for (auto& group : groups) {
- group.sort_and_finalize();
- }
+ // Sort each group by ASOF value only (pure integer comparison).
+ {
+ SCOPED_TIMER(_asof_index_sort_timer);
+ for (auto& group : groups) {
+ group.sort_and_finalize();
}
- }},
- _shared_state->hash_table_variant_vector.front()->method_variant);
+ }
+ }},
+ _shared_state->hash_table_variant_vector.front()->method_variant);
+ };
+
+ // Dispatch on the build column once. Homogeneous joins retain their compact physical key;
+ // mixed TIMESTAMP_NS/DATETIMEV2 joins use the exact common civil key.
+ // Sub-group by actual key equality within each hash bucket (hash collisions),
+ // extract integer representation of ASOF values, then sort by inline values.
+ asof_column_dispatch(build_col_nested.get(), [&](const auto* typed_col) {
+ using ColType = std::remove_const_t<std::remove_pointer_t<decltype(typed_col)>>;
+ if constexpr (std::is_same_v<ColType, IColumn>) {
+ throw Exception(ErrorCode::INTERNAL_ERROR,
+ "Unsupported ASOF column type for inline optimization");
+ } else {
+ const auto& col_data = typed_col->get_data();
+ if (p._asof_mixed_timestamp_ns_datetimev2) {
+ DORIS_CHECK((std::is_same_v<ColType, ColumnDateTimeV2> ||
+ std::is_same_v<ColType, ColumnTimeStampNs>));
+ build_typed_groups(
+ col_data, [](const auto& value) { return asof_mixed_datetime_key(value); });
+ } else {
+ build_typed_groups(col_data,
+ [](const auto& value) { return value.to_date_int_val(); });
+ }
}
});
@@ -786,7 +793,15 @@ Status HashJoinBuildSinkOperatorX::init(const TPlanNode& tnode, RuntimeState* st
DORIS_CHECK(full_conjunct->root());
DORIS_CHECK(full_conjunct->root()->get_num_children() == 2);
_asof_opcode = full_conjunct->root()->op();
+ const PrimitiveType probe_type =
+ remove_nullable(full_conjunct->root()->get_child(0)->data_type())
+ ->get_primitive_type();
auto right_child_expr = full_conjunct->root()->get_child(1);
+ const PrimitiveType build_type =
+ remove_nullable(right_child_expr->data_type())->get_primitive_type();
+ _asof_mixed_timestamp_ns_datetimev2 =
+ (probe_type == TYPE_TIMESTAMP_NS && build_type == TYPE_DATETIMEV2) ||
+ (probe_type == TYPE_DATETIMEV2 && build_type == TYPE_TIMESTAMP_NS);
_asof_build_side_expr = std::make_shared<VExprContext>(right_child_expr);
}
diff --git a/be/src/exec/operator/hashjoin_build_sink.h b/be/src/exec/operator/hashjoin_build_sink.h
index af2155bab1c..412bf0a6290 100644
--- a/be/src/exec/operator/hashjoin_build_sink.h
+++ b/be/src/exec/operator/hashjoin_build_sink.h
@@ -195,6 +195,7 @@ private:
// Prepared against build child's row_desc directly (no intermediate tuple needed)
VExprContextSPtr _asof_build_side_expr;
TExprOpcode::type _asof_opcode = TExprOpcode::INVALID_OPCODE;
+ bool _asof_mixed_timestamp_ns_datetimev2 = false;
bool _use_shared_hash_table = false;
std::atomic<bool> _signaled = false;
diff --git a/be/src/exec/operator/join/process_hash_table_probe_impl.h b/be/src/exec/operator/join/process_hash_table_probe_impl.h
index 11bc670db09..88767d467c7 100644
--- a/be/src/exec/operator/join/process_hash_table_probe_impl.h
+++ b/be/src/exec/operator/join/process_hash_table_probe_impl.h
@@ -310,8 +310,8 @@ uint32_t ProcessHashTableProbe<JoinOpType>::
bool is_strict = shared_state->asof_inequality_is_strict;
// Two-phase probe with compile-time dispatched binary search
- [[maybe_unused]] auto probe_with_index = [&](const auto& asof_groups,
- const auto* typed_probe_col) -> uint32_t {
+ [[maybe_unused]] auto probe_with_index =
+ [&](const auto& asof_groups, const auto* typed_probe_col, auto key_getter) -> uint32_t {
using IntType =
typename std::remove_reference_t<decltype(asof_groups)>::value_type::int_type;
const auto& probe_data = typed_probe_col->get_data();
@@ -390,7 +390,7 @@ uint32_t ProcessHashTableProbe<JoinOpType>::
uint32_t group_id = bucket_row_to_group[match];
DORIS_CHECK(group_id < asof_groups.size());
resolved_group_ids[i] = group_id + 1;
- probe_values[i] = static_cast<IntType>(probe_data[pi].to_date_int_val());
+ probe_values[i] = key_getter(probe_data[pi]);
}
}
@@ -473,16 +473,44 @@ uint32_t ProcessHashTableProbe<JoinOpType>::
},
[&](std::vector<AsofIndexGroup<uint32_t>>& groups) -> uint32_t {
// DateV2
- return probe_with_index(groups,
- assert_cast<const ColumnDateV2*>(probe_col));
+ return probe_with_index(
+ groups, assert_cast<const ColumnDateV2*>(probe_col),
+ [](const auto& value) { return value.to_date_int_val(); });
},
[&](std::vector<AsofIndexGroup<uint64_t>>& groups) -> uint32_t {
// DateTimeV2 or TimestampTZ
if (const auto* c = check_and_get_column<ColumnDateTimeV2>(probe_col)) {
- return probe_with_index(groups, c);
+ return probe_with_index(groups, c, [](const auto& value) {
+ return value.to_date_int_val();
+ });
}
- return probe_with_index(groups,
- assert_cast<const ColumnTimeStampTz*>(probe_col));
+ return probe_with_index(
+ groups, assert_cast<const ColumnTimeStampTz*>(probe_col),
+ [](const auto& value) { return value.to_date_int_val(); });
+ },
+ [&](std::vector<AsofIndexGroup<int64_t>>& groups) -> uint32_t {
+ return probe_with_index(
+ groups, assert_cast<const ColumnTimeStampNs*>(probe_col),
+ [](const auto& value) { return value.to_date_int_val(); });
+ },
+ [&](std::vector<AsofIndexGroup<AsofMixedDateTimeKey>>& groups) -> uint32_t {
+ return asof_column_dispatch(
+ probe_col, [&](const auto* typed_col) -> uint32_t {
+ using ColType = std::remove_const_t<
+ std::remove_pointer_t<decltype(typed_col)>>;
+ if constexpr (std::is_same_v<ColType, ColumnDateTimeV2> ||
+ std::is_same_v<ColType, ColumnTimeStampNs>) {
+ return probe_with_index(
+ groups, typed_col, [](const auto& value) {
+ return asof_mixed_datetime_key(value);
+ });
+ } else {
+ throw Exception(ErrorCode::INTERNAL_ERROR,
+ "Mixed TIMESTAMP_NS/DATETIMEV2 ASOF "
+ "index received {}",
+ probe_col->get_name());
+ }
+ });
}},
shared_state->asof_index_groups);
diff --git a/be/src/exec/operator/scan_operator.cpp b/be/src/exec/operator/scan_operator.cpp
index 5850233df4b..6aaeaad4c4e 100644
--- a/be/src/exec/operator/scan_operator.cpp
+++ b/be/src/exec/operator/scan_operator.cpp
@@ -224,9 +224,9 @@ Status ScanLocalState<Derived>::open(RuntimeState* state) {
return status;
}
-static void init_slot_value_range(
+void ScanLocalStateBase::_init_slot_value_range(
phmap::flat_hash_map<int, ColumnValueRangeType>& slot_id_to_value_range,
- SlotDescriptor* slot, const DataTypePtr type_desc) {
+ SlotDescriptor* slot, const DataTypePtr& type_desc) {
switch (type_desc->get_primitive_type()) {
#define M(NAME) \
case TYPE_##NAME: { \
@@ -249,6 +249,7 @@ static void init_slot_value_range(
M(DATETIME) \
M(DATEV2) \
M(DATETIMEV2) \
+ M(TIMESTAMP_NS) \
M(TIMESTAMPTZ) \
M(VARCHAR) \
M(STRING) \
@@ -299,7 +300,7 @@ Status ScanLocalState<Derived>::_normalize_conjuncts(RuntimeState* state) {
continue;
}
}
- init_slot_value_range(_slot_id_to_value_range, slot, slot->type());
+ _init_slot_value_range(_slot_id_to_value_range, slot, slot->type());
_slot_id_to_predicates.insert(
{slot->id(), std::vector<std::shared_ptr<ColumnPredicate>>()});
}
@@ -307,7 +308,7 @@ Status ScanLocalState<Derived>::_normalize_conjuncts(RuntimeState* state) {
get_cast_types_for_variants();
for (const auto& [colname, type] : _cast_types_for_variants) {
auto* slot = p._slot_id_to_slot_desc[p._colname_to_slot_id[colname]];
- init_slot_value_range(_slot_id_to_value_range, slot, type);
+ _init_slot_value_range(_slot_id_to_value_range, slot, type);
_slot_id_to_predicates.insert(
{slot->id(), std::vector<std::shared_ptr<ColumnPredicate>>()});
}
@@ -899,12 +900,13 @@ Status ScanLocalStateBase::_change_value_range(bool is_equal_op,
func(temp_range, to_olap_filter_type(fn_name), tmp_value);
}
} else if constexpr ((PrimitiveType == TYPE_DECIMALV2) || (PrimitiveType == TYPE_DATETIMEV2) ||
- (PrimitiveType == TYPE_TINYINT) || (PrimitiveType == TYPE_SMALLINT) ||
- (PrimitiveType == TYPE_INT) || (PrimitiveType == TYPE_BIGINT) ||
- (PrimitiveType == TYPE_LARGEINT) || (PrimitiveType == TYPE_FLOAT) ||
- (PrimitiveType == TYPE_DOUBLE) || (PrimitiveType == TYPE_IPV4) ||
- (PrimitiveType == TYPE_IPV6) || (PrimitiveType == TYPE_DECIMAL32) ||
- (PrimitiveType == TYPE_DECIMAL64) || (PrimitiveType == TYPE_DECIMAL128I) ||
+ (PrimitiveType == TYPE_TIMESTAMP_NS) || (PrimitiveType == TYPE_TINYINT) ||
+ (PrimitiveType == TYPE_SMALLINT) || (PrimitiveType == TYPE_INT) ||
+ (PrimitiveType == TYPE_BIGINT) || (PrimitiveType == TYPE_LARGEINT) ||
+ (PrimitiveType == TYPE_FLOAT) || (PrimitiveType == TYPE_DOUBLE) ||
+ (PrimitiveType == TYPE_IPV4) || (PrimitiveType == TYPE_IPV6) ||
+ (PrimitiveType == TYPE_DECIMAL32) || (PrimitiveType == TYPE_DECIMAL64) ||
+ (PrimitiveType == TYPE_DECIMAL128I) ||
(PrimitiveType == TYPE_DECIMAL256) || (PrimitiveType == TYPE_BOOLEAN) ||
(PrimitiveType == TYPE_DATEV2) || (PrimitiveType == TYPE_TIMESTAMPTZ) ||
(PrimitiveType == TYPE_DATETIME) || is_string_type(PrimitiveType)) {
diff --git a/be/src/exec/operator/scan_operator.h b/be/src/exec/operator/scan_operator.h
index 629bf18ad20..05cab007d90 100644
--- a/be/src/exec/operator/scan_operator.h
+++ b/be/src/exec/operator/scan_operator.h
@@ -180,6 +180,10 @@ protected:
return Status::OK();
}
+ static void _init_slot_value_range(
+ phmap::flat_hash_map<int, ColumnValueRangeType>& slot_id_to_value_range,
+ SlotDescriptor* slot, const DataTypePtr& type_desc);
+
// Non-templated normalize methods, moved here to avoid re-compilation per Derived type.
Status _eval_const_conjuncts(VExprContext* expr_ctx, PushDownType* pdt);
Status _normalize_bloom_filter(VExprContext* expr_ctx, const VExprSPtr& root,
diff --git a/be/src/exec/pipeline/dependency.h b/be/src/exec/pipeline/dependency.h
index f7a90ea0c34..0442964df80 100644
--- a/be/src/exec/pipeline/dependency.h
+++ b/be/src/exec/pipeline/dependency.h
@@ -572,7 +572,8 @@ struct HashJoinSharedState : public JoinSharedState {
bool asof_inequality_is_strict = false;
// ASOF JOIN pre-sorted index with inline values for O(log K) branchless lookup
- // Typed AsofIndexGroups stored in a variant (uint32_t for DateV2, uint64_t for DateTimeV2/TimestampTZ)
+ // Typed AsofIndexGroups stored in a variant (uint32_t for DateV2, uint64_t for
+ // DateTimeV2/TimestampTZ, int64_t for TimestampNs)
AsofIndexVariant asof_index_groups;
// build_row_index -> bucket_id for O(1) reverse lookup
std::vector<uint32_t> asof_build_row_to_bucket;
diff --git a/be/src/exec/runtime_filter/runtime_filter_wrapper.cpp b/be/src/exec/runtime_filter/runtime_filter_wrapper.cpp
index 330bcb6895e..9f9216d9455 100644
--- a/be/src/exec/runtime_filter/runtime_filter_wrapper.cpp
+++ b/be/src/exec/runtime_filter/runtime_filter_wrapper.cpp
@@ -285,6 +285,8 @@ Status RuntimeFilterWrapper::_assign(const PInFilter& in_filter, bool contain_nu
}
case TYPE_TIMESTAMPTZ:
[[fallthrough]];
+ case TYPE_TIMESTAMP_NS:
+ [[fallthrough]];
case TYPE_DATETIMEV2: {
batch_assign(in_filter, [](std::shared_ptr<HybridSetBase>& set, PColumnValue& column) {
auto date_v2_val = column.longval();
@@ -451,6 +453,8 @@ Status RuntimeFilterWrapper::_assign(const PMinMaxFilter& minmax_filter, bool co
}
case TYPE_TIMESTAMPTZ:
[[fallthrough]];
+ case TYPE_TIMESTAMP_NS:
+ [[fallthrough]];
case TYPE_DATETIMEV2: {
int64_t min_val = minmax_filter.min_val().longval();
int64_t max_val = minmax_filter.max_val().longval();
diff --git a/be/src/exec/runtime_filter/utils.h b/be/src/exec/runtime_filter/utils.h
index 37aa47cde85..2ab8f65f2f2 100644
--- a/be/src/exec/runtime_filter/utils.h
+++ b/be/src/exec/runtime_filter/utils.h
@@ -25,6 +25,7 @@
#include "core/extended_types.h"
#include "core/types.h"
#include "core/value/large_int_value.h"
+#include "core/value/timestamp_ns_value.h"
#include "core/value/timestamptz_value.h"
#include "exec/runtime_filter/runtime_filter_definitions.h"
#include "exprs/vexpr_fwd.h"
@@ -76,6 +77,8 @@ auto get_convertor() {
return [](PColumnValue* value, const T& data) {
value->set_longval(data.to_date_int_val());
};
+ } else if constexpr (std::is_same_v<T, TimeStampNsValue>) {
+ return [](PColumnValue* value, const T& data) { value->set_longval(data.epoch_nanos()); };
} else {
throw Exception(ErrorCode::INTERNAL_ERROR,
"runtime filter data convertor meet invalid type {}", typeid(T).name());
diff --git a/be/src/exec/sink/vrow_distribution.h b/be/src/exec/sink/vrow_distribution.h
index c00e7c99d1c..7ecf8740e57 100644
--- a/be/src/exec/sink/vrow_distribution.h
+++ b/be/src/exec/sink/vrow_distribution.h
@@ -133,6 +133,7 @@ public:
auto [part_ctxs, part_funcs] = _get_partition_function();
for (auto part_ctx : part_ctxs) {
RETURN_IF_ERROR(part_ctx->prepare(_state, *output_row_desc));
+ part_ctx->set_auto_partition_boundary_context();
RETURN_IF_ERROR(part_ctx->open(_state));
}
}
diff --git a/be/src/exprs/aggregate/aggregate_function_approx_count_distinct.cpp b/be/src/exprs/aggregate/aggregate_function_approx_count_distinct.cpp
index 314476b2e6e..bb53f64391d 100644
--- a/be/src/exprs/aggregate/aggregate_function_approx_count_distinct.cpp
+++ b/be/src/exprs/aggregate/aggregate_function_approx_count_distinct.cpp
@@ -31,7 +31,7 @@ AggregateFunctionPtr create_aggregate_function_approx_count_distinct(
TYPE_BOOLEAN, TYPE_TINYINT, TYPE_SMALLINT, TYPE_INT, TYPE_BIGINT, TYPE_LARGEINT,
TYPE_FLOAT, TYPE_DOUBLE, TYPE_DECIMALV2, TYPE_DECIMAL32, TYPE_DECIMAL64,
TYPE_DECIMAL128I, TYPE_DECIMAL256, TYPE_VARCHAR, TYPE_DATEV2, TYPE_DATETIMEV2,
- TYPE_IPV4, TYPE_IPV6,
+ TYPE_TIMESTAMP_NS, TYPE_IPV4, TYPE_IPV6,
TYPE_TIMESTAMPTZ>::create<AggregateFunctionApproxCountDistinct>(argument_types,
result_is_nullable,
attr);
diff --git a/be/src/exprs/aggregate/aggregate_function_approx_count_distinct.h b/be/src/exprs/aggregate/aggregate_function_approx_count_distinct.h
index 6bf92387fdd..82420f43a83 100644
--- a/be/src/exprs/aggregate/aggregate_function_approx_count_distinct.h
+++ b/be/src/exprs/aggregate/aggregate_function_approx_count_distinct.h
@@ -96,8 +96,9 @@ public:
void add(AggregateDataPtr __restrict place, const IColumn** columns, ssize_t row_num,
Arena&) const override {
if constexpr (is_decimal(type) || is_int_or_bool(type) || is_ip(type) ||
- is_date_type(type) || is_timestamptz_type(type) || is_float_or_double(type) ||
- type == TYPE_TIME || type == TYPE_TIMEV2) {
+ is_date_type(type) || is_timestamp_ns_type(type) ||
+ is_timestamptz_type(type) || is_float_or_double(type) || type == TYPE_TIME ||
+ type == TYPE_TIMEV2) {
auto column =
assert_cast<const ColumnDataType*, TypeCheckOnRelease::DISABLE>(columns[0]);
auto value = column->get_element(row_num);
diff --git a/be/src/exprs/aggregate/aggregate_function_array_agg.h b/be/src/exprs/aggregate/aggregate_function_array_agg.h
index b82af1f2d9b..e4a0935eb47 100644
--- a/be/src/exprs/aggregate/aggregate_function_array_agg.h
+++ b/be/src/exprs/aggregate/aggregate_function_array_agg.h
@@ -226,7 +226,7 @@ struct AggregateFunctionArrayAggData<T> {
template <PrimitiveType T>
requires(!is_string_type(T) && !is_int_or_bool(T) && !is_float_or_double(T) && !is_decimal(T) &&
- !is_date_type(T) && !is_ip(T))
+ !is_date_type(T) && !is_timestamp_ns_type(T) && !is_ip(T))
struct AggregateFunctionArrayAggData<T> {
static constexpr PrimitiveType PType = T;
using ElementType = StringRef;
@@ -377,7 +377,8 @@ public:
vec.insert_from(vec_src, i);
} else if constexpr (!is_string_type(Data::PType) && !is_int_or_bool(Data::PType) &&
!is_float_or_double(Data::PType) && !is_decimal(Data::PType) &&
- !is_date_type(Data::PType) && !is_ip(Data::PType)) {
+ !is_date_type(Data::PType) && !is_timestamp_ns_type(Data::PType) &&
+ !is_ip(Data::PType)) {
auto& vec = col_null->get_nested_column();
vec.insert_from(col_src.get_nested_column(), i);
} else {
diff --git a/be/src/exprs/aggregate/aggregate_function_collect.h b/be/src/exprs/aggregate/aggregate_function_collect.h
index 63a3c634822..007e78b397a 100644
--- a/be/src/exprs/aggregate/aggregate_function_collect.h
+++ b/be/src/exprs/aggregate/aggregate_function_collect.h
@@ -306,7 +306,7 @@ struct AggregateFunctionCollectListData<T, HasLimit> {
template <PrimitiveType T, bool HasLimit>
requires(!is_string_type(T) && !is_int_or_bool(T) && !is_float_or_double(T) && !is_decimal(T) &&
- !is_date_type(T) && !is_ip(T) && !is_timestamptz_type(T))
+ !is_date_type(T) && !is_timestamp_ns_type(T) && !is_ip(T) && !is_timestamptz_type(T))
struct AggregateFunctionCollectListData<T, HasLimit> {
static constexpr PrimitiveType PType = T;
using ElementType = StringRef;
diff --git a/be/src/exprs/aggregate/aggregate_function_histogram.cpp b/be/src/exprs/aggregate/aggregate_function_histogram.cpp
index 696d711fc65..bf0dc461528 100644
--- a/be/src/exprs/aggregate/aggregate_function_histogram.cpp
+++ b/be/src/exprs/aggregate/aggregate_function_histogram.cpp
@@ -40,11 +40,11 @@ AggregateFunctionPtr create_aggregate_function_histogram(const std::string& name
const bool result_is_nullable,
const AggregateFunctionAttr& attr) {
assert_arity_range(name, argument_types, 1, 2);
- using creator =
- creator_with_type_list<TYPE_BOOLEAN, TYPE_TINYINT, TYPE_SMALLINT, TYPE_INT, TYPE_BIGINT,
- TYPE_LARGEINT, TYPE_FLOAT, TYPE_DOUBLE, TYPE_DECIMAL32,
- TYPE_DECIMAL64, TYPE_DECIMAL128I, TYPE_DECIMAL256, TYPE_VARCHAR,
- TYPE_DATEV2, TYPE_DATETIMEV2, TYPE_TIMESTAMPTZ>;
+ using creator = creator_with_type_list<TYPE_BOOLEAN, TYPE_TINYINT, TYPE_SMALLINT, TYPE_INT,
+ TYPE_BIGINT, TYPE_LARGEINT, TYPE_FLOAT, TYPE_DOUBLE,
+ TYPE_DECIMAL32, TYPE_DECIMAL64, TYPE_DECIMAL128I,
+ TYPE_DECIMAL256, TYPE_VARCHAR, TYPE_DATEV2,
+ TYPE_DATETIMEV2, TYPE_TIMESTAMP_NS, TYPE_TIMESTAMPTZ>;
if (argument_types.size() == 2) {
return creator::create<HistogramWithInputParam, AggregateFunctionHistogramData>(
argument_types, result_is_nullable, attr);
@@ -59,4 +59,4 @@ void register_aggregate_function_histogram(AggregateFunctionSimpleFactory& facto
factory.register_alias("histogram", "hist");
}
-} // namespace doris
\ No newline at end of file
+} // namespace doris
diff --git a/be/src/exprs/aggregate/aggregate_function_map_v2.cpp b/be/src/exprs/aggregate/aggregate_function_map_v2.cpp
index 309c079a198..81001dc6c15 100644
--- a/be/src/exprs/aggregate/aggregate_function_map_v2.cpp
+++ b/be/src/exprs/aggregate/aggregate_function_map_v2.cpp
@@ -54,6 +54,7 @@ AggregateFunctionPtr create_aggregate_function_map_agg_v2(const std::string& nam
case PrimitiveType::TYPE_VARCHAR:
case PrimitiveType::TYPE_DATEV2:
case PrimitiveType::TYPE_DATETIMEV2:
+ case PrimitiveType::TYPE_TIMESTAMP_NS:
case PrimitiveType::TYPE_TIMEV2:
case PrimitiveType::TYPE_TIMESTAMPTZ:
return create_agg_function_map_agg_v2(argument_types, result_is_nullable, attr);
@@ -68,4 +69,4 @@ void register_aggregate_function_map_agg_v2(AggregateFunctionSimpleFactory& fact
factory.register_function_both("map_agg_v2", create_aggregate_function_map_agg_v2);
}
-} // namespace doris
\ No newline at end of file
+} // namespace doris
diff --git a/be/src/exprs/aggregate/aggregate_function_min_max_by.cpp b/be/src/exprs/aggregate/aggregate_function_min_max_by.cpp
index 7d237018d40..69611177eaa 100644
--- a/be/src/exprs/aggregate/aggregate_function_min_max_by.cpp
+++ b/be/src/exprs/aggregate/aggregate_function_min_max_by.cpp
@@ -35,6 +35,10 @@ std::unique_ptr<MaxMinValueBase> create_max_min_value(const DataTypePtr& type, i
}
return true;
};
+ if (type->get_primitive_type() == TYPE_TIMESTAMP_NS) {
+ call(DispatchDataType<TYPE_TIMESTAMP_NS>());
+ return result;
+ }
if (dispatch_switch_scalar(type->get_primitive_type(), call)) {
return result;
}
diff --git a/be/src/exprs/aggregate/aggregate_function_min_max_by.h b/be/src/exprs/aggregate/aggregate_function_min_max_by.h
index 153d9ce2198..3318e8565b8 100644
--- a/be/src/exprs/aggregate/aggregate_function_min_max_by.h
+++ b/be/src/exprs/aggregate/aggregate_function_min_max_by.h
@@ -337,6 +337,12 @@ AggregateFunctionPtr create_aggregate_function_min_max_by(const String& name,
}
return true;
};
+ // Keep nano dispatch local because the shared scalar dispatcher is also used by templates
+ // that assume arithmetic support.
+ if (argument_types[1]->get_primitive_type() == TYPE_TIMESTAMP_NS) {
+ call(DispatchDataType<TYPE_TIMESTAMP_NS>());
+ return result;
+ }
if (dispatch_switch_scalar(argument_types[1]->get_primitive_type(), call)) {
return result;
}
diff --git a/be/src/exprs/aggregate/aggregate_function_min_max_impl.h b/be/src/exprs/aggregate/aggregate_function_min_max_impl.h
index 7c5dd5fd651..6f8a9a16926 100644
--- a/be/src/exprs/aggregate/aggregate_function_min_max_impl.h
+++ b/be/src/exprs/aggregate/aggregate_function_min_max_impl.h
@@ -60,6 +60,10 @@ AggregateFunctionPtr create_aggregate_function_single_value(const String& name,
return creator_without_type::create_unary_arguments<
AggregateFunctionsSingleValue<Data<SingleValueDataFixed<TYPE_DATETIMEV2>>>>(
argument_types, result_is_nullable, attr);
+ case PrimitiveType::TYPE_TIMESTAMP_NS:
+ return creator_without_type::create_unary_arguments<
+ AggregateFunctionsSingleValue<Data<SingleValueDataFixed<TYPE_TIMESTAMP_NS>>>>(
+ argument_types, result_is_nullable, attr);
case PrimitiveType::TYPE_TIMESTAMPTZ:
return creator_without_type::create_unary_arguments<
AggregateFunctionsSingleValue<Data<SingleValueDataFixed<TYPE_TIMESTAMPTZ>>>>(
diff --git a/be/src/exprs/aggregate/aggregate_function_sequence_match.cpp b/be/src/exprs/aggregate/aggregate_function_sequence_match.cpp
index 755dd6a2520..46061d26e42 100644
--- a/be/src/exprs/aggregate/aggregate_function_sequence_match.cpp
+++ b/be/src/exprs/aggregate/aggregate_function_sequence_match.cpp
@@ -49,6 +49,9 @@ AggregateFunctionPtr create_aggregate_function_sequence_base(const std::string&
case TYPE_DATETIMEV2:
return creator_without_type::create<AggregateFunction<TYPE_DATETIMEV2>>(
argument_types, result_is_nullable, attr);
+ case TYPE_TIMESTAMP_NS:
+ return creator_without_type::create<AggregateFunction<TYPE_TIMESTAMP_NS>>(
+ argument_types, result_is_nullable, attr);
case TYPE_DATEV2:
return creator_without_type::create<AggregateFunction<TYPE_DATEV2>>(
argument_types, result_is_nullable, attr);
diff --git a/be/src/exprs/aggregate/aggregate_function_topn_array.cpp b/be/src/exprs/aggregate/aggregate_function_topn_array.cpp
index 4e61d4eaf5a..61ba128ebca 100644
--- a/be/src/exprs/aggregate/aggregate_function_topn_array.cpp
+++ b/be/src/exprs/aggregate/aggregate_function_topn_array.cpp
@@ -28,10 +28,12 @@ using ImplArray = AggregateFunctionTopNImplArray<T, false>;
template <PrimitiveType T>
using ImplArrayWithDefault = AggregateFunctionTopNImplArray<T, true>;
-using topn_array_creator = creator_with_type_list<
- TYPE_BOOLEAN, TYPE_TINYINT, TYPE_SMALLINT, TYPE_INT, TYPE_BIGINT, TYPE_LARGEINT, TYPE_FLOAT,
- TYPE_DOUBLE, TYPE_DECIMAL32, TYPE_DECIMAL64, TYPE_DECIMAL128I, TYPE_DECIMAL256,
- TYPE_VARCHAR, TYPE_DATEV2, TYPE_DATETIMEV2, TYPE_TIMESTAMPTZ, TYPE_IPV4, TYPE_IPV6>;
+using topn_array_creator =
+ creator_with_type_list<TYPE_BOOLEAN, TYPE_TINYINT, TYPE_SMALLINT, TYPE_INT, TYPE_BIGINT,
+ TYPE_LARGEINT, TYPE_FLOAT, TYPE_DOUBLE, TYPE_DECIMAL32,
+ TYPE_DECIMAL64, TYPE_DECIMAL128I, TYPE_DECIMAL256, TYPE_VARCHAR,
+ TYPE_DATEV2, TYPE_DATETIMEV2, TYPE_TIMESTAMP_NS, TYPE_TIMESTAMPTZ,
+ TYPE_IPV4, TYPE_IPV6>;
AggregateFunctionPtr create_aggregate_function_topn_array(const std::string& name,
const DataTypes& argument_types,
diff --git a/be/src/exprs/aggregate/aggregate_function_topn_weighted.cpp b/be/src/exprs/aggregate/aggregate_function_topn_weighted.cpp
index 92dd6ac54a5..a84b92b5b3d 100644
--- a/be/src/exprs/aggregate/aggregate_function_topn_weighted.cpp
+++ b/be/src/exprs/aggregate/aggregate_function_topn_weighted.cpp
@@ -28,10 +28,12 @@ using ImplWeight = AggregateFunctionTopNImplWeight<T, false>;
template <PrimitiveType T>
using ImplWeightWithDefault = AggregateFunctionTopNImplWeight<T, true>;
-using topn_weighted_creator = creator_with_type_list<
- TYPE_BOOLEAN, TYPE_TINYINT, TYPE_SMALLINT, TYPE_INT, TYPE_BIGINT, TYPE_LARGEINT, TYPE_FLOAT,
- TYPE_DOUBLE, TYPE_DECIMAL32, TYPE_DECIMAL64, TYPE_DECIMAL128I, TYPE_DECIMAL256,
- TYPE_VARCHAR, TYPE_DATEV2, TYPE_DATETIMEV2, TYPE_TIMESTAMPTZ, TYPE_IPV4, TYPE_IPV6>;
+using topn_weighted_creator =
+ creator_with_type_list<TYPE_BOOLEAN, TYPE_TINYINT, TYPE_SMALLINT, TYPE_INT, TYPE_BIGINT,
+ TYPE_LARGEINT, TYPE_FLOAT, TYPE_DOUBLE, TYPE_DECIMAL32,
+ TYPE_DECIMAL64, TYPE_DECIMAL128I, TYPE_DECIMAL256, TYPE_VARCHAR,
+ TYPE_DATEV2, TYPE_DATETIMEV2, TYPE_TIMESTAMP_NS, TYPE_TIMESTAMPTZ,
+ TYPE_IPV4, TYPE_IPV6>;
AggregateFunctionPtr create_aggregate_function_topn_weighted(const std::string& name,
const DataTypes& argument_types,
diff --git a/be/src/exprs/aggregate/aggregate_function_uniq.cpp b/be/src/exprs/aggregate/aggregate_function_uniq.cpp
index 34f3abbbe9c..2d568931bcf 100644
--- a/be/src/exprs/aggregate/aggregate_function_uniq.cpp
+++ b/be/src/exprs/aggregate/aggregate_function_uniq.cpp
@@ -53,12 +53,13 @@ AggregateFunctionPtr create_aggregate_function_uniq(const std::string& name,
argument_types, result_is_nullable, attr);
}
}
- return creator_with_type_list<
- TYPE_BOOLEAN, TYPE_TINYINT, TYPE_SMALLINT, TYPE_INT, TYPE_BIGINT, TYPE_LARGEINT,
- TYPE_DECIMAL32, TYPE_DECIMAL64, TYPE_DECIMAL128I, TYPE_DECIMAL256, TYPE_VARCHAR,
- TYPE_ARRAY, TYPE_FLOAT, TYPE_DOUBLE, TYPE_DATEV2, TYPE_DATETIMEV2, TYPE_TIMESTAMPTZ,
- TYPE_VARBINARY>::create<AggregateFunctionUniq, Data>(argument_types, result_is_nullable,
- attr);
+ return creator_with_type_list<TYPE_BOOLEAN, TYPE_TINYINT, TYPE_SMALLINT, TYPE_INT, TYPE_BIGINT,
+ TYPE_LARGEINT, TYPE_DECIMAL32, TYPE_DECIMAL64, TYPE_DECIMAL128I,
+ TYPE_DECIMAL256, TYPE_VARCHAR, TYPE_ARRAY, TYPE_FLOAT,
+ TYPE_DOUBLE, TYPE_DATEV2, TYPE_DATETIMEV2, TYPE_TIMESTAMP_NS,
+ TYPE_TIMESTAMPTZ, TYPE_VARBINARY>::create<AggregateFunctionUniq,
+ Data>(
+ argument_types, result_is_nullable, attr);
}
void register_aggregate_function_uniq(AggregateFunctionSimpleFactory& factory) {
diff --git a/be/src/exprs/aggregate/aggregate_function_window_funnel.cpp b/be/src/exprs/aggregate/aggregate_function_window_funnel.cpp
index 8249e3ee812..be6f457d417 100644
--- a/be/src/exprs/aggregate/aggregate_function_window_funnel.cpp
+++ b/be/src/exprs/aggregate/aggregate_function_window_funnel.cpp
@@ -40,6 +40,9 @@ AggregateFunctionPtr create_aggregate_function_window_funnel(const std::string&
if (argument_types[2]->get_primitive_type() == TYPE_DATETIMEV2) {
return creator_without_type::create<AggregateFunctionWindowFunnel<TYPE_DATETIMEV2>>(
argument_types, result_is_nullable, attr);
+ } else if (argument_types[2]->get_primitive_type() == TYPE_TIMESTAMP_NS) {
+ return creator_without_type::create<AggregateFunctionWindowFunnel<TYPE_TIMESTAMP_NS>>(
+ argument_types, result_is_nullable, attr);
} else if (argument_types[2]->get_primitive_type() == TYPE_TIMESTAMPTZ) {
return creator_without_type::create<AggregateFunctionWindowFunnel<TYPE_TIMESTAMPTZ>>(
argument_types, result_is_nullable, attr);
diff --git a/be/src/exprs/aggregate/aggregate_function_window_funnel.h b/be/src/exprs/aggregate/aggregate_function_window_funnel.h
index 778bc903ee9..fb4386dfb6b 100644
--- a/be/src/exprs/aggregate/aggregate_function_window_funnel.h
+++ b/be/src/exprs/aggregate/aggregate_function_window_funnel.h
@@ -157,6 +157,18 @@ struct WindowFunnelState {
}
}
+ bool _within_window(const DateValueType& base_timestamp, const DateValueType& current_timestamp,
+ const DateValueType& end_timestamp) const {
+ if constexpr (T == TYPE_TIMESTAMP_NS) {
+ const auto elapsed_nanos = static_cast<__int128>(current_timestamp.epoch_nanos()) -
+ base_timestamp.epoch_nanos();
+ const auto window_nanos =
+ static_cast<__int128>(window) * TimeStampNsValue::NANOS_PER_SECOND;
+ return elapsed_nanos <= window_nanos;
+ }
+ return current_timestamp <= end_timestamp;
+ }
+
template <WindowFunnelMode WINDOW_FUNNEL_MODE>
int _match_event_list(size_t& start_row, size_t row_count) const {
int matched_count = 0;
@@ -167,7 +179,6 @@ struct WindowFunnelState {
"the sliding time window must be a positive integer, but got: {}",
window);
}
- TimeInterval interval(SECOND, window, false);
int column_idx = 0;
const auto& timestamp_data = events_list.dt;
const auto& first_event_data = events_list.event_columns_data[column_idx].data();
@@ -175,8 +186,12 @@ struct WindowFunnelState {
start_row = match_row + 1;
if (match_row < row_count) {
auto prev_timestamp = timestamp_data[match_row];
- end_timestamp = prev_timestamp;
- end_timestamp.template date_add_interval<SECOND>(interval);
+ const auto first_timestamp = prev_timestamp;
+ if constexpr (T != TYPE_TIMESTAMP_NS) {
+ TimeInterval interval(SECOND, window, false);
+ end_timestamp = first_timestamp;
+ end_timestamp.template date_add_interval<SECOND>(interval);
+ }
matched_count++;
column_idx++;
@@ -187,7 +202,7 @@ struct WindowFunnelState {
if constexpr (WINDOW_FUNNEL_MODE == WindowFunnelMode::FIXED) {
if (event_data[match_row] == 1) {
auto current_timestamp = timestamp_data[match_row];
- if (current_timestamp <= end_timestamp) {
+ if (_within_window(first_timestamp, current_timestamp, end_timestamp)) {
matched_count++;
continue;
}
@@ -197,7 +212,8 @@ struct WindowFunnelState {
match_row = simd::find_one(event_data.data(), match_row, row_count);
if (match_row < row_count) {
auto current_timestamp = timestamp_data[match_row];
- bool is_matched = current_timestamp <= end_timestamp;
+ bool is_matched =
+ _within_window(first_timestamp, current_timestamp, end_timestamp);
if (is_matched) {
if constexpr (WINDOW_FUNNEL_MODE == WindowFunnelMode::INCREASE) {
is_matched = current_timestamp > prev_timestamp;
@@ -333,7 +349,11 @@ struct WindowFunnelState {
for (auto i = 0; i < size; i++) {
Int64 timestamp = 0;
read_var_int(timestamp, in);
- events_list.dt[i] = DateValueType(static_cast<UInt64>(timestamp));
+ if constexpr (T == TYPE_TIMESTAMP_NS) {
+ events_list.dt[i] = DateValueType(timestamp);
+ } else {
+ events_list.dt[i] = DateValueType(static_cast<UInt64>(timestamp));
+ }
}
events_list.event_columns_data.resize(event_count);
for (int64_t i = 0; i < event_count; i++) {
diff --git a/be/src/exprs/aggregate/aggregate_function_window_funnel_v2.cpp b/be/src/exprs/aggregate/aggregate_function_window_funnel_v2.cpp
index 0445602a0b1..fe23e357f01 100644
--- a/be/src/exprs/aggregate/aggregate_function_window_funnel_v2.cpp
+++ b/be/src/exprs/aggregate/aggregate_function_window_funnel_v2.cpp
@@ -42,6 +42,9 @@ AggregateFunctionPtr create_aggregate_function_window_funnel_v2(const std::strin
if (argument_types[2]->get_primitive_type() == TYPE_DATETIMEV2) {
return creator_without_type::create<AggregateFunctionWindowFunnelV2<TYPE_DATETIMEV2>>(
argument_types, result_is_nullable, attr);
+ } else if (argument_types[2]->get_primitive_type() == TYPE_TIMESTAMP_NS) {
+ return creator_without_type::create<AggregateFunctionWindowFunnelV2<TYPE_TIMESTAMP_NS>>(
+ argument_types, result_is_nullable, attr);
} else if (argument_types[2]->get_primitive_type() == TYPE_TIMESTAMPTZ) {
return creator_without_type::create<AggregateFunctionWindowFunnelV2<TYPE_TIMESTAMPTZ>>(
argument_types, result_is_nullable, attr);
diff --git a/be/src/exprs/aggregate/aggregate_function_window_funnel_v2.h b/be/src/exprs/aggregate/aggregate_function_window_funnel_v2.h
index 25557c039c9..782a83974ce 100644
--- a/be/src/exprs/aggregate/aggregate_function_window_funnel_v2.h
+++ b/be/src/exprs/aggregate/aggregate_function_window_funnel_v2.h
@@ -76,9 +76,11 @@ void merge_events_list(T& events_list, size_t prefix_size, bool prefix_sorted, b
/// row has bit 7 = 0, and subsequent events from the same row have bit 7 = 1.
/// The algorithm uses this to ensure each funnel step comes from a different row.
///
-/// This approach adds ZERO storage overhead — each event remains 9 bytes (UInt64 + UInt8).
+/// This approach adds ZERO storage overhead — each event remains 9 bytes (timestamp + UInt8).
template <PrimitiveType T>
struct WindowFunnelStateV2 {
+ using TimestampType = std::conditional_t<T == TYPE_TIMESTAMP_NS, Int64, UInt64>;
+
/// (timestamp_int_val, 1-based event_index with continuation flag in bit 7)
///
/// Bit layout of event_idx:
@@ -89,7 +91,7 @@ struct WindowFunnelStateV2 {
/// Sorted by timestamp only (via operator<). stable_sort preserves insertion order
/// for equal timestamps, so same-row events remain consecutive after sorting.
struct TimestampEvent {
- UInt64 timestamp;
+ TimestampType timestamp;
UInt8 event_idx; // includes continuation flag in bit 7
/// Sort by timestamp only. For same timestamp, stable_sort preserves insertion
@@ -242,33 +244,46 @@ struct WindowFunnelStateV2 {
}
}
- using DateValueType = DateV2Value<DateTimeV2ValueType>;
+ using DateValueType = std::conditional_t<T == TYPE_TIMESTAMP_NS, TimeStampNsValue,
+ DateV2Value<DateTimeV2ValueType>>;
- /// Reconstruct DateV2Value from packed UInt64.
- static DateValueType _ts_from_int(UInt64 packed) { return DateValueType(packed); }
+ static DateValueType _ts_from_int(TimestampType timestamp) { return DateValueType(timestamp); }
/// Check if `current_ts` is within `window` seconds of `base_ts`.
- /// Both are packed UInt64 from DateV2Value::to_date_int_val().
- bool _within_window(UInt64 base_ts, UInt64 current_ts) const {
- DateValueType end_ts = _ts_from_int(base_ts);
- TimeInterval interval(SECOND, window, false);
- end_ts.template date_add_interval<SECOND>(interval);
- return current_ts <= end_ts.to_date_int_val();
+ bool _within_window(TimestampType base_ts, TimestampType current_ts) const {
+ if constexpr (T == TYPE_TIMESTAMP_NS) {
+ const auto elapsed_nanos =
+ static_cast<__int128>(current_ts) - static_cast<__int128>(base_ts);
+ const auto window_nanos =
+ static_cast<__int128>(window) * TimeStampNsValue::NANOS_PER_SECOND;
+ return elapsed_nanos <= window_nanos;
+ } else {
+ DateValueType end_ts = _ts_from_int(base_ts);
+ TimeInterval interval(SECOND, window, false);
+ end_ts.template date_add_interval<SECOND>(interval);
+ return current_ts <= end_ts.to_date_int_val();
+ }
}
/// Track (first_timestamp, last_timestamp, last_event_list_idx) for each event level.
- /// Uses packed UInt64 values; 0 means unset for first_ts.
/// last_list_idx tracks the position in events_list of the event that set this level,
/// used to check continuation flag on subsequent events to detect same-row advancement.
struct TimestampPair {
- UInt64 first_ts = 0;
- UInt64 last_ts = 0;
+ TimestampType first_ts = 0;
+ TimestampType last_ts = 0;
size_t last_list_idx = 0;
- bool has_value() const { return first_ts != 0; }
+ bool initialized = false;
+ bool has_value() const {
+ if constexpr (T == TYPE_TIMESTAMP_NS) {
+ return initialized;
+ }
+ return first_ts != 0;
+ }
void reset() {
first_ts = 0;
last_ts = 0;
last_list_idx = 0;
+ initialized = false;
}
};
@@ -315,13 +330,13 @@ private:
int event_idx = get_event_idx(evt.event_idx) - 1;
if (event_idx == 0) {
- events_timestamp[0] = {evt.timestamp, evt.timestamp, i};
+ events_timestamp[0] = {evt.timestamp, evt.timestamp, i, true};
} else if (events_timestamp[event_idx - 1].has_value() &&
!_is_same_row(events_timestamp[event_idx - 1].last_list_idx, i)) {
// Must be from a DIFFERENT row than the predecessor level
if (_within_window(events_timestamp[event_idx - 1].first_ts, evt.timestamp)) {
events_timestamp[event_idx] = {events_timestamp[event_idx - 1].first_ts,
- evt.timestamp, i};
+ evt.timestamp, i, true};
if (event_idx + 1 == event_count) {
return event_count;
}
@@ -374,7 +389,7 @@ private:
// Try building a chain from this event-0
std::vector<TimestampPair> events_timestamp(event_count);
events_timestamp[0] = {events_list[start].timestamp, events_list[start].timestamp,
- start};
+ start, true};
int curr_level = 0;
for (size_t i = start + 1; i < list_size; ++i) {
@@ -393,7 +408,7 @@ private:
matched = matched && events_timestamp[event_idx - 1].last_ts < evt.timestamp;
if (matched) {
events_timestamp[event_idx] = {events_timestamp[event_idx - 1].first_ts,
- evt.timestamp, i};
+ evt.timestamp, i, true};
curr_level = std::max(event_idx, curr_level);
if (event_idx + 1 == event_count) {
return event_count;
@@ -435,7 +450,7 @@ private:
}
// Build a chain from this event-0
- UInt64 first_ts = events_list[start].timestamp;
+ TimestampType first_ts = events_list[start].timestamp;
int curr_level = 0;
size_t last_advance_idx = start;
@@ -517,7 +532,7 @@ private:
// Only start from c1 events (condition 0 in 0-based)
if (get_event_idx(events_list[start_i].event_idx) != 1) continue;
- UInt64 chain_start_ts = events_list[start_i].timestamp;
+ TimestampType chain_start_ts = events_list[start_i].timestamp;
int curr_level = 0;
// c1 is always the last event of its row, so start_i+1 is
diff --git a/be/src/exprs/create_predicate_function.h b/be/src/exprs/create_predicate_function.h
index f79ace9ead9..d84aa1abf13 100644
--- a/be/src/exprs/create_predicate_function.h
+++ b/be/src/exprs/create_predicate_function.h
@@ -89,6 +89,7 @@ public:
M(TYPE_DATETIME) \
M(TYPE_DATEV2) \
M(TYPE_DATETIMEV2) \
+ M(TYPE_TIMESTAMP_NS) \
M(TYPE_TIMESTAMPTZ) \
M(TYPE_TIMEV2) \
M(TYPE_CHAR) \
diff --git a/be/src/exprs/function/array/function_array_aggregation.cpp b/be/src/exprs/function/array/function_array_aggregation.cpp
index d2c6e65452f..600f83a2f65 100644
--- a/be/src/exprs/function/array/function_array_aggregation.cpp
+++ b/be/src/exprs/function/array/function_array_aggregation.cpp
@@ -190,6 +190,7 @@ struct ArrayAggregateImpl {
execute_type<TYPE_DECIMAL256>(res, type, data, offsets) ||
execute_type<TYPE_DATEV2>(res, type, data, offsets) ||
execute_type<TYPE_DATETIMEV2>(res, type, data, offsets) ||
+ execute_type<TYPE_TIMESTAMP_NS>(res, type, data, offsets) ||
execute_type<TYPE_TIMESTAMPTZ>(res, type, data, offsets) ||
execute_type<TYPE_VARCHAR>(res, type, data, offsets)) {
block.replace_by_position(result, std::move(res));
@@ -260,8 +261,8 @@ struct ArrayAggregateImpl {
if constexpr ((operation == AggregateOperation::SUM ||
operation == AggregateOperation::PRODUCT ||
operation == AggregateOperation::AVERAGE) &&
- (is_date_type(Element) || is_timestamptz_type(Element) ||
- is_decimalv3(Element))) {
+ (is_date_type(Element) || is_timestamp_ns_type(Element) ||
+ is_timestamptz_type(Element) || is_decimalv3(Element))) {
return false;
} else {
using ColVecType = typename PrimitiveTypeTraits<Element>::ColumnType;
diff --git a/be/src/exprs/function/array/function_array_index.h b/be/src/exprs/function/array/function_array_index.h
index 3f39fc627b4..b883028340e 100644
--- a/be/src/exprs/function/array/function_array_index.h
+++ b/be/src/exprs/function/array/function_array_index.h
@@ -426,7 +426,11 @@ private:
}
return false;
};
- dispatch_switch_scalar(right_type->get_primitive_type(), call);
+ if (right_type->get_primitive_type() == TYPE_TIMESTAMP_NS) {
+ call(DispatchDataType<TYPE_TIMESTAMP_NS>());
+ } else {
+ dispatch_switch_scalar(right_type->get_primitive_type(), call);
+ }
}
if (return_column) {
diff --git a/be/src/exprs/function/array/function_array_range.cpp b/be/src/exprs/function/array/function_array_range.cpp
index 358d4ac9370..76e75e1c6da 100644
--- a/be/src/exprs/function/array/function_array_range.cpp
+++ b/be/src/exprs/function/array/function_array_range.cpp
@@ -98,8 +98,9 @@ struct RangeImplUtil {
static DataTypePtr get_data_type() { return std::make_shared<DataType>(); }
static constexpr const char* get_function_name() {
- if constexpr (std::is_same_v<SourceDataType, DateV2Value<DateTimeV2ValueType>> &&
- !std::is_same_v<TimeUnitOrVoid, void>) {
+ if constexpr ((std::is_same_v<SourceDataType, DateV2Value<DateTimeV2ValueType>> ||
+ std::is_same_v<SourceDataType,
+ TimeStampNsValue>)&&!std::is_same_v<TimeUnitOrVoid, void>) {
if constexpr (std::is_same_v<TimeUnitOrVoid,
std::integral_constant<TimeUnit, TimeUnit::YEAR>>) {
return "array_range_year_unit";
@@ -228,11 +229,8 @@ private:
dest_offsets.push_back(new_offset);
}
} else {
- const auto& idx_0 = reinterpret_cast<const DateV2Value<DateTimeV2ValueType>&>(idx);
- const auto& end_row_cast =
- reinterpret_cast<const DateV2Value<DateTimeV2ValueType>&>(end_row);
- bool is_null = !idx_0.is_valid_date();
- bool is_end_row_invalid = !end_row_cast.is_valid_date();
+ bool is_null = !idx.is_valid_date();
+ bool is_end_row_invalid = !end_row.is_valid_date();
if (args_null_map_row || step_row <= 0 || is_null || is_end_row_invalid) {
args_null_map[row] = 1;
dest_offsets.push_back(dest_offsets.back());
@@ -255,8 +253,9 @@ private:
move++;
auto next = idx;
bool advanced;
- if constexpr (UNIT::value == TimeUnit::DAY ||
- UNIT::value == TimeUnit::WEEK) {
+ if constexpr (SourceDataPType != TYPE_TIMESTAMP_NS &&
+ (UNIT::value == TimeUnit::DAY ||
+ UNIT::value == TimeUnit::WEEK)) {
const Int64 days = static_cast<Int64>(step_row) *
(UNIT::value == TimeUnit::WEEK ? 7 : 1);
advanced = next.template date_add_days<false>(days);
@@ -346,6 +345,7 @@ void register_function_array_range(SimpleFunctionFactory& factory) {
/// Two arguments, for Int32 and DateTimeV2 without Interval
factory.register_function<FunctionArrayRange<RangeTwoImpl<TYPE_INT>>>();
factory.register_function<FunctionArrayRange<RangeTwoImpl<TYPE_DATETIMEV2>>>();
+ factory.register_function<FunctionArrayRange<RangeTwoImpl<TYPE_TIMESTAMP_NS>>>();
/// Three arguments, for Int32 and DateTimeV2 with YEAR to SECOND Interval
factory.register_function<FunctionArrayRange<RangeThreeImpl<TYPE_INT>>>();
@@ -365,6 +365,22 @@ void register_function_array_range(SimpleFunctionFactory& factory) {
RangeThreeImpl<TYPE_DATETIMEV2, std::integral_constant<TimeUnit, TimeUnit::MINUTE>>>>();
factory.register_function<FunctionArrayRange<
RangeThreeImpl<TYPE_DATETIMEV2, std::integral_constant<TimeUnit, TimeUnit::SECOND>>>>();
+ factory.register_function<FunctionArrayRange<
+ RangeThreeImpl<TYPE_TIMESTAMP_NS, std::integral_constant<TimeUnit, TimeUnit::YEAR>>>>();
+ factory.register_function<FunctionArrayRange<RangeThreeImpl<
+ TYPE_TIMESTAMP_NS, std::integral_constant<TimeUnit, TimeUnit::QUARTER>>>>();
+ factory.register_function<FunctionArrayRange<RangeThreeImpl<
+ TYPE_TIMESTAMP_NS, std::integral_constant<TimeUnit, TimeUnit::MONTH>>>>();
+ factory.register_function<FunctionArrayRange<
+ RangeThreeImpl<TYPE_TIMESTAMP_NS, std::integral_constant<TimeUnit, TimeUnit::WEEK>>>>();
+ factory.register_function<FunctionArrayRange<
+ RangeThreeImpl<TYPE_TIMESTAMP_NS, std::integral_constant<TimeUnit, TimeUnit::DAY>>>>();
+ factory.register_function<FunctionArrayRange<
+ RangeThreeImpl<TYPE_TIMESTAMP_NS, std::integral_constant<TimeUnit, TimeUnit::HOUR>>>>();
+ factory.register_function<FunctionArrayRange<RangeThreeImpl<
+ TYPE_TIMESTAMP_NS, std::integral_constant<TimeUnit, TimeUnit::MINUTE>>>>();
+ factory.register_function<FunctionArrayRange<RangeThreeImpl<
+ TYPE_TIMESTAMP_NS, std::integral_constant<TimeUnit, TimeUnit::SECOND>>>>();
// alias
factory.register_alias("array_range", "sequence");
diff --git a/be/src/exprs/function/array/function_array_remove.h b/be/src/exprs/function/array/function_array_remove.h
index 09a43e7651c..c7e5493389f 100644
--- a/be/src/exprs/function/array/function_array_remove.h
+++ b/be/src/exprs/function/array/function_array_remove.h
@@ -403,6 +403,11 @@ private:
res = _execute_number_expanded<ColumnTimeStampTz>(
offsets, *nested_column, *right_column, nested_null_map, right_nested_null_map,
array_null_map);
+ } else if (is_timestamp_ns_type(right_type->get_primitive_type()) &&
+ is_timestamp_ns_type(left_element_type->get_primitive_type())) {
+ res = _execute_number_expanded<ColumnTimeStampNs>(
+ offsets, *nested_column, *right_column, nested_null_map, right_nested_null_map,
+ array_null_map);
} else if (is_ip(right_type->get_primitive_type()) &&
is_ip(left_element_type->get_primitive_type())) {
if (left_element_type->get_primitive_type() == TYPE_IPV4) {
diff --git a/be/src/exprs/function/cast/cast_to_basic_number_common.h b/be/src/exprs/function/cast/cast_to_basic_number_common.h
index e79fbf06407..a33bf41c0e5 100644
--- a/be/src/exprs/function/cast/cast_to_basic_number_common.h
+++ b/be/src/exprs/function/cast/cast_to_basic_number_common.h
@@ -25,6 +25,7 @@
#include "core/data_type/data_type_decimal.h"
#include "core/data_type/data_type_number.h"
#include "core/data_type/data_type_string.h"
+#include "core/data_type/data_type_timestamp_ns.h"
#include "core/data_type/define_primitive_type.h"
#include "core/data_type/primitive_type.h"
#include "core/types.h"
@@ -394,6 +395,8 @@ Status static_cast_no_overflow(FunctionContext* context, Block& block,
} else if constexpr (IsDatelikeV1Types<FromDataType>) {
CastToInt::from_datetime(reinterpret_cast<const VecDateTimeValue&>(vec_from[i]),
vec_to[i], params);
+ } else if constexpr (std::is_same_v<FromDataType, DataTypeTimeStampNs>) {
+ CastToInt::from_datetime(vec_from[i].to_datetime(), vec_to[i], params);
} else if constexpr (IsDateTimeV2Type<FromDataType>) {
CastToInt::from_datetime(
reinterpret_cast<const DateV2Value<DateTimeV2ValueType>&>(vec_from[i]),
@@ -419,6 +422,8 @@ Status static_cast_no_overflow(FunctionContext* context, Block& block,
} else if constexpr (IsDatelikeV1Types<FromDataType>) {
CastToFloat::from_datetime(reinterpret_cast<const VecDateTimeValue&>(vec_from[i]),
vec_to[i], params);
+ } else if constexpr (std::is_same_v<FromDataType, DataTypeTimeStampNs>) {
+ CastToFloat::from_datetime(vec_from[i].to_datetime(), vec_to[i], params);
} else if constexpr (IsDateTimeV2Type<FromDataType>) {
CastToFloat::from_datetime(
reinterpret_cast<const DateV2Value<DateTimeV2ValueType>&>(vec_from[i]),
@@ -448,7 +453,8 @@ Status static_cast_no_overflow(FunctionContext* context, Block& block,
template <typename T>
constexpr static bool type_allow_cast_to_basic_number =
std::is_same_v<T, DataTypeString> || IsDataTypeNumber<T> || IsDataTypeDecimal<T> ||
- IsDatelikeV1Types<T> || IsDatelikeV2Types<T> || std::is_same_v<T, DataTypeTimeV2>;
+ IsDatelikeV1Types<T> || IsDatelikeV2Types<T> || std::is_same_v<T, DataTypeTimeStampNs> ||
+ std::is_same_v<T, DataTypeTimeV2>;
// common implementation for casting string to basic number types,
// including integer, float and double
diff --git a/be/src/exprs/function/cast/cast_to_date.h b/be/src/exprs/function/cast/cast_to_date.h
index 630a47bf8fa..fbc9c9906a2 100644
--- a/be/src/exprs/function/cast/cast_to_date.h
+++ b/be/src/exprs/function/cast/cast_to_date.h
@@ -19,6 +19,7 @@
#include <sys/types.h>
+#include <algorithm>
#include <cstdint>
#include <cstdlib>
#include <type_traits>
@@ -32,6 +33,7 @@
#include "core/data_type/data_type_number.h"
#include "core/data_type/data_type_string.h"
#include "core/data_type/data_type_time.h"
+#include "core/data_type/data_type_timestamp_ns.h"
#include "core/data_type/primitive_type.h"
#include "core/data_type_serde/data_type_serde.h"
#include "core/types.h"
@@ -130,6 +132,73 @@ public:
}
};
+template <CastModeType CastMode, typename FromDataType, typename ToDataType>
+ requires(std::is_same_v<FromDataType, DataTypeTimeStampNs> && IsDatelikeTypes<ToDataType>)
+class CastToImpl<CastMode, FromDataType, ToDataType> : public CastToBase {
+public:
+ Status execute_impl(FunctionContext* /*context*/, Block& block, const ColumnNumbers& arguments,
+ uint32_t result, size_t input_rows_count,
+ const NullMap::value_type* null_map = nullptr) const override {
+ const auto& col_from =
+ assert_cast<const ColumnTimeStampNs&>(*block.get_by_position(arguments[0]).column);
+ auto col_to = ToDataType::ColumnType::create(input_rows_count);
+
+ for (size_t i = 0; i < input_rows_count; ++i) {
+ if (null_map && null_map[i]) {
+ continue;
+ }
+
+ const auto& source = col_from.get_data()[i];
+ if constexpr (IsDateType<ToDataType>) {
+ const auto civil = source.to_datetime();
+ DataTypeDateTimeV2::cast_to_date(civil, col_to->get_data()[i]);
+ } else if constexpr (IsDateV2Type<ToDataType>) {
+ col_to->get_data()[i] = source.to_date();
+ } else if constexpr (IsDateTimeType<ToDataType>) {
+ const auto civil = source.to_datetime();
+ DataTypeDateTimeV2::cast_to_date_time(civil, col_to->get_data()[i]);
+ } else if constexpr (IsDateTimeV2Type<ToDataType>) {
+ const auto to_scale = block.get_by_position(result).type->get_scale();
+ const bool success =
+ transform_date_scale(to_scale, TimeStampNsValue::FRACTIONAL_DIGITS,
+ col_to->get_data()[i], source);
+ DORIS_CHECK(success);
+ } else {
+ static_assert(IsTimeV2Type<ToDataType>);
+ const auto to_scale = block.get_by_position(result).type->get_scale();
+ const int64_t seconds = source.time_part_to_seconds();
+ uint32_t hour = static_cast<uint32_t>(seconds / 3600);
+ uint32_t minute = static_cast<uint32_t>(seconds / 60 % 60);
+ uint32_t second = static_cast<uint32_t>(seconds % 60);
+ uint32_t nanoseconds = source.nanosecond();
+ const auto divisor = static_cast<uint32_t>(common::exp10_i64(9 - to_scale));
+ const uint32_t remainder = nanoseconds % divisor;
+ nanoseconds = nanoseconds / divisor * divisor;
+ if (remainder >= divisor / 2) {
+ nanoseconds += divisor;
+ }
+ uint32_t microseconds = nanoseconds / TimeStampNsValue::NANOS_PER_MICROSECOND;
+ if (microseconds >= TimeValue::ONE_SECOND_MICROSECONDS) {
+ microseconds = 0;
+ if (++second == 60) {
+ second = 0;
+ if (++minute == 60) {
+ minute = 0;
+ ++hour;
+ }
+ }
+ }
+ col_to->get_data()[i] =
+ ((hour * 60 + minute) * 60 + second) * TimeValue::ONE_SECOND_MICROSECONDS +
+ microseconds;
+ }
+ }
+
+ block.get_by_position(result).column = std::move(col_to);
+ return Status::OK();
+ }
+};
+
template <CastModeType CastMode, typename FromDataType, typename ToDataType>
requires(IsDatelikeTypes<FromDataType> && IsDatelikeTypes<ToDataType>)
class CastToImpl<CastMode, FromDataType, ToDataType> : public CastToBase {
diff --git a/be/src/exprs/function/cast/cast_to_datetimev2_impl.hpp b/be/src/exprs/function/cast/cast_to_datetimev2_impl.hpp
index 8ce2a45d36c..2885e64b0c9 100644
--- a/be/src/exprs/function/cast/cast_to_datetimev2_impl.hpp
+++ b/be/src/exprs/function/cast/cast_to_datetimev2_impl.hpp
@@ -27,6 +27,7 @@
#include "core/data_type_serde/data_type_serde.h"
#include "core/data_type_serde/datelike_serde_common.hpp"
#include "core/types.h"
+#include "core/value/timestamp_ns_value.h"
#include "core/value/vdatetime_value.h"
#include "exprs/function/cast/cast_base.h" // IWYU pragma: keep
#include "util/asan_util.h"
@@ -192,17 +193,16 @@ struct CastToDatetimeV2 {
// this code follow rules of strict mode, but whether it RUNNING IN strict mode or not depends on the `IsStrict`
// parameter. if it's false, we dont set error code for performance and we dont need.
- template <DatelikeParseMode ParseMode>
- static inline bool from_string_strict_mode(const StringRef& str,
- DateV2Value<DateTimeV2ValueType>& res,
+ template <DatelikeParseMode ParseMode, typename ResultType>
+ static inline bool from_string_strict_mode(const StringRef& str, ResultType& res,
const cctz::time_zone* local_time_zone,
uint32_t to_scale, CastParameters& params) {
return from_string_strict_mode_internal<ParseMode, DataTimeCastEnumType::DATE_TIME>(
str, res, local_time_zone, to_scale, params);
}
- static inline bool from_string_non_strict_mode(const StringRef& str,
- DateV2Value<DateTimeV2ValueType>& res,
+ template <typename ResultType>
+ static inline bool from_string_non_strict_mode(const StringRef& str, ResultType& res,
const cctz::time_zone* local_time_zone,
uint32_t to_scale, CastParameters& params) {
return CastToDatetimeV2::from_string_strict_mode<DatelikeParseMode::NON_STRICT>(
@@ -213,15 +213,28 @@ struct CastToDatetimeV2 {
}
private:
- template <DatelikeParseMode ParseMode, DataTimeCastEnumType type>
- static inline bool from_string_strict_mode_internal(const StringRef& str,
- DateV2Value<DateTimeV2ValueType>& res,
+ template <typename ResultType>
+ static constexpr bool is_timestamp_ns_result =
+ std::is_same_v<std::remove_cvref_t<ResultType>, TimeStampNsValue>;
+
+ template <DatelikeParseMode ParseMode, typename ResultType>
+ static inline bool parse_fractional_seconds(const char* start, size_t length,
+ DateV2Value<DateTimeV2ValueType>& datetime,
+ uint16_t& nanosecond_remainder, uint32_t to_scale,
+ CastParameters& params);
+
+ template <DatelikeParseMode ParseMode, typename ResultType>
+ static inline bool finalize_string_result(DateV2Value<DateTimeV2ValueType>& datetime,
+ uint16_t nanosecond_remainder, ResultType& result,
+ CastParameters& params);
+
+ template <DatelikeParseMode ParseMode, DataTimeCastEnumType type, typename ResultType>
+ static inline bool from_string_strict_mode_internal(const StringRef& str, ResultType& res,
const cctz::time_zone* local_time_zone,
uint32_t to_scale, CastParameters& params);
- template <DataTimeCastEnumType type>
- static inline bool from_string_non_strict_mode_internal(const StringRef& str,
- DateV2Value<DateTimeV2ValueType>& res,
+ template <DataTimeCastEnumType type, typename ResultType>
+ static inline bool from_string_non_strict_mode_internal(const StringRef& str, ResultType& res,
const cctz::time_zone* local_time_zone,
uint32_t to_scale,
CastParameters& params);
@@ -297,6 +310,62 @@ inline bool CastToDatetimeV2::from_integer(T input, DateV2Value<DateTimeV2ValueT
return true;
}
+template <DatelikeParseMode ParseMode, typename ResultType>
+inline bool CastToDatetimeV2::parse_fractional_seconds(const char* start, size_t length,
+ DateV2Value<DateTimeV2ValueType>& datetime,
+ uint16_t& nanosecond_remainder,
+ uint32_t to_scale, CastParameters& params) {
+ constexpr bool IsStrict = is_datelike_parse_strict(ParseMode);
+ constexpr uint32_t ResultScale =
+ is_timestamp_ns_result<ResultType> ? TimeStampNsValue::FRACTIONAL_DIGITS : 6;
+ DCHECK_LE(to_scale, ResultScale);
+
+ StringParser::ParseResult success;
+ const auto parsed_scale =
+ static_cast<uint32_t>(std::min<size_t>(length, static_cast<size_t>(to_scale)));
+ auto fraction = StringParser::string_to_uint_greedy_no_overflow<uint32_t>(start, parsed_scale,
+ &success);
+ SET_PARAMS_RET_FALSE_IFN(success == StringParser::PARSE_SUCCESS,
+ "invalid fractional part in datetime string '{}'",
+ std::string {start, start + length});
+
+ if (length > to_scale) {
+ if (*(start + to_scale) - '0' >= 5) {
+ ++fraction;
+ DCHECK_LE(fraction, common::exp10_i32(to_scale));
+ if (fraction == common::exp10_i32(to_scale)) {
+ SET_PARAMS_RET_FALSE_IFN(datetime.date_add_interval<TimeUnit::SECOND>(
+ TimeInterval {TimeUnit::SECOND, 1, false}),
+ "datetime overflow when rounding up to next second");
+ fraction = 0;
+ }
+ }
+ }
+
+ fraction *= common::exp10_i32(ResultScale - parsed_scale);
+ if constexpr (is_timestamp_ns_result<ResultType>) {
+ datetime.unchecked_set_time_unit<TimeUnit::MICROSECOND>(
+ fraction / TimeStampNsValue::NANOS_PER_MICROSECOND);
+ nanosecond_remainder =
+ static_cast<uint16_t>(fraction % TimeStampNsValue::NANOS_PER_MICROSECOND);
+ } else {
+ datetime.unchecked_set_time_unit<TimeUnit::MICROSECOND>(fraction);
+ }
+ return true;
+}
+
+template <DatelikeParseMode ParseMode, typename ResultType>
+inline bool CastToDatetimeV2::finalize_string_result(DateV2Value<DateTimeV2ValueType>& datetime,
+ uint16_t nanosecond_remainder,
+ ResultType& result, CastParameters& params) {
+ constexpr bool IsStrict = is_datelike_parse_strict(ParseMode);
+ if constexpr (is_timestamp_ns_result<ResultType>) {
+ SET_PARAMS_RET_FALSE_IFN(result.from_datetime(datetime, nanosecond_remainder),
+ "timestamp_ns value is outside the signed epoch-nanosecond range");
+ }
+ return true;
+}
+
/**
<datetime> ::= <date> (("T" | " ") <time> <whitespace>* <offset>?)?
@@ -349,11 +418,21 @@ inline bool CastToDatetimeV2::from_integer(T input, DateV2Value<DateTimeV2ValueT
<alpha> ::= "A" | … | "Z" | "a" | … | "z"
<whitespace> ::= " " | "\t" | "\n" | "\r" | "\v" | "\f"
*/
-template <DatelikeParseMode ParseMode, DataTimeCastEnumType type>
+template <DatelikeParseMode ParseMode, DataTimeCastEnumType type, typename ResultType>
inline bool CastToDatetimeV2::from_string_strict_mode_internal(
- const StringRef& str, DateV2Value<DateTimeV2ValueType>& res,
- const cctz::time_zone* local_time_zone, uint32_t to_scale, CastParameters& params) {
+ const StringRef& str, ResultType& result, const cctz::time_zone* local_time_zone,
+ uint32_t to_scale, CastParameters& params) {
constexpr bool IsStrict = is_datelike_parse_strict(ParseMode);
+ static_assert(
+ std::is_same_v<std::remove_cvref_t<ResultType>, DateV2Value<DateTimeV2ValueType>> ||
+ is_timestamp_ns_result<ResultType>);
+ DateV2Value<DateTimeV2ValueType> timestamp_ns_datetime;
+ DateV2Value<DateTimeV2ValueType>* datetime = ×tamp_ns_datetime;
+ if constexpr (!is_timestamp_ns_result<ResultType>) {
+ datetime = &result;
+ }
+ auto& res = *datetime;
+ uint16_t nanosecond_remainder = 0;
const char* ptr = str.data;
const char* end = ptr + str.size;
AsanPoisonGuard defer(end, 1);
@@ -597,33 +676,8 @@ FRAC:
auto length = ptr - start;
if (length > 0) {
- StringParser::ParseResult success;
- auto frac_literal = StringParser::string_to_uint_greedy_no_overflow<uint32_t>(
- start, std::min<int>((int)length, to_scale), &success);
- SET_PARAMS_RET_FALSE_IFN(success == StringParser::PARSE_SUCCESS,
- "invalid fractional part in datetime string '{}'",
- std::string {start, ptr});
-
- if (length > to_scale) { // to_scale is up to 6
- // round off to at most `to_scale` digits
- if (*(start + to_scale) - '0' >= 5) {
- frac_literal++;
- DCHECK(frac_literal <= 1000000);
- if (frac_literal == common::exp10_i32(to_scale)) {
- // overflow, round up to next second
- SET_PARAMS_RET_FALSE_IFN(
- res.date_add_interval<TimeUnit::SECOND>(
- TimeInterval {TimeUnit::SECOND, 1, false}),
- "datetime overflow when rounding up to next second");
- frac_literal = 0;
- }
- }
- res.unchecked_set_time_unit<TimeUnit::MICROSECOND>(
- (int32_t)frac_literal * common::exp10_i32(6 - (int)to_scale));
- } else { // length <= to_scale
- res.unchecked_set_time_unit<TimeUnit::MICROSECOND>(
- (int32_t)frac_literal * common::exp10_i32(6 - (int)length));
- }
+ PROPAGATE_FALSE((parse_fractional_seconds<ParseMode, ResultType>(
+ start, length, res, nanosecond_remainder, to_scale, params)));
}
} else {
res.unchecked_set_time_unit<TimeUnit::MICROSECOND>(0);
@@ -738,7 +792,7 @@ FRAC:
"invalid datetime string '{}', extra characters after timezone",
std::string {ptr, end});
- return true;
+ return finalize_string_result<ParseMode>(res, nanosecond_remainder, result, params);
}
POST_PROCESS:
@@ -760,7 +814,7 @@ POST_PROCESS:
SET_PARAMS_RET_FALSE_IFN(res.year() <= 9999, "datetime year {} out of range [0, 9999]",
res.year());
}
- return true;
+ return finalize_string_result<ParseMode>(res, nanosecond_remainder, result, params);
}
/**
@@ -806,11 +860,21 @@ POST_PROCESS:
<alpha> ::= "A" | … | "Z" | "a" | … | "z"
*/
-template <DataTimeCastEnumType type>
+template <DataTimeCastEnumType type, typename ResultType>
inline bool CastToDatetimeV2::from_string_non_strict_mode_internal(
- const StringRef& str, DateV2Value<DateTimeV2ValueType>& res,
- const cctz::time_zone* local_time_zone, uint32_t to_scale, CastParameters& params) {
+ const StringRef& str, ResultType& result, const cctz::time_zone* local_time_zone,
+ uint32_t to_scale, CastParameters& params) {
constexpr bool IsStrict = false;
+ static_assert(
+ std::is_same_v<std::remove_cvref_t<ResultType>, DateV2Value<DateTimeV2ValueType>> ||
+ is_timestamp_ns_result<ResultType>);
+ DateV2Value<DateTimeV2ValueType> timestamp_ns_datetime;
+ DateV2Value<DateTimeV2ValueType>* datetime = ×tamp_ns_datetime;
+ if constexpr (!is_timestamp_ns_result<ResultType>) {
+ datetime = &result;
+ }
+ auto& res = *datetime;
+ uint16_t nanosecond_remainder = 0;
const char* ptr = str.data;
const char* end = ptr + str.size;
AsanPoisonGuard defer(end, 1);
@@ -899,33 +963,8 @@ inline bool CastToDatetimeV2::from_string_non_strict_mode_internal(
auto length = ptr - start;
if (length > 0) {
- StringParser::ParseResult success;
- auto frac_literal = StringParser::string_to_uint_greedy_no_overflow<uint32_t>(
- start, std::min<int>((int)length, to_scale), &success);
- SET_PARAMS_RET_FALSE_IFN(success == StringParser::PARSE_SUCCESS,
- "invalid fractional part in datetime string '{}'",
- std::string {start, ptr});
-
- if (length > to_scale) { // to_scale is up to 6
- // round off to at most `to_scale` digits
- if (*(start + to_scale) - '0' >= 5) {
- frac_literal++;
- DCHECK(frac_literal <= 1000000);
- if (frac_literal == common::exp10_i32(to_scale)) {
- // overflow, round up to next second
- SET_PARAMS_RET_FALSE_IFN(
- res.date_add_interval<TimeUnit::SECOND>(
- TimeInterval {TimeUnit::SECOND, 1, false}),
- "datetime overflow when rounding up to next second");
- frac_literal = 0;
- }
- }
- res.unchecked_set_time_unit<TimeUnit::MICROSECOND>(
- (int32_t)frac_literal * common::exp10_i32(6 - (int)to_scale));
- } else { // length <= to_scale
- res.unchecked_set_time_unit<TimeUnit::MICROSECOND>(
- (int32_t)frac_literal * common::exp10_i32(6 - (int)length));
- }
+ PROPAGATE_FALSE((parse_fractional_seconds<DatelikeParseMode::NON_STRICT, ResultType>(
+ start, length, res, nanosecond_remainder, to_scale, params)));
} else {
res.unchecked_set_time_unit<TimeUnit::MICROSECOND>(0);
}
@@ -1038,7 +1077,8 @@ inline bool CastToDatetimeV2::from_string_non_strict_mode_internal(
"invalid datetime string '{}', extra characters after parsing",
std::string {ptr, end});
- return true;
+ return finalize_string_result<DatelikeParseMode::NON_STRICT>(res, nanosecond_remainder,
+ result, params);
}
// skip trailing whitespace
@@ -1067,7 +1107,8 @@ POST_PROCESS:
res.year());
}
- return true;
+ return finalize_string_result<DatelikeParseMode::NON_STRICT>(res, nanosecond_remainder, result,
+ params);
}
// NOLINTEND(readability-function-cognitive-complexity)
@@ -1116,6 +1157,42 @@ inline bool transform_date_scale(UInt32 to_scale, UInt32 from_scale,
return true;
}
+// Round the nine-digit fraction to a DATETIMEV2/TIMEV2 target scale. The rounded civil value is
+// allowed to lie outside the TIMESTAMP_NS range because the destination type has the wider
+// year-0000..9999 domain. This matters at the lower boundary, whose microsecond rounding is 192ns
+// below the TIMESTAMP_NS minimum but is still a valid DATETIMEV2 value.
+inline bool round_timestamp_ns_fraction(UInt32 to_scale, DateV2Value<DateTimeV2ValueType>& datetime,
+ uint32_t& nanoseconds) {
+ DCHECK_LE(to_scale, 6);
+ const auto divisor = static_cast<uint32_t>(common::exp10_i64(9 - to_scale));
+ const uint32_t remainder = nanoseconds % divisor;
+ nanoseconds = nanoseconds / divisor * divisor;
+ if (remainder >= divisor / 2) {
+ nanoseconds += divisor;
+ }
+ if (nanoseconds >= TimeStampNsValue::NANOS_PER_SECOND) {
+ nanoseconds = 0;
+ if (!datetime.template date_add_interval<TimeUnit::SECOND>(
+ TimeInterval {TimeUnit::SECOND, 1, false})) {
+ return false;
+ }
+ }
+ datetime.set_microsecond(nanoseconds / TimeStampNsValue::NANOS_PER_MICROSECOND);
+ return true;
+}
+
+inline bool transform_date_scale(UInt32 to_scale, UInt32 /*from_scale*/,
+ DateV2Value<DateTimeV2ValueType>& to_value,
+ const TimeStampNsValue& from_value) {
+ auto datetime = from_value.to_datetime();
+ uint32_t nanoseconds = from_value.nanosecond();
+ if (!round_timestamp_ns_fraction(to_scale, datetime, nanoseconds)) {
+ return false;
+ }
+ to_value = datetime;
+ return true;
+}
+
inline bool transform_date_scale(UInt32 to_scale, UInt32 from_scale,
PrimitiveTypeTraits<TYPE_TIMESTAMPTZ>::CppType& to_value,
const PrimitiveTypeTraits<TYPE_TIMESTAMPTZ>::CppType& from_value) {
diff --git a/be/src/exprs/function/cast/cast_to_float.h b/be/src/exprs/function/cast/cast_to_float.h
index 83967346387..f28e3f70332 100644
--- a/be/src/exprs/function/cast/cast_to_float.h
+++ b/be/src/exprs/function/cast/cast_to_float.h
@@ -81,6 +81,7 @@ public:
template <typename FromDataType, typename ToDataType>
requires(IsDataTypeFloat<ToDataType> &&
(IsDatelikeV1Types<FromDataType> || IsDatelikeV2Types<FromDataType> ||
+ std::is_same_v<FromDataType, DataTypeTimeStampNs> ||
std::is_same_v<FromDataType, DataTypeTimeV2>))
class CastToImpl<CastModeType::NonStrictMode, FromDataType, ToDataType> : public CastToBase {
public:
diff --git a/be/src/exprs/function/cast/cast_to_int.h b/be/src/exprs/function/cast/cast_to_int.h
index d1c1ce47851..ba76ad590f8 100644
--- a/be/src/exprs/function/cast/cast_to_int.h
+++ b/be/src/exprs/function/cast/cast_to_int.h
@@ -213,8 +213,9 @@ constexpr static bool int_allow_cast_from_datetime =
template <CastModeType CastMode, typename FromDataType, typename ToDataType>
requires(((IsDateType<FromDataType> ||
IsDateV2Type<FromDataType>)&&int_allow_cast_from_date<ToDataType>) ||
- ((IsDateTimeType<FromDataType> ||
- IsDateTimeV2Type<FromDataType>)&&int_allow_cast_from_datetime<ToDataType>))
+ ((IsDateTimeType<FromDataType> || IsDateTimeV2Type<FromDataType> ||
+ std::is_same_v<FromDataType,
+ DataTypeTimeStampNs>)&&int_allow_cast_from_datetime<ToDataType>))
class CastToImpl<CastMode, FromDataType, ToDataType> : public CastToBase {
public:
Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments,
diff --git a/be/src/exprs/function/cast/cast_to_string.h b/be/src/exprs/function/cast/cast_to_string.h
index b9e96f86235..0f030f565be 100644
--- a/be/src/exprs/function/cast/cast_to_string.h
+++ b/be/src/exprs/function/cast/cast_to_string.h
@@ -22,6 +22,7 @@
#include "core/data_type_serde/data_type_serde.h"
#include "core/types.h"
#include "core/value/time_value.h"
+#include "core/value/timestamp_ns_value.h"
#include "exprs/function/cast/cast_base.h"
#include "util/mysql_global.h"
#include "util/to_string.h"
@@ -68,6 +69,7 @@ struct CastToString {
static inline std::string from_datetimev2(const DateV2Value<DateTimeV2ValueType>& from,
UInt32 scale);
+ static inline std::string from_timestamp_ns(const TimeStampNsValue& from);
static inline std::string from_timestamptz(const TimestampTzValue& from, UInt32 scale,
const cctz::time_zone* timezone = nullptr);
static inline void push_datetimev2(const DateV2Value<DateTimeV2ValueType>& from, UInt32 scale,
@@ -75,6 +77,7 @@ struct CastToString {
static inline void push_datetimev2(const DateV2Value<DateTimeV2ValueType>& from, UInt32 scale,
BufferWritable& bw);
+ static inline void push_timestamp_ns(const TimeStampNsValue& from, BufferWritable& bw);
static inline void push_timestamptz(const TimestampTzValue& from, UInt32 scale,
BufferWritable& bw,
const DataTypeSerDe::FormatOptions& options);
@@ -474,6 +477,10 @@ inline std::string CastToString::from_datetimev2(const DateV2Value<DateTimeV2Val
return std::string(buf, pos - 1);
}
+inline std::string CastToString::from_timestamp_ns(const TimeStampNsValue& from) {
+ return from.to_string();
+}
+
inline std::string CastToString::from_timestamptz(const TimestampTzValue& from, UInt32 scale,
const cctz::time_zone* timezone) {
cctz::time_zone tz;
@@ -500,6 +507,12 @@ inline void CastToString::push_datetimev2(const DateV2Value<DateTimeV2ValueType>
bw.write(buf, pos - buf - 1);
}
+inline void CastToString::push_timestamp_ns(const TimeStampNsValue& from, BufferWritable& bw) {
+ char buf[40];
+ const int32_t length = from.to_buffer(buf);
+ bw.write(buf, length);
+}
+
inline void CastToString::push_timestamptz(const TimestampTzValue& from, UInt32 scale,
BufferWritable& bw,
const DataTypeSerDe::FormatOptions& options) {
diff --git a/be/src/exprs/function/cast/cast_to_timestamp_ns.h b/be/src/exprs/function/cast/cast_to_timestamp_ns.h
new file mode 100644
index 00000000000..2ba379d940b
--- /dev/null
+++ b/be/src/exprs/function/cast/cast_to_timestamp_ns.h
@@ -0,0 +1,403 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <fmt/format.h>
+
+#include <cmath>
+#include <cstdint>
+#include <limits>
+#include <type_traits>
+
+#include "common/status.h"
+#include "core/binary_cast.hpp"
+#include "core/column/column_nullable.h"
+#include "core/data_type/data_type_date_or_datetime_v2.h"
+#include "core/data_type/data_type_date_time.h"
+#include "core/data_type/data_type_decimal.h"
+#include "core/data_type/data_type_number.h"
+#include "core/data_type/data_type_string.h"
+#include "core/data_type/data_type_time.h"
+#include "core/data_type/data_type_timestamp_ns.h"
+#include "core/data_type/data_type_timestamptz.h"
+#include "core/data_type_serde/data_type_serde.h"
+#include "core/data_type_serde/data_type_timestamp_ns_serde.h"
+#include "core/types.h"
+#include "core/value/time_value.h"
+#include "core/value/timestamptz_value.h"
+#include "exprs/function/cast/cast_base.h"
+#include "exprs/function/cast/cast_to_datetimev2_impl.hpp"
+#include "runtime/runtime_state.h"
+
+namespace doris {
+
+struct CastToTimestampNs {
+private:
+ struct FloatDecimalParts {
+ int64_t integer;
+ int64_t fraction;
+ uint32_t scale;
+ };
+
+ template <typename T>
+ requires std::is_floating_point_v<T>
+ static FloatDecimalParts split_float_to_decimal(T value) {
+ // fmt 7.x keeps Dragonbox in detail, so isolate the dependency in this helper. It is the
+ // same shortest-decimal conversion used by fmt's default floating-point formatter.
+ const auto decimal = fmt::detail::dragonbox::to_decimal(value);
+ const auto significand = decimal.significand;
+ const auto decimal_exponent = decimal.exponent;
+ if (decimal_exponent >= 0) {
+ const auto integer =
+ significand * common::exp10_i64(static_cast<uint32_t>(decimal_exponent));
+ return {.integer = static_cast<int64_t>(integer), .fraction = 0, .scale = 0};
+ }
+
+ const auto scale = static_cast<uint32_t>(-decimal_exponent);
+ if (scale >= 19) {
+ return {.integer = 0, .fraction = 0, .scale = 0};
+ }
+ const auto scale_multiplier = common::exp10_i64(scale);
+ return {.integer = static_cast<int64_t>(significand / scale_multiplier),
+ .fraction = static_cast<int64_t>(significand % scale_multiplier),
+ .scale = scale};
+ }
+
+public:
+ template <typename T>
+ static bool from_integer(T int_value, TimeStampNsValue& result, CastParameters& params) {
+ if (params.is_strict) {
+ return from_integer<DatelikeParseMode::STRICT>(int_value, result, params);
+ }
+ return from_integer<DatelikeParseMode::NON_STRICT>(int_value, result, params);
+ }
+
+ template <DatelikeParseMode ParseMode, typename T>
+ static bool from_integer(T int_value, TimeStampNsValue& result, CastParameters& params) {
+ constexpr bool IsStrict = is_datelike_parse_strict(ParseMode);
+ DCHECK(IsStrict == params.is_strict);
+
+ DateV2Value<DateTimeV2ValueType> datetime;
+ if (!CastToDatetimeV2::from_integer<ParseMode>(int_value, datetime, params)) {
+ return false;
+ }
+ SET_PARAMS_RET_FALSE_IFN(result.from_datetime(datetime),
+ "timestamp_ns value is outside the signed epoch-nanosecond range");
+ return true;
+ }
+
+ // Floating-point input has no fixed decimal scale. Use its shortest round-trippable decimal
+ // representation so digits 7-9 are not polluted by binary floating-point subtraction and the
+ // result preserves the existing string-formatting cast semantics.
+ template <typename T>
+ requires std::is_floating_point_v<T>
+ static bool from_float(T float_value, TimeStampNsValue& result, CastParameters& params) {
+ if (params.is_strict) {
+ return from_float<DatelikeParseMode::STRICT>(float_value, result, params);
+ }
+ return from_float<DatelikeParseMode::NON_STRICT>(float_value, result, params);
+ }
+
+ template <DatelikeParseMode ParseMode, typename T>
+ requires std::is_floating_point_v<T>
+ static bool from_float(T float_value, TimeStampNsValue& result, CastParameters& params) {
+ constexpr bool IsStrict = is_datelike_parse_strict(ParseMode);
+ DCHECK(IsStrict == params.is_strict);
+ SET_PARAMS_RET_FALSE_IFN(
+ float_value > 0 && std::isfinite(float_value) &&
+ float_value < static_cast<double>(std::numeric_limits<int64_t>::max()),
+ "invalid float value for timestamp_ns: {}", float_value);
+
+ const auto [integer, fraction, scale] = split_float_to_decimal(float_value);
+ return from_decimal<ParseMode>(integer, fraction, scale, result, params);
+ }
+
+ template <typename T>
+ static bool from_decimal(const T& int_part, const T& frac_part, int64_t decimal_scale,
+ TimeStampNsValue& result, CastParameters& params) {
+ if (params.is_strict) {
+ return from_decimal<DatelikeParseMode::STRICT>(int_part, frac_part, decimal_scale,
+ result, params);
+ }
+ return from_decimal<DatelikeParseMode::NON_STRICT>(int_part, frac_part, decimal_scale,
+ result, params);
+ }
+
+ template <DatelikeParseMode ParseMode, typename T>
+ static bool from_decimal(const T& int_part, const T& frac_part, int64_t decimal_scale,
+ TimeStampNsValue& result, CastParameters& params) {
+ constexpr bool IsStrict = is_datelike_parse_strict(ParseMode);
+ constexpr int64_t target_scale = TimeStampNsValue::FRACTIONAL_DIGITS;
+ DCHECK(IsStrict == params.is_strict);
+ SET_PARAMS_RET_FALSE_IFN(int_part <= std::numeric_limits<int64_t>::max() && int_part >= 1,
+ "invalid decimal value for timestamp_ns: {}.{}", int_part,
+ frac_part);
+
+ DateV2Value<DateTimeV2ValueType> datetime;
+ if (!CastToDatetimeV2::from_integer<ParseMode>(int_part, datetime, params)) {
+ return false;
+ }
+
+ T scaled_fraction = frac_part;
+ if (decimal_scale > target_scale) {
+ const auto divisor = decimal_scale_multiplier<T>(
+ static_cast<uint32_t>(decimal_scale - target_scale));
+ const auto remainder = scaled_fraction % divisor;
+ scaled_fraction /= divisor;
+ if (remainder >= divisor / 2) {
+ ++scaled_fraction;
+ }
+ } else if (decimal_scale < target_scale) {
+ scaled_fraction *= decimal_scale_multiplier<T>(
+ static_cast<uint32_t>(target_scale - decimal_scale));
+ }
+
+ const auto fraction_limit =
+ decimal_scale_multiplier<T>(static_cast<uint32_t>(target_scale));
+ if (scaled_fraction == fraction_limit) {
+ SET_PARAMS_RET_FALSE_IFN(datetime.template date_add_interval<TimeUnit::SECOND>(
+ TimeInterval {TimeUnit::SECOND, 1, false}),
+ "timestamp_ns overflow when rounding up to next second");
+ scaled_fraction = 0;
+ }
+
+ const auto nanoseconds = static_cast<uint32_t>(scaled_fraction);
+ datetime.set_microsecond(nanoseconds / TimeStampNsValue::NANOS_PER_MICROSECOND);
+ SET_PARAMS_RET_FALSE_IFN(
+ result.from_datetime(datetime,
+ nanoseconds % TimeStampNsValue::NANOS_PER_MICROSECOND),
+ "timestamp_ns value is outside the signed epoch-nanosecond range");
+ return true;
+ }
+
+ template <DatelikeParseMode ParseMode>
+ static inline bool from_string_strict_mode(const StringRef& str, TimeStampNsValue& res,
+ const cctz::time_zone* local_time_zone,
+ uint32_t to_scale, CastParameters& params) {
+ return CastToDatetimeV2::from_string_strict_mode<ParseMode>(str, res, local_time_zone,
+ to_scale, params);
+ }
+
+ static inline bool from_string_non_strict_mode(const StringRef& str, TimeStampNsValue& res,
+ const cctz::time_zone* local_time_zone,
+ uint32_t to_scale, CastParameters& params) {
+ return CastToDatetimeV2::from_string_non_strict_mode(str, res, local_time_zone, to_scale,
+ params);
+ }
+};
+
+template <CastModeType CastMode, typename FromDataType>
+ requires IsStringType<FromDataType>
+class CastToImpl<CastMode, FromDataType, DataTypeTimeStampNs> : public CastToBase {
+public:
+ Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments,
+ uint32_t result, size_t /*input_rows_count*/,
+ const NullMap::value_type* null_map = nullptr) const override {
+ const auto* col_from = assert_cast<const DataTypeString::ColumnType*>(
+ block.get_by_position(arguments[0]).column.get());
+
+ auto nested_to_type = remove_nullable(block.get_by_position(result).type);
+ auto serde = nested_to_type->get_serde();
+
+ DataTypeSerDe::FormatOptions options;
+ options.timezone = &context->state()->timezone_obj();
+
+ if constexpr (CastMode == CastModeType::StrictMode) {
+ MutableColumnPtr column_to = nested_to_type->create_column();
+ RETURN_IF_ERROR(
+ serde->from_string_strict_mode_batch(*col_from, *column_to, options, null_map));
+ block.get_by_position(result).column = std::move(column_to);
+ } else {
+ auto nullable_col_to = create_empty_nullable_column(nested_to_type);
+ RETURN_IF_ERROR(serde->from_string_batch(*col_from, *nullable_col_to, options));
+ block.get_by_position(result).column = std::move(nullable_col_to);
+ }
+
+ return Status::OK();
+ }
+};
+
+template <CastModeType CastMode, typename FromDataType>
+ requires CastUtil::IsPureDigitType<FromDataType>
+class CastToImpl<CastMode, FromDataType, DataTypeTimeStampNs> : public CastToBase {
+public:
+ Status execute_impl(FunctionContext* /*context*/, Block& block, const ColumnNumbers& arguments,
+ uint32_t result, size_t /*input_rows_count*/,
+ const NullMap::value_type* null_map = nullptr) const override {
+ const auto* col_from = assert_cast<const typename FromDataType::ColumnType*>(
+ block.get_by_position(arguments[0]).column.get());
+ auto nested_to_type = remove_nullable(block.get_by_position(result).type);
+ auto concrete_serde =
+ std::dynamic_pointer_cast<DataTypeTimeStampNsSerDe>(nested_to_type->get_serde());
+
+ if constexpr (CastMode == CastModeType::StrictMode) {
+ MutableColumnPtr column_to = nested_to_type->create_column();
+ RETURN_IF_ERROR(from_number_strict(*concrete_serde, *col_from, *column_to));
+ block.get_by_position(result).column = std::move(column_to);
+ } else {
+ auto nullable_col_to = create_empty_nullable_column(nested_to_type);
+ RETURN_IF_ERROR(from_number_non_strict(*concrete_serde, *col_from, *nullable_col_to));
+ block.get_by_position(result).column = std::move(nullable_col_to);
+ }
+ return Status::OK();
+ }
+
+private:
+ static Status from_number_strict(const DataTypeTimeStampNsSerDe& serde,
+ const typename FromDataType::ColumnType& column_from,
+ IColumn& column_to) {
+ if constexpr (IsDataTypeDecimal<FromDataType>) {
+ return serde.template from_decimal_strict_mode_batch<FromDataType>(column_from,
+ column_to);
+ } else if constexpr (IsDataTypeInt<FromDataType>) {
+ return serde.template from_int_strict_mode_batch<FromDataType>(column_from, column_to);
+ } else {
+ static_assert(IsDataTypeFloat<FromDataType>);
+ return serde.template from_float_strict_mode_batch<FromDataType>(column_from,
+ column_to);
+ }
+ }
+
+ static Status from_number_non_strict(const DataTypeTimeStampNsSerDe& serde,
+ const typename FromDataType::ColumnType& column_from,
+ ColumnNullable& column_to) {
+ if constexpr (IsDataTypeDecimal<FromDataType>) {
+ return serde.template from_decimal_batch<FromDataType>(column_from, column_to);
+ } else if constexpr (IsDataTypeInt<FromDataType>) {
+ return serde.template from_int_batch<FromDataType>(column_from, column_to);
+ } else {
+ static_assert(IsDataTypeFloat<FromDataType>);
+ return serde.template from_float_batch<FromDataType>(column_from, column_to);
+ }
+ }
+};
+
+template <CastModeType CastMode, typename FromDataType>
+ requires IsDatelikeTypes<FromDataType>
+class CastToImpl<CastMode, FromDataType, DataTypeTimeStampNs> : public CastToBase {
+public:
+ Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments,
+ uint32_t result, size_t input_rows_count,
+ const NullMap::value_type* null_map = nullptr) const override {
+ const auto& col_from = assert_cast<const typename FromDataType::ColumnType&>(
+ *block.get_by_position(arguments[0]).column);
+ auto col_to = ColumnTimeStampNs::create(input_rows_count);
+ auto col_nullmap = ColumnUInt8::create(input_rows_count, 0);
+
+ for (size_t i = 0; i < input_rows_count; ++i) {
+ if (null_map && null_map[i]) {
+ continue;
+ }
+
+ DateV2Value<DateTimeV2ValueType> datetime;
+ if constexpr (IsDateType<FromDataType>) {
+ const auto source = col_from.get_data()[i];
+ datetime = binary_cast<uint64_t, DateV2Value<DateTimeV2ValueType>>(
+ source.to_datetime_v2());
+ } else if constexpr (IsDateV2Type<FromDataType>) {
+ DataTypeDateV2::cast_to_date_time_v2(col_from.get_data()[i], datetime);
+ } else if constexpr (IsDateTimeType<FromDataType>) {
+ const auto source = col_from.get_data()[i];
+ datetime = binary_cast<uint64_t, DateV2Value<DateTimeV2ValueType>>(
+ source.to_datetime_v2());
+ } else if constexpr (IsDateTimeV2Type<FromDataType>) {
+ datetime = col_from.get_data()[i];
+ } else {
+ static_assert(IsTimeV2Type<FromDataType>);
+ const auto scale = block.get_by_position(arguments[0]).type->get_scale();
+ datetime.from_unixtime(context->state()->timestamp_ms() / 1000,
+ context->state()->nano_seconds(),
+ context->state()->timezone_obj(), scale);
+ datetime.reset_time_part();
+
+ const auto time_value = col_from.get_data()[i];
+ const bool negative = TimeValue::sign(time_value) < 0;
+ datetime.template date_add_interval<TimeUnit::HOUR, false>(
+ TimeInterval(HOUR, TimeValue::hour(time_value), negative));
+ datetime.template date_add_interval<TimeUnit::MINUTE, false>(
+ TimeInterval(MINUTE, TimeValue::minute(time_value), negative));
+ datetime.template date_add_interval<TimeUnit::SECOND, false>(
+ TimeInterval(SECOND, TimeValue::second(time_value), negative));
+ datetime.template date_add_interval<TimeUnit::MICROSECOND, false>(
+ TimeInterval(MICROSECOND, TimeValue::microsecond(time_value), negative));
+ }
+
+ if (!col_to->get_data()[i].from_datetime(datetime)) {
+ if constexpr (CastMode == CastModeType::StrictMode) {
+ return Status::InvalidArgument(
+ "TIMESTAMP_NS overflow when casting row {} from {} to TIMESTAMP_NS", i,
+ block.get_by_position(arguments[0]).type->get_name());
+ }
+ col_nullmap->get_data()[i] = true;
+ }
+ }
+
+ if constexpr (CastMode == CastModeType::StrictMode) {
+ block.get_by_position(result).column = std::move(col_to);
+ } else {
+ block.get_by_position(result).column =
+ ColumnNullable::create(std::move(col_to), std::move(col_nullmap));
+ }
+ return Status::OK();
+ }
+};
+
+template <CastModeType CastMode>
+class CastToImpl<CastMode, DataTypeTimeStampTz, DataTypeTimeStampNs> : public CastToBase {
+public:
+ Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments,
+ uint32_t result, size_t input_rows_count,
+ const NullMap::value_type* null_map = nullptr) const override {
+ const auto& col_from =
+ assert_cast<const ColumnTimeStampTz&>(*block.get_by_position(arguments[0]).column)
+ .get_data();
+ auto col_to = ColumnTimeStampNs::create(input_rows_count);
+ auto col_null = ColumnBool::create(input_rows_count, 0);
+ const auto& local_time_zone = context->state()->timezone_obj();
+ const auto source_scale = block.get_by_position(arguments[0]).type->get_scale();
+
+ for (size_t i = 0; i < input_rows_count; ++i) {
+ if (null_map && null_map[i]) {
+ continue;
+ }
+ TimestampTzValue source {col_from[i]};
+ DateV2Value<DateTimeV2ValueType> datetime;
+ const bool converted =
+ source.to_datetime(datetime, local_time_zone, source_scale, source_scale) &&
+ col_to->get_data()[i].from_datetime(datetime);
+ if (!converted) {
+ if constexpr (CastMode == CastModeType::StrictMode) {
+ return Status::InvalidArgument(
+ "can not cast timestamptz {} to TIMESTAMP_NS in timezone {}",
+ source.to_string(local_time_zone), context->state()->timezone());
+ }
+ col_null->get_data()[i] = true;
+ }
+ }
+
+ if constexpr (CastMode == CastModeType::StrictMode) {
+ block.get_by_position(result).column = std::move(col_to);
+ } else {
+ block.get_by_position(result).column =
+ ColumnNullable::create(std::move(col_to), std::move(col_null));
+ }
+ return Status::OK();
+ }
+};
+
+} // namespace doris
diff --git a/be/src/exprs/function/cast/cast_to_timestamptz.h b/be/src/exprs/function/cast/cast_to_timestamptz.h
index 6e4cf356609..4657467dbfa 100644
--- a/be/src/exprs/function/cast/cast_to_timestamptz.h
+++ b/be/src/exprs/function/cast/cast_to_timestamptz.h
@@ -19,6 +19,7 @@
#include "core/assert_cast.h"
#include "core/data_type/data_type_date_or_datetime_v2.h"
+#include "core/data_type/data_type_timestamp_ns.h"
#include "core/data_type/data_type_timestamptz.h"
#include "core/data_type/primitive_type.h"
#include "core/types.h"
@@ -138,6 +139,53 @@ public:
}
};
+template <CastModeType CastMode>
+class CastToImpl<CastMode, DataTypeTimeStampNs, DataTypeTimeStampTz> : public CastToBase {
+public:
+ Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments,
+ uint32_t result, size_t input_rows_count,
+ const NullMap::value_type* null_map = nullptr) const override {
+ const auto& col_from =
+ assert_cast<const ColumnTimeStampNs&>(*block.get_by_position(arguments[0]).column)
+ .get_data();
+ auto col_to = ColumnTimeStampTz::create(input_rows_count);
+ auto col_null = ColumnBool::create(input_rows_count, 0);
+ const auto& local_time_zone = context->state()->timezone_obj();
+ const auto target_scale = block.get_by_position(result).type->get_scale();
+
+ for (size_t i = 0; i < input_rows_count; ++i) {
+ if (null_map && null_map[i]) {
+ continue;
+ }
+ DateV2Value<DateTimeV2ValueType> rounded_datetime;
+ TimestampTzValue target;
+ const bool converted =
+ transform_date_scale(target_scale, TimeStampNsValue::FRACTIONAL_DIGITS,
+ rounded_datetime, col_from[i]) &&
+ target.from_datetime(rounded_datetime, local_time_zone, target_scale,
+ target_scale);
+ if (!converted) {
+ if constexpr (CastMode == CastModeType::StrictMode) {
+ return Status::InvalidArgument(
+ "can not cast TIMESTAMP_NS {} to timestamptz in timezone {}",
+ col_from[i].to_string(), context->state()->timezone());
+ }
+ col_null->get_data()[i] = true;
+ continue;
+ }
+ col_to->get_data()[i] = target;
+ }
+
+ if constexpr (CastMode == CastModeType::StrictMode) {
+ block.get_by_position(result).column = std::move(col_to);
+ } else {
+ block.get_by_position(result).column =
+ ColumnNullable::create(std::move(col_to), std::move(col_null));
+ }
+ return Status::OK();
+ }
+};
+
template <>
class CastToImpl<CastModeType::StrictMode, DataTypeTimeStampTz, DataTypeTimeStampTz>
: public CastToBase {
diff --git a/be/src/exprs/function/cast/cast_wrapper_decls.h b/be/src/exprs/function/cast/cast_wrapper_decls.h
index 987c852880f..1bcd6bfde5d 100644
--- a/be/src/exprs/function/cast/cast_wrapper_decls.h
+++ b/be/src/exprs/function/cast/cast_wrapper_decls.h
@@ -37,7 +37,10 @@ WrapperType create_decimal_wrapper(FunctionContext* context, const DataTypePtr&
WrapperType create_datelike_wrapper(FunctionContext* context, const DataTypePtr& from_type,
PrimitiveType to_type);
-// Implemented in function_cast_date.cpp
+// Implemented in function_cast_timestamp_ns.cpp
+WrapperType create_timestamp_ns_wrapper(FunctionContext* context, const DataTypePtr& from_type);
+
+// Implemented in function_cast_timestamptz.cpp
WrapperType create_timestamptz_wrapper(FunctionContext* context, const DataTypePtr& from_type);
// Implemented in function_cast_ip.cpp
diff --git a/be/src/exprs/function/cast/function_cast.cpp b/be/src/exprs/function/cast/function_cast.cpp
index 75879ebf6b3..c838ba6fb46 100644
--- a/be/src/exprs/function/cast/function_cast.cpp
+++ b/be/src/exprs/function/cast/function_cast.cpp
@@ -19,7 +19,8 @@
#include "core/data_type/data_type_agg_state.h"
#include "core/data_type/data_type_decimal.h"
-#include "core/data_type/data_type_number.h" // IWYU pragma: keep
+#include "core/data_type/data_type_number.h" // IWYU pragma: keep
+#include "core/data_type/data_type_timestamp_ns.h" // IWYU pragma: keep
#include "core/data_type/data_type_variant.h"
#include "core/data_type/data_type_variant_v2.h"
#include "core/data_type/primitive_type.h"
@@ -301,6 +302,8 @@ WrapperType prepare_impl(FunctionContext* context, const DataTypePtr& origin_fro
case PrimitiveType::TYPE_DATETIMEV2:
case PrimitiveType::TYPE_TIMEV2:
return create_datelike_wrapper(context, from_type, to_type->get_primitive_type());
+ case PrimitiveType::TYPE_TIMESTAMP_NS:
+ return create_timestamp_ns_wrapper(context, from_type);
case PrimitiveType::TYPE_TIMESTAMPTZ:
return create_timestamptz_wrapper(context, from_type);
case PrimitiveType::TYPE_IPV4:
diff --git a/be/src/exprs/function/cast/function_cast_date.cpp b/be/src/exprs/function/cast/function_cast_date.cpp
index 88013ab5e3a..104720ade61 100644
--- a/be/src/exprs/function/cast/function_cast_date.cpp
+++ b/be/src/exprs/function/cast/function_cast_date.cpp
@@ -19,6 +19,7 @@
#include "core/data_type/data_type_date_or_datetime_v2.h"
#include "core/data_type/data_type_date_time.h"
#include "core/data_type/data_type_time.h"
+#include "core/data_type/data_type_timestamp_ns.h"
#include "exprs/function/cast/cast_to_date.h"
namespace doris::CastWrapper {
@@ -30,7 +31,8 @@ WrapperType create_datelike_wrapper(FunctionContext* context, const DataTypePtr&
auto make_datelike_wrapper = [&](const auto& types) -> bool {
using Types = std::decay_t<decltype(types)>;
using FromDataType = typename Types::LeftType;
- if constexpr (CastUtil::IsPureDigitType<FromDataType> || IsDatelikeTypes<FromDataType> ||
+ if constexpr (std::is_same_v<FromDataType, DataTypeTimeStampNs> ||
+ CastUtil::IsPureDigitType<FromDataType> || IsDatelikeTypes<FromDataType> ||
IsStringType<FromDataType> ||
std::is_same_v<FromDataType, DataTypeTimeStampTz>) {
if (context->enable_strict_mode()) {
diff --git a/be/src/exprs/function/cast/function_cast_timestamptz.cpp b/be/src/exprs/function/cast/function_cast_timestamp_ns.cpp
similarity index 52%
copy from be/src/exprs/function/cast/function_cast_timestamptz.cpp
copy to be/src/exprs/function/cast/function_cast_timestamp_ns.cpp
index a26e1d5c5c2..9b86a3d3235 100644
--- a/be/src/exprs/function/cast/function_cast_timestamptz.cpp
+++ b/be/src/exprs/function/cast/function_cast_timestamp_ns.cpp
@@ -15,43 +15,45 @@
// specific language governing permissions and limitations
// under the License.
-#include "exprs/function/cast/cast_to_timestamptz.h"
+#include "exprs/function/cast/cast_to_timestamp_ns.h"
namespace doris::CastWrapper {
-WrapperType create_timestamptz_wrapper(FunctionContext* context, const DataTypePtr& from_type) {
- std::shared_ptr<CastToBase> cast_to_timestamptz;
- auto make_timestamptz_wrapper = [&](const auto& types) -> bool {
+WrapperType create_timestamp_ns_wrapper(FunctionContext* context, const DataTypePtr& from_type) {
+ std::shared_ptr<CastToBase> cast_to_timestamp_ns;
+
+ auto make_timestamp_ns_wrapper = [&](const auto& types) -> bool {
using Types = std::decay_t<decltype(types)>;
using FromDataType = typename Types::LeftType;
- if constexpr (CastUtil::IsBaseCastFromType<FromDataType> ||
- IsTimeStampTzType<FromDataType>) {
+ if constexpr (IsStringType<FromDataType> || CastUtil::IsPureDigitType<FromDataType> ||
+ IsDatelikeTypes<FromDataType> ||
+ std::is_same_v<FromDataType, DataTypeTimeStampTz>) {
if (context->enable_strict_mode()) {
- cast_to_timestamptz = std::make_shared<
- CastToImpl<CastModeType::StrictMode, FromDataType, DataTypeTimeStampTz>>();
+ cast_to_timestamp_ns = std::make_shared<
+ CastToImpl<CastModeType::StrictMode, FromDataType, DataTypeTimeStampNs>>();
} else {
- cast_to_timestamptz =
+ cast_to_timestamp_ns =
std::make_shared<CastToImpl<CastModeType::NonStrictMode, FromDataType,
- DataTypeTimeStampTz>>();
+ DataTypeTimeStampNs>>();
}
return true;
- } else {
- return false;
}
+ return false;
};
if (!call_on_index_and_data_type<void>(from_type->get_primitive_type(),
- make_timestamptz_wrapper)) {
+ make_timestamp_ns_wrapper)) {
return create_unsupport_wrapper(
- fmt::format("CAST AS timestamptz not supported {}", from_type->get_name()));
+ fmt::format("CAST AS TIMESTAMP_NS not supported {}", from_type->get_name()));
}
- return [cast_to_timestamptz](FunctionContext* context, Block& block,
- const ColumnNumbers& arguments, uint32_t result,
- size_t input_rows_count,
- const NullMap::value_type* null_map = nullptr) {
- return cast_to_timestamptz->execute_impl(context, block, arguments, result,
- input_rows_count, null_map);
+ return [cast_to_timestamp_ns](FunctionContext* context, Block& block,
+ const ColumnNumbers& arguments, uint32_t result,
+ size_t input_rows_count,
+ const NullMap::value_type* null_map = nullptr) {
+ return cast_to_timestamp_ns->execute_impl(context, block, arguments, result,
+ input_rows_count, null_map);
};
}
+
} // namespace doris::CastWrapper
diff --git a/be/src/exprs/function/cast/function_cast_timestamptz.cpp b/be/src/exprs/function/cast/function_cast_timestamptz.cpp
index a26e1d5c5c2..a2d30292370 100644
--- a/be/src/exprs/function/cast/function_cast_timestamptz.cpp
+++ b/be/src/exprs/function/cast/function_cast_timestamptz.cpp
@@ -25,7 +25,8 @@ WrapperType create_timestamptz_wrapper(FunctionContext* context, const DataTypeP
using Types = std::decay_t<decltype(types)>;
using FromDataType = typename Types::LeftType;
if constexpr (CastUtil::IsBaseCastFromType<FromDataType> ||
- IsTimeStampTzType<FromDataType>) {
+ IsTimeStampTzType<FromDataType> ||
+ std::is_same_v<FromDataType, DataTypeTimeStampNs>) {
if (context->enable_strict_mode()) {
cast_to_timestamptz = std::make_shared<
CastToImpl<CastModeType::StrictMode, FromDataType, DataTypeTimeStampTz>>();
diff --git a/be/src/exprs/function/cast/variant_v2/cast_variant_to_scalar.cpp b/be/src/exprs/function/cast/variant_v2/cast_variant_to_scalar.cpp
index f5e6ede566b..3b09ccfa948 100644
--- a/be/src/exprs/function/cast/variant_v2/cast_variant_to_scalar.cpp
+++ b/be/src/exprs/function/cast/variant_v2/cast_variant_to_scalar.cpp
@@ -16,11 +16,15 @@
// under the License.
#include <cctz/time_zone.h>
+#include <fmt/format.h>
#include <algorithm>
#include <array>
#include <cstdint>
+#include <iterator>
+#include <limits>
#include <memory>
+#include <string>
#include <utility>
#include "common/exception.h"
@@ -36,12 +40,14 @@
#include "core/data_type/data_type_nullable.h"
#include "core/data_type/data_type_number.h"
#include "core/data_type/data_type_string.h"
+#include "core/data_type/data_type_timestamp_ns.h"
#include "core/data_type/data_type_timestamptz.h"
#include "core/value/timestamptz_value.h"
#include "core/value/vdatetime_value.h"
#include "exprs/function/cast/cast_base.h"
#include "exprs/function/cast/variant_v2/cast_variant_v2_internal.h"
#include "exprs/function_context.h"
+#include "runtime/runtime_state.h"
namespace doris::CastWrapper::variant_v2_internal {
namespace {
@@ -57,7 +63,8 @@ enum GroupIndex : size_t {
DECIMAL_GROUP_BEGIN = DOUBLE_GROUP + 1,
DATE_GROUP = DECIMAL_GROUP_BEGIN + DECIMAL_SCALE_COUNT,
TIMESTAMP_NTZ_GROUP = DATE_GROUP + 1,
- TIMESTAMP_TZ_GROUP = TIMESTAMP_NTZ_GROUP + 1,
+ TIMESTAMP_NANOS_GROUP = TIMESTAMP_NTZ_GROUP + 1,
+ TIMESTAMP_TZ_GROUP = TIMESTAMP_NANOS_GROUP + 1,
STRING_GROUP = TIMESTAMP_TZ_GROUP + 1,
GROUP_COUNT = STRING_GROUP + 1,
};
@@ -203,6 +210,29 @@ void append_timestamp(ScalarGroups& groups, size_t row, int64_t micros, bool utc
group.source_rows.push_back(row);
}
+void append_timestamp_nanos(FunctionContext* context, ScalarGroups& groups, size_t row,
+ int64_t nanos, bool utc_adjusted) {
+ if (utc_adjusted) {
+ DORIS_CHECK(context != nullptr);
+ DORIS_CHECK(context->state() != nullptr);
+ const auto seconds = TimeStampNsValue(nanos).epoch_seconds();
+ const auto instant = cctz::time_point<cctz::seconds>(cctz::seconds(seconds));
+ const auto offset = context->state()->timezone_obj().lookup(instant).offset;
+ const auto local_nanos = static_cast<__int128>(nanos) +
+ static_cast<__int128>(offset) * TimeStampNsValue::NANOS_PER_SECOND;
+ if (local_nanos < std::numeric_limits<int64_t>::min() ||
+ local_nanos > std::numeric_limits<int64_t>::max()) {
+ append_invalid(groups, row);
+ return;
+ }
+ nanos = static_cast<int64_t>(local_nanos);
+ }
+ auto& group = initialize_group(groups, TIMESTAMP_NANOS_GROUP,
+ [] { return std::make_shared<DataTypeTimeStampNs>(); });
+ assert_cast<ColumnTimeStampNs&>(*group.values).insert_value(TimeStampNsValue(nanos));
+ group.source_rows.push_back(row);
+}
+
void append_string(ScalarGroups& groups, size_t row, StringRef value) {
auto& group = initialize_group(groups, STRING_GROUP,
[] { return std::make_shared<DataTypeString>(); });
@@ -210,7 +240,25 @@ void append_string(ScalarGroups& groups, size_t row, StringRef value) {
group.source_rows.push_back(row);
}
-void classify_value(ScalarGroups& groups, size_t row, VariantRef value, bool forced_null) {
+void append_timestamp_nanos_string(FunctionContext* context, ScalarGroups& groups, size_t row,
+ int64_t nanos, bool utc_adjusted) {
+ const TimeStampNsValue value(nanos);
+ if (!utc_adjusted) {
+ const std::string text = value.to_string();
+ append_string(groups, row, StringRef(text.data(), text.size()));
+ return;
+ }
+
+ DORIS_CHECK(context != nullptr);
+ DORIS_CHECK(context->state() != nullptr);
+ const auto instant = cctz::time_point<cctz::seconds>(cctz::seconds(value.epoch_seconds()));
+ std::string text = cctz::format("%Y-%m-%d %H:%M:%S", instant, context->state()->timezone_obj());
+ fmt::format_to(std::back_inserter(text), ".{:09}", value.nanosecond());
+ append_string(groups, row, StringRef(text.data(), text.size()));
+}
+
+void classify_value(FunctionContext* context, ScalarGroups& groups, size_t row, VariantRef value,
+ PrimitiveType target_primitive, bool forced_null) {
if (forced_null) {
append_invalid(groups, row);
return;
@@ -267,9 +315,26 @@ void classify_value(ScalarGroups& groups, size_t row, VariantRef value, bool for
append_timestamp(groups, row, value.get_timestamp_ntz_micros(), false);
return;
case VariantPrimitiveId::TIMESTAMP_NANOS:
+ if (target_primitive == TYPE_STRING) {
+ append_timestamp_nanos_string(context, groups, row, value.get_timestamp_nanos(), true);
+ return;
+ }
+ if (target_primitive == TYPE_TIMESTAMP_NS) {
+ append_timestamp_nanos(context, groups, row, value.get_timestamp_nanos(), true);
+ return;
+ }
append_timestamp(groups, row, nanos_to_micros(value.get_timestamp_nanos()), true);
return;
case VariantPrimitiveId::TIMESTAMP_NTZ_NANOS:
+ if (target_primitive == TYPE_STRING) {
+ append_timestamp_nanos_string(context, groups, row, value.get_timestamp_ntz_nanos(),
+ false);
+ return;
+ }
+ if (target_primitive == TYPE_TIMESTAMP_NS) {
+ append_timestamp_nanos(context, groups, row, value.get_timestamp_ntz_nanos(), false);
+ return;
+ }
append_timestamp(groups, row, nanos_to_micros(value.get_timestamp_ntz_nanos()), false);
return;
case VariantPrimitiveId::STRING:
@@ -404,6 +469,7 @@ bool is_supported_scalar_target(const DataTypePtr& type) {
case TYPE_DATEV2:
case TYPE_DATETIME:
case TYPE_DATETIMEV2:
+ case TYPE_TIMESTAMP_NS:
case TYPE_TIMESTAMPTZ:
case TYPE_IPV4:
case TYPE_IPV6:
@@ -455,7 +521,8 @@ Status cast_variant_refs_to_scalar(FunctionContext* context, std::span<const Var
}
ScalarGroups groups;
for (size_t row = 0; row < values.size(); ++row) {
- classify_value(groups, row, values[row], !forced_nulls.empty() && forced_nulls[row] != 0);
+ classify_value(context, groups, row, values[row], target_type->get_primitive_type(),
+ !forced_nulls.empty() && forced_nulls[row] != 0);
}
return assemble_groups(context, groups, target_type, values.size(), output);
}
@@ -469,7 +536,8 @@ Status cast_encoded_variant_to_scalar(FunctionContext* context, const ColumnVari
}
ScalarGroups groups;
for (size_t row = 0; row < rows; ++row) {
- classify_value(groups, row, source.get_value_ref(row),
+ classify_value(context, groups, row, source.get_value_ref(row),
+ target_type->get_primitive_type(),
!forced_nulls.empty() && forced_nulls[row] != 0);
}
return assemble_groups(context, groups, target_type, rows, output);
@@ -484,7 +552,10 @@ Status cast_variant_values_to_scalar(FunctionContext* context, const ColumnVaria
ScalarGroups groups;
visit_variant_v2_values(
source, 0, rows, forced_nulls, [&](size_t row) { append_invalid(groups, row); },
- [&](size_t row, VariantRef value) { classify_value(groups, row, value, false); });
+ [&](size_t row, VariantRef value) {
+ classify_value(context, groups, row, value, target_type->get_primitive_type(),
+ false);
+ });
return assemble_groups(context, groups, target_type, rows, output);
}
diff --git a/be/src/exprs/function/date_format_type.h b/be/src/exprs/function/date_format_type.h
index 7869f3e7be2..d72df4dcc83 100644
--- a/be/src/exprs/function/date_format_type.h
+++ b/be/src/exprs/function/date_format_type.h
@@ -113,7 +113,8 @@ struct yyyy_MM_dd_HH_mm_ssImpl {
struct yyyy_MM_dd_HH_mm_ss_SSSSSSImpl {
constexpr static size_t row_size = 26;
- size_t static date_to_str(const DateV2Value<DateTimeV2ValueType>& date_value, char* buf) {
+ template <typename DateType>
+ size_t static date_to_str(const DateType& date_value, char* buf) {
int i = 0;
put_year(date_value.year(), buf, i);
buf[i++] = '-';
diff --git a/be/src/exprs/function/date_time_transforms.h b/be/src/exprs/function/date_time_transforms.h
index c5de24aa845..6dbf56b04fb 100644
--- a/be/src/exprs/function/date_time_transforms.h
+++ b/be/src/exprs/function/date_time_transforms.h
@@ -103,6 +103,25 @@ TO_TIME_FUNCTION(ToHourImpl, hour);
TO_TIME_FUNCTION(ToMinuteImpl, minute);
TO_TIME_FUNCTION(ToSecondImpl, second);
TO_TIME_FUNCTION(ToMicroSecondImpl, microsecond);
+template <PrimitiveType PType>
+struct ToNanoSecondImpl {
+ static constexpr PrimitiveType OpArgType = PType;
+ using CppType = typename PrimitiveTypeTraits<PType>::CppType;
+ static constexpr auto name = "nanosecond";
+
+ static inline auto execute(const CppType& date_time_value) {
+ if constexpr (PType == TYPE_TIMESTAMP_NS) {
+ return date_time_value.nanosecond();
+ } else {
+ return static_cast<uint32_t>(date_time_value.microsecond()) *
+ static_cast<uint32_t>(TimeStampNsValue::NANOS_PER_MICROSECOND);
+ }
+ }
+
+ static DataTypes get_variadic_argument_types() {
+ return {std::make_shared<typename PrimitiveTypeTraits<PType>::DataType>()};
+ }
+};
TIME_FUNCTION_IMPL(WeekOfYearImpl, weekofyear, week(mysql_week_mode(3)));
TIME_FUNCTION_IMPL(DayOfYearImpl, dayofyear, day_of_year());
@@ -146,6 +165,8 @@ struct ToDateImpl {
} else if constexpr (std::is_same_v<DateType, VecDateTimeValue>) {
t.cast_to_date();
return t;
+ } else if constexpr (std::is_same_v<DateType, TimeStampNsValue>) {
+ return t.to_date();
} else {
return binary_cast<UInt32, DateV2Value<DateV2ValueType>>(
(UInt32)(t.to_date_int_val() >> TIME_PART_LENGTH));
@@ -206,14 +227,27 @@ struct ToIso8601Impl {
static constexpr PrimitiveType OpArgType = PType;
using ArgType = typename PrimitiveTypeTraits<PType>::CppType;
static constexpr auto name = "to_iso8601";
- static constexpr auto max_size = PType == TYPE_DATEV2 ? 10 : 26;
+ static constexpr auto max_size = [] {
+ if constexpr (PType == TYPE_DATEV2) {
+ return 10;
+ } else if constexpr (PType == TYPE_TIMESTAMP_NS) {
+ return 29;
+ } else {
+ return 26;
+ }
+ }();
static auto execute(const typename PrimitiveTypeTraits<PType>::CppType& dt,
ColumnString::Chars& res_data, size_t& offset,
const char* const* /*names_ptr*/, FunctionContext* /*context*/) {
- auto length = dt.to_buffer((char*)res_data.data() + offset,
- std::is_same_v<ArgType, UInt32> ? -1 : 6);
- if (PType == TYPE_DATETIMEV2 || PType == TYPE_TIMESTAMPTZ) {
+ int length = 0;
+ if constexpr (PType == TYPE_TIMESTAMP_NS) {
+ length = dt.to_buffer((char*)res_data.data() + offset);
+ } else {
+ constexpr int scale = PType == TYPE_DATEV2 ? -1 : 6;
+ length = dt.to_buffer((char*)res_data.data() + offset, scale);
+ }
+ if (PType == TYPE_DATETIMEV2 || PType == TYPE_TIMESTAMP_NS || PType == TYPE_TIMESTAMPTZ) {
res_data[offset + 10] = 'T';
}
@@ -303,22 +337,37 @@ struct MonthNameImpl {
}
};
-template <typename FormatImpl, const char* FuncName>
+template <PrimitiveType PType, typename FormatImpl, const char* FuncName>
struct DateTimeV2FormatImpl {
- static constexpr PrimitiveType OpArgType = TYPE_DATETIMEV2;
- using ArgType = typename PrimitiveTypeTraits<TYPE_DATETIMEV2>::CppType;
+ static_assert(PType == TYPE_DATETIMEV2 || PType == TYPE_TIMESTAMP_NS);
+ static constexpr PrimitiveType OpArgType = PType;
+ using ArgType = typename PrimitiveTypeTraits<PType>::CppType;
static constexpr auto name = FuncName;
static constexpr auto max_size = FormatImpl::row_size;
static auto execute(const ArgType& dt, ColumnString::Chars& res_data, size_t& offset,
const char* const* /*names_ptr*/, FunctionContext* /*context*/) {
auto* buf = reinterpret_cast<char*>(&res_data[offset]);
- offset += FormatImpl::date_to_str(dt, buf);
+ constexpr bool needs_civil_datetime =
+ std::is_same_v<FormatImpl, time_format_type::dd_HHImpl> ||
+ std::is_same_v<FormatImpl, time_format_type::dd_HH_mmImpl> ||
+ std::is_same_v<FormatImpl, time_format_type::dd_HH_mm_ssImpl> ||
+ std::is_same_v<FormatImpl, time_format_type::dd_HH_mm_ss_SSSSSSImpl>;
+ if constexpr (PType == TYPE_TIMESTAMP_NS &&
+ std::is_same_v<FormatImpl, time_format_type::yyyy_MMImpl>) {
+ const auto civil_value = dt.to_date();
+ offset += FormatImpl::date_to_str(civil_value, buf);
+ } else if constexpr (PType == TYPE_TIMESTAMP_NS && needs_civil_datetime) {
+ const auto civil_value = dt.to_datetime();
+ offset += FormatImpl::date_to_str(civil_value, buf);
+ } else {
+ offset += FormatImpl::date_to_str(dt, buf);
+ }
return offset;
}
static DataTypes get_variadic_argument_types() {
- return {std::make_shared<typename PrimitiveTypeTraits<TYPE_DATETIMEV2>::DataType>()};
+ return {std::make_shared<typename PrimitiveTypeTraits<PType>::DataType>()};
}
};
@@ -334,21 +383,34 @@ inline constexpr char kMinuteSecondName[] = "minute_second";
inline constexpr char kMinuteMicrosecondName[] = "minute_microsecond";
inline constexpr char kSecondMicrosecondName[] = "second_microsecond";
-using YearMonthImpl = DateTimeV2FormatImpl<time_format_type::yyyy_MMImpl, kYearMonthName>;
-using DayHourImpl = DateTimeV2FormatImpl<time_format_type::dd_HHImpl, kDayHourName>;
-using DayMinuteImpl = DateTimeV2FormatImpl<time_format_type::dd_HH_mmImpl, kDayMinuteName>;
-using DaySecondImpl = DateTimeV2FormatImpl<time_format_type::dd_HH_mm_ssImpl, kDaySecondName>;
+template <PrimitiveType PType>
+using YearMonthImpl = DateTimeV2FormatImpl<PType, time_format_type::yyyy_MMImpl, kYearMonthName>;
+template <PrimitiveType PType>
+using DayHourImpl = DateTimeV2FormatImpl<PType, time_format_type::dd_HHImpl, kDayHourName>;
+template <PrimitiveType PType>
+using DayMinuteImpl = DateTimeV2FormatImpl<PType, time_format_type::dd_HH_mmImpl, kDayMinuteName>;
+template <PrimitiveType PType>
+using DaySecondImpl =
+ DateTimeV2FormatImpl<PType, time_format_type::dd_HH_mm_ssImpl, kDaySecondName>;
+template <PrimitiveType PType>
using DayMicrosecondImpl =
- DateTimeV2FormatImpl<time_format_type::dd_HH_mm_ss_SSSSSSImpl, kDayMicrosecondName>;
-using HourMinuteImpl = DateTimeV2FormatImpl<time_format_type::HH_mmImpl, kHourMinuteName>;
-using HourSecondImpl = DateTimeV2FormatImpl<time_format_type::HH_mm_ssImpl, kHourSecondName>;
+ DateTimeV2FormatImpl<PType, time_format_type::dd_HH_mm_ss_SSSSSSImpl, kDayMicrosecondName>;
+template <PrimitiveType PType>
+using HourMinuteImpl = DateTimeV2FormatImpl<PType, time_format_type::HH_mmImpl, kHourMinuteName>;
+template <PrimitiveType PType>
+using HourSecondImpl = DateTimeV2FormatImpl<PType, time_format_type::HH_mm_ssImpl, kHourSecondName>;
+template <PrimitiveType PType>
using HourMicrosecondImpl =
- DateTimeV2FormatImpl<time_format_type::HH_mm_ss_SSSSSSImpl, kHourMicrosecondName>;
-using MinuteSecondImpl = DateTimeV2FormatImpl<time_format_type::mm_ssImpl, kMinuteSecondName>;
+ DateTimeV2FormatImpl<PType, time_format_type::HH_mm_ss_SSSSSSImpl, kHourMicrosecondName>;
+template <PrimitiveType PType>
+using MinuteSecondImpl =
+ DateTimeV2FormatImpl<PType, time_format_type::mm_ssImpl, kMinuteSecondName>;
+template <PrimitiveType PType>
using MinuteMicrosecondImpl =
- DateTimeV2FormatImpl<time_format_type::mm_ss_SSSSSSImpl, kMinuteMicrosecondName>;
+ DateTimeV2FormatImpl<PType, time_format_type::mm_ss_SSSSSSImpl, kMinuteMicrosecondName>;
+template <PrimitiveType PType>
using SecondMicrosecondImpl =
- DateTimeV2FormatImpl<time_format_type::ss_SSSSSSImpl, kSecondMicrosecondName>;
+ DateTimeV2FormatImpl<PType, time_format_type::ss_SSSSSSImpl, kSecondMicrosecondName>;
template <PrimitiveType PType>
struct DateFormatImpl {
@@ -364,8 +426,17 @@ struct DateFormatImpl {
if constexpr (std::is_same_v<Impl, time_format_type::UserDefinedImpl>) {
// Handle non-special formats.
char buf[100 + SAFE_FORMAT_STRING_MARGIN];
- if (!dt.to_format_string_conservative(format.data, format.size, buf,
- 100 + SAFE_FORMAT_STRING_MARGIN)) {
+ const bool formatted = [&]() {
+ if constexpr (PType == PrimitiveType::TYPE_DATETIMEV2) {
+ return dt.to_format_string_conservative(
+ format.data, format.size, buf, 100 + SAFE_FORMAT_STRING_MARGIN,
+ static_cast<int>(dt.microsecond()) * 1000);
+ } else {
+ return dt.to_format_string_conservative(format.data, format.size, buf,
+ 100 + SAFE_FORMAT_STRING_MARGIN);
+ }
+ }();
+ if (!formatted) {
return true;
}
@@ -380,7 +451,23 @@ struct DateFormatImpl {
// No buffer is needed here because these specially optimized formats have fixed lengths,
// and sufficient memory has already been reserved.
- auto len = Impl::date_to_str(dt, (char*)res_data.data() + offset);
+ const auto len = [&]() {
+ if constexpr (PType == TYPE_TIMESTAMP_NS) {
+ constexpr bool needs_civil_time =
+ std::is_same_v<Impl, time_format_type::yyyy_MM_dd_HH_mm_ssImpl> ||
+ std::is_same_v<Impl, time_format_type::yyyy_MM_dd_HH_mm_ss_SSSSSSImpl>;
+ const auto civil_value = [&]() {
+ if constexpr (needs_civil_time) {
+ return dt.to_datetime();
+ } else {
+ return dt.to_date();
+ }
+ }();
+ return Impl::date_to_str(civil_value, (char*)res_data.data() + offset);
+ } else {
+ return Impl::date_to_str(dt, (char*)res_data.data() + offset);
+ }
+ }();
offset += len;
return false;
@@ -393,6 +480,24 @@ struct DateFormatImpl {
}
};
+inline bool contains_from_unixtime_format_specifier(StringRef format, char specifier) {
+ for (size_t i = 0; i + 1 < format.size; ++i) {
+ // Consume the character after every '%', so "%%n" is escaped text rather than %n.
+ if (format.data[i] == '%' && format.data[++i] == specifier) {
+ return true;
+ }
+ }
+ return false;
+}
+
+inline bool has_conflicting_from_unixtime_fraction_specifiers(StringRef format) {
+ // %f formats the microsecond-rounded instant, while %n formats the original nanosecond
+ // instant. For example, 0.999999500 becomes 1.000000 for %f but stays 0.999999500 for %n,
+ // so one result cannot contain both fields consistently.
+ return contains_from_unixtime_format_specifier(format, 'f') &&
+ contains_from_unixtime_format_specifier(format, 'n');
+}
+
template <bool WithStringArg, bool NewVersion = false>
struct FromUnixTimeImpl {
using ArgType = Int64;
@@ -432,7 +537,7 @@ struct FromUnixTimeImpl {
template <typename Impl>
static bool execute(const ArgType& val, StringRef format, ColumnString::Chars& res_data,
size_t& offset, const cctz::time_zone& time_zone) {
- if (!check_valid(val)) {
+ if (!check_valid(val) || has_conflicting_from_unixtime_fraction_specifiers(format)) {
return true;
}
DateV2Value<DateTimeV2ValueType> dt = get_datetime_value(val, time_zone);
@@ -442,7 +547,7 @@ struct FromUnixTimeImpl {
if constexpr (std::is_same_v<Impl, time_format_type::UserDefinedImpl>) {
char buf[100 + SAFE_FORMAT_STRING_MARGIN];
if (!dt.to_format_string_conservative(format.data, format.size, buf,
- 100 + SAFE_FORMAT_STRING_MARGIN)) {
+ 100 + SAFE_FORMAT_STRING_MARGIN, 0)) {
return true;
}
@@ -460,14 +565,18 @@ struct FromUnixTimeImpl {
};
// only new verison
-template <bool WithStringArg>
+template <bool WithStringArg, bool WithNanosecond = false>
struct FromUnixTimeDecimalImpl {
- using ArgType = Int64;
- static constexpr PrimitiveType FromPType = TYPE_DECIMAL64;
- constexpr static short Scale = 6; // same with argument's scale in FE's signature
+ static_assert(!WithNanosecond || WithStringArg);
+ using ArgType = std::conditional_t<WithNanosecond, Int128, Int64>;
+ static constexpr PrimitiveType FromPType = WithNanosecond ? TYPE_DECIMAL128I : TYPE_DECIMAL64;
+ constexpr static short Scale = WithNanosecond ? 9 : 6;
static DataTypes get_variadic_argument_types() {
if constexpr (WithStringArg) {
+ if constexpr (WithNanosecond) {
+ return {std::make_shared<DataTypeDecimal128>(), std::make_shared<DataTypeString>()};
+ }
return {std::make_shared<DataTypeDecimal64>(), std::make_shared<DataTypeString>()};
} else {
return {std::make_shared<DataTypeDecimal64>()};
@@ -482,12 +591,31 @@ struct FromUnixTimeDecimalImpl {
return true;
}
+ static int32_t get_nanosecond(const ArgType& fraction) {
+ return static_cast<int32_t>(fraction) * common::exp10_i32(9 - Scale);
+ }
+
static DateV2Value<DateTimeV2ValueType> get_datetime_value(const ArgType& interger,
const ArgType& fraction,
- const cctz::time_zone& time_zone) {
+ const cctz::time_zone& time_zone,
+ bool preserve_nanosecond) {
+ auto epoch_second = static_cast<int64_t>(interger);
+ int32_t nanosecond = get_nanosecond(fraction);
+ if constexpr (WithNanosecond) {
+ if (!preserve_nanosecond) {
+ // Keep the established %f behavior by rounding the complete instant, including
+ // carry: 0.999999500 seconds is formatted as 01.000000, not 00.000000.
+ constexpr int32_t MICROS_PER_SECOND = 1000000;
+ const int32_t rounded_microseconds =
+ (nanosecond + TimeStampNsValue::NANOS_PER_MICROSECOND / 2) /
+ TimeStampNsValue::NANOS_PER_MICROSECOND;
+ epoch_second += rounded_microseconds / MICROS_PER_SECOND;
+ nanosecond = rounded_microseconds % MICROS_PER_SECOND *
+ TimeStampNsValue::NANOS_PER_MICROSECOND;
+ }
+ }
DateV2Value<DateTimeV2ValueType> dt;
- // 9 is nanoseconds, our input's scale is 6
- dt.from_unixtime(interger, (int32_t)fraction * common::exp10_i32(9 - Scale), time_zone, 6);
+ dt.from_unixtime(epoch_second, nanosecond, time_zone, 6);
return dt;
}
@@ -496,17 +624,29 @@ struct FromUnixTimeDecimalImpl {
static bool execute_decimal(const ArgType& interger, const ArgType& fraction, StringRef format,
ColumnString::Chars& res_data, size_t& offset,
const cctz::time_zone& time_zone) {
- if (!check_valid(interger + (fraction > 0 ? 1 : ((fraction < 0) ? -1 : 0)))) [[unlikely]] {
+ if (has_conflicting_from_unixtime_fraction_specifiers(format)) [[unlikely]] {
+ return true;
+ }
+ int64_t fraction_adjustment = 0;
+ if (fraction > 0) {
+ fraction_adjustment = 1;
+ } else if (fraction < 0) {
+ fraction_adjustment = -1;
+ }
+ if (!check_valid(interger + fraction_adjustment)) [[unlikely]] {
return true;
}
- DateV2Value<DateTimeV2ValueType> dt = get_datetime_value(interger, fraction, time_zone);
+ const bool preserve_nanosecond = contains_from_unixtime_format_specifier(format, 'n');
+ DateV2Value<DateTimeV2ValueType> dt =
+ get_datetime_value(interger, fraction, time_zone, preserve_nanosecond);
if (!dt.is_valid_date()) [[unlikely]] {
return true;
}
if constexpr (std::is_same_v<Impl, time_format_type::UserDefinedImpl>) {
char buf[100 + SAFE_FORMAT_STRING_MARGIN];
if (!dt.to_format_string_conservative(format.data, format.size, buf,
- 100 + SAFE_FORMAT_STRING_MARGIN)) {
+ 100 + SAFE_FORMAT_STRING_MARGIN,
+ get_nanosecond(fraction))) {
return true;
}
@@ -569,8 +709,22 @@ public:
TimeValue::TimeType time = get_time_value(datetime_val);
char buf[100 + SAFE_FORMAT_STRING_MARGIN];
- if (!TimeValue::to_format_string_conservative(format.data, format.size, buf,
- 100 + SAFE_FORMAT_STRING_MARGIN, time)) {
+ const bool formatted = [&]() {
+ if constexpr (ArgPType == PrimitiveType::TYPE_TIMESTAMP_NS) {
+ return TimeValue::to_format_string_conservative(
+ format.data, format.size, buf, 100 + SAFE_FORMAT_STRING_MARGIN, time,
+ datetime_val.nanosecond());
+ } else if constexpr (ArgPType == PrimitiveType::TYPE_DATETIMEV2 ||
+ ArgPType == PrimitiveType::TYPE_TIMEV2) {
+ return TimeValue::to_format_string_conservative(
+ format.data, format.size, buf, 100 + SAFE_FORMAT_STRING_MARGIN, time,
+ TimeValue::microsecond(time) * 1000);
+ } else {
+ return TimeValue::to_format_string_conservative(
+ format.data, format.size, buf, 100 + SAFE_FORMAT_STRING_MARGIN, time);
+ }
+ }();
+ if (!formatted) {
null_map_data[i] = 1;
res_offsets.push_back(res_chars.size());
continue;
@@ -587,6 +741,8 @@ private:
TimeValue::TimeType get_time_value(const ArgCppType& datetime_val) const {
if constexpr (ArgPType == PrimitiveType::TYPE_TIMEV2) {
return static_cast<TimeValue::TimeType>(datetime_val);
+ } else if constexpr (ArgPType == PrimitiveType::TYPE_TIMESTAMP_NS) {
+ return static_cast<TimeValue::TimeType>(datetime_val.time_part_to_microsecond());
} else {
return TimeValue::make_time(datetime_val.hour(), datetime_val.minute(),
datetime_val.second(), datetime_val.microsecond());
diff --git a/be/src/exprs/function/dictionary.h b/be/src/exprs/function/dictionary.h
index 3c7a934dbb5..f40d0ad040c 100644
--- a/be/src/exprs/function/dictionary.h
+++ b/be/src/exprs/function/dictionary.h
@@ -33,6 +33,7 @@
#include "core/data_type/data_type_ipv6.h"
#include "core/data_type/data_type_number.h"
#include "core/data_type/data_type_string.h"
+#include "core/data_type/data_type_timestamp_ns.h"
#include "core/types.h"
#include "exprs/function/cast_type_to_either.h"
@@ -114,9 +115,9 @@ public:
return cast_type_to_either<DataTypeUInt8, DataTypeInt8, DataTypeInt16, DataTypeInt32,
DataTypeInt64, DataTypeInt128, DataTypeFloat32, DataTypeFloat64,
DataTypeIPv4, DataTypeIPv6, DataTypeString, DataTypeDateV2,
- DataTypeDateTimeV2, DataTypeDecimal32, DataTypeDecimal64,
- DataTypeDecimal128, DataTypeDecimal256>(type,
- std::forward<F>(f));
+ DataTypeDateTimeV2, DataTypeTimeStampNs, DataTypeDecimal32,
+ DataTypeDecimal64, DataTypeDecimal128, DataTypeDecimal256>(
+ type, std::forward<F>(f));
}
virtual size_t allocated_bytes() const;
@@ -190,6 +191,7 @@ protected:
ColumnWithType<DataTypeString>, ColumnWithType<DictDataTypeString64>,
ColumnWithType<DataTypeDateV2>, ColumnWithType<DataTypeDateTimeV2>,
+ ColumnWithType<DataTypeTimeStampNs>,
ColumnWithType<DataTypeDecimal32>, ColumnWithType<DataTypeDecimal64>,
ColumnWithType<DataTypeDecimal128>, ColumnWithType<DataTypeDecimal256>>;
diff --git a/be/src/exprs/function/function.cpp b/be/src/exprs/function/function.cpp
index a1834bb3e90..95cbe3f5f84 100644
--- a/be/src/exprs/function/function.cpp
+++ b/be/src/exprs/function/function.cpp
@@ -362,6 +362,7 @@ bool contains_date_or_datetime_or_decimal(const DataTypePtr& type) {
// For scalar types, check if it's date/datetime/decimal
return is_date_or_datetime(type_ptr->get_primitive_type()) ||
is_date_v2_or_datetime_v2(type_ptr->get_primitive_type()) ||
+ is_timestamp_ns_type(type_ptr->get_primitive_type()) ||
is_decimal(type_ptr->get_primitive_type()) ||
is_time_type(type_ptr->get_primitive_type());
}
@@ -377,6 +378,12 @@ bool FunctionBuilderImpl::is_nested_type_date_or_datetime_or_decimal(
func_return_type->is_nullable()
? ((DataTypeNullable*)func_return_type.get())->get_nested_type()
: func_return_type;
+ // A mismatched parent (for example, a STRUCT with one DATETIMEV2 scale difference) can still
+ // contain identical TIMESTAMP_NS or ordinary scalar children. Accept those exact subtrees
+ // before applying the historical date/decimal family compatibility rules below.
+ if (return_type_ptr->equals(*func_return_type_ptr)) {
+ return true;
+ }
// make sure that map/struct/array also need to check
if (return_type_ptr->get_primitive_type() != func_return_type_ptr->get_primitive_type()) {
return false;
diff --git a/be/src/exprs/function/function.h b/be/src/exprs/function/function.h
index e7ae33b34eb..d14f7701425 100644
--- a/be/src/exprs/function/function.h
+++ b/be/src/exprs/function/function.h
@@ -37,7 +37,6 @@
#include "core/block/columns_with_type_and_name.h"
#include "core/data_type/data_type.h"
#include "core/data_type/data_type_array.h"
-#include "core/data_type/data_type_map.h"
#include "core/data_type/data_type_nullable.h"
#include "core/data_type/data_type_struct.h"
#include "core/data_type/define_primitive_type.h"
@@ -65,7 +64,8 @@ struct FunctionAttr {
for (auto it : arguments) { \
is_nullable = is_nullable || it.type->is_nullable(); \
is_datev2 = is_datev2 || it.type->get_primitive_type() == TYPE_DATEV2 || \
- it.type->get_primitive_type() == TYPE_DATETIMEV2; \
+ it.type->get_primitive_type() == TYPE_DATETIMEV2 || \
+ it.type->get_primitive_type() == TYPE_TIMESTAMP_NS; \
} \
return is_nullable || !is_datev2 \
? make_nullable( \
diff --git a/be/src/exprs/function/function_convert_tz.cpp b/be/src/exprs/function/function_convert_tz.cpp
index f098ad87ac8..79ccd0b381c 100644
--- a/be/src/exprs/function/function_convert_tz.cpp
+++ b/be/src/exprs/function/function_convert_tz.cpp
@@ -21,6 +21,7 @@
#include <cstdint>
#include <memory>
#include <string>
+#include <tuple>
#include <utility>
#include "common/status.h"
@@ -40,6 +41,7 @@
#include "core/data_type/data_type_date_time.h"
#include "core/data_type/data_type_nullable.h"
#include "core/data_type/data_type_string.h"
+#include "core/data_type/data_type_timestamp_ns.h"
#include "core/data_type/define_primitive_type.h"
#include "core/data_type/primitive_type.h"
#include "core/string_ref.h"
@@ -64,21 +66,34 @@ struct ConvertTzState {
cctz::time_zone to_tz;
};
+template <PrimitiveType Type>
class FunctionConvertTZ : public IFunction {
- constexpr static PrimitiveType PType = PrimitiveType::TYPE_DATETIMEV2;
+ constexpr static PrimitiveType PType = Type;
using DateValueType = PrimitiveTypeTraits<PType>::CppType;
using ColumnType = PrimitiveTypeTraits<PType>::ColumnType;
public:
static constexpr auto name = "convert_tz";
- static FunctionPtr create() { return std::make_shared<FunctionConvertTZ>(); }
+ static FunctionPtr create() { return std::make_shared<FunctionConvertTZ<PType>>(); }
String get_name() const override { return name; }
size_t get_number_of_arguments() const override { return 3; }
+ DataTypes get_variadic_argument_types_impl() const override {
+ if constexpr (PType == TYPE_TIMESTAMP_NS) {
+ return {std::make_shared<DataTypeTimeStampNs>(), std::make_shared<DataTypeString>(),
+ std::make_shared<DataTypeString>()};
+ }
+ return {};
+ }
+
DataTypePtr get_return_type_impl(const DataTypes& arguments) const override {
+ if constexpr (PType == TYPE_TIMESTAMP_NS) {
+ return have_nullable(arguments) ? make_nullable(std::make_shared<DataTypeTimeStampNs>())
+ : std::make_shared<DataTypeTimeStampNs>();
+ }
return have_nullable(arguments) ? make_nullable(std::make_shared<DataTypeDateTimeV2>())
: std::make_shared<DataTypeDateTimeV2>();
}
@@ -213,17 +228,41 @@ private:
}
}
- static std::pair<int64_t, int64_t> unix_timestamp_for_convert_tz(
+ static std::tuple<int64_t, int64_t, bool> unix_timestamp_for_convert_tz(
const DateValueType& ts_value, const cctz::time_zone& from_tz) {
- cctz::civil_second civil_time(ts_value.year(), ts_value.month(), ts_value.day(),
- ts_value.hour(), ts_value.minute(), ts_value.second());
+ const auto civil_value = [&]() {
+ if constexpr (PType == TYPE_TIMESTAMP_NS) {
+ return ts_value.to_datetime();
+ } else {
+ return ts_value;
+ }
+ }();
+ cctz::civil_second civil_time(civil_value.year(), civil_value.month(), civil_value.day(),
+ civil_value.hour(), civil_value.minute(),
+ civil_value.second());
const auto lookup = from_tz.lookup(civil_time);
const bool skipped = lookup.kind == cctz::time_zone::civil_lookup::SKIPPED;
const auto tp = skipped ? lookup.trans : lookup.pre;
// Skipped civil times map to the transition instant. Do not keep the
// input fractional part inside a local time interval that never existed.
- return {tp.time_since_epoch().count(), skipped ? 0 : ts_value.microsecond()};
+ return {tp.time_since_epoch().count(), skipped ? 0 : civil_value.microsecond(), skipped};
+ }
+
+ static bool convert_value(const DateValueType& source, const cctz::time_zone& from_tz,
+ const cctz::time_zone& to_tz, DateValueType* result) {
+ const auto [seconds, microseconds, skipped] =
+ unix_timestamp_for_convert_tz(source, from_tz);
+ const std::pair<int64_t, int64_t> timestamp {seconds, microseconds};
+ if constexpr (PType == TYPE_TIMESTAMP_NS) {
+ DateV2Value<DateTimeV2ValueType> converted;
+ converted.from_unixtime(timestamp, to_tz);
+ const uint16_t remainder = skipped ? 0 : source.nanosecond_remainder();
+ return result->from_datetime(converted, remainder);
+ } else {
+ result->from_unixtime(timestamp, to_tz);
+ return result->is_valid_date();
+ }
}
static void execute_tz_const_with_state(ConvertTzState* convert_tz_state,
@@ -253,14 +292,12 @@ private:
DateValueType ts_value = date_column->get_element(i);
DateValueType ts_value2;
- ts_value2.from_unixtime(unix_timestamp_for_convert_tz(ts_value, from_tz), to_tz);
-
- if (!ts_value2.is_valid_date()) [[unlikely]] {
+ if (!convert_value(ts_value, from_tz, to_tz, &ts_value2)) [[unlikely]] {
throw_out_of_bound_convert_tz<DateValueType>(date_column->get_element(i),
from_tz.name(), to_tz.name());
}
- result_column->insert(Field::create_field<TYPE_DATETIMEV2>(ts_value2));
+ result_column->insert(Field::create_field<PType>(ts_value2));
}
}
@@ -304,19 +341,18 @@ private:
to_tz_name);
}
- ts_value2.from_unixtime(unix_timestamp_for_convert_tz(ts_value, from_tz), to_tz);
-
- if (!ts_value2.is_valid_date()) [[unlikely]] {
+ if (!convert_value(ts_value, from_tz, to_tz, &ts_value2)) [[unlikely]] {
throw_out_of_bound_convert_tz<DateValueType>(date_column->get_element(index_now),
from_tz.name(), to_tz.name());
}
- result_column->insert(Field::create_field<TYPE_DATETIMEV2>(ts_value2));
+ result_column->insert(Field::create_field<PType>(ts_value2));
}
};
void register_function_convert_tz(SimpleFunctionFactory& factory) {
- factory.register_function<FunctionConvertTZ>();
+ factory.register_function<FunctionConvertTZ<TYPE_DATETIMEV2>>();
+ factory.register_function<FunctionConvertTZ<TYPE_TIMESTAMP_NS>>();
}
} // namespace doris
diff --git a/be/src/exprs/function/function_date_or_datetime_computation.cpp b/be/src/exprs/function/function_date_or_datetime_computation.cpp
index 8973cab77e3..1dfb73bbf8b 100644
--- a/be/src/exprs/function/function_date_or_datetime_computation.cpp
+++ b/be/src/exprs/function/function_date_or_datetime_computation.cpp
@@ -88,6 +88,35 @@ using FunctionDatetimeAddWeeks = FunctionDateOrDateTimeComputation<AddWeeksImpl<
using FunctionDatetimeAddMonths = FunctionDateOrDateTimeComputation<AddMonthsImpl<TYPE_DATETIMEV2>>;
using FunctionDatetimeAddYears = FunctionDateOrDateTimeComputation<AddYearsImpl<TYPE_DATETIMEV2>>;
+#define TIMESTAMP_NS_COMPUTATION_ALIAS(NAME, IMPL) \
+ using FunctionTimeStampNs##NAME = FunctionDateOrDateTimeComputation<IMPL<TYPE_TIMESTAMP_NS>>
+
+TIMESTAMP_NS_COMPUTATION_ALIAS(AddMicroseconds, AddMicrosecondsImpl);
+using FunctionTimeStampNsAddNanoseconds = FunctionDateOrDateTimeComputation<AddNanosecondsImpl>;
+TIMESTAMP_NS_COMPUTATION_ALIAS(AddMilliseconds, AddMillisecondsImpl);
+TIMESTAMP_NS_COMPUTATION_ALIAS(AddSeconds, AddSecondsImpl);
+TIMESTAMP_NS_COMPUTATION_ALIAS(AddMinutes, AddMinutesImpl);
+TIMESTAMP_NS_COMPUTATION_ALIAS(AddHours, AddHoursImpl);
+TIMESTAMP_NS_COMPUTATION_ALIAS(AddDays, AddDaysImpl);
+TIMESTAMP_NS_COMPUTATION_ALIAS(AddWeeks, AddWeeksImpl);
+TIMESTAMP_NS_COMPUTATION_ALIAS(AddMonths, AddMonthsImpl);
+TIMESTAMP_NS_COMPUTATION_ALIAS(AddYears, AddYearsImpl);
+TIMESTAMP_NS_COMPUTATION_ALIAS(AddQuarters, AddQuartersImpl);
+TIMESTAMP_NS_COMPUTATION_ALIAS(SubMicroseconds, SubtractMicrosecondsImpl);
+using FunctionTimeStampNsSubNanoseconds =
+ FunctionDateOrDateTimeComputation<SubtractNanosecondsImpl>;
+TIMESTAMP_NS_COMPUTATION_ALIAS(SubMilliseconds, SubtractMillisecondsImpl);
+TIMESTAMP_NS_COMPUTATION_ALIAS(SubSeconds, SubtractSecondsImpl);
+TIMESTAMP_NS_COMPUTATION_ALIAS(SubMinutes, SubtractMinutesImpl);
+TIMESTAMP_NS_COMPUTATION_ALIAS(SubHours, SubtractHoursImpl);
+TIMESTAMP_NS_COMPUTATION_ALIAS(SubDays, SubtractDaysImpl);
+TIMESTAMP_NS_COMPUTATION_ALIAS(SubWeeks, SubtractWeeksImpl);
+TIMESTAMP_NS_COMPUTATION_ALIAS(SubMonths, SubtractMonthsImpl);
+TIMESTAMP_NS_COMPUTATION_ALIAS(SubQuarters, SubtractQuartersImpl);
+TIMESTAMP_NS_COMPUTATION_ALIAS(SubYears, SubtractYearsImpl);
+TIMESTAMP_NS_COMPUTATION_ALIAS(ToYearWeekTwoArgs, ToYearWeekTwoArgsImpl);
+TIMESTAMP_NS_COMPUTATION_ALIAS(ToWeekTwoArgs, ToWeekTwoArgsImpl);
+
using FunctionTimestamptzAddMicroseconds =
FunctionDateOrDateTimeComputation<AddMicrosecondsImpl<TYPE_TIMESTAMPTZ>>;
using FunctionTimestamptzAddMilliseconds =
@@ -105,14 +134,18 @@ using FunctionTimestamptzAddMonths =
FunctionDateOrDateTimeComputation<AddMonthsImpl<TYPE_TIMESTAMPTZ>>;
using FunctionTimestamptzAddYears =
FunctionDateOrDateTimeComputation<AddYearsImpl<TYPE_TIMESTAMPTZ>>;
-#define FUNCTION_TIME_UNION_CAL(TYPE) \
- using FunctionDatetimeAdd##TYPE = \
- FunctionDateOrDateTimeComputation<Add##TYPE##Impl<TYPE_DATETIMEV2>>; \
- using FunctionDatetimeSub##TYPE = \
- FunctionDateOrDateTimeComputation<Subtract##TYPE##Impl<TYPE_DATETIMEV2>>; \
- using FunctionTimestamptzAdd##TYPE = \
- FunctionDateOrDateTimeComputation<Add##TYPE##Impl<TYPE_TIMESTAMPTZ>>; \
- using FunctionTimestamptzSub##TYPE = \
+#define FUNCTION_TIME_UNION_CAL(TYPE) \
+ using FunctionDatetimeAdd##TYPE = \
+ FunctionDateOrDateTimeComputation<Add##TYPE##Impl<TYPE_DATETIMEV2>>; \
+ using FunctionDatetimeSub##TYPE = \
+ FunctionDateOrDateTimeComputation<Subtract##TYPE##Impl<TYPE_DATETIMEV2>>; \
+ using FunctionTimeStampNsAdd##TYPE = \
+ FunctionDateOrDateTimeComputation<Add##TYPE##Impl<TYPE_TIMESTAMP_NS>>; \
+ using FunctionTimeStampNsSub##TYPE = \
+ FunctionDateOrDateTimeComputation<Subtract##TYPE##Impl<TYPE_TIMESTAMP_NS>>; \
+ using FunctionTimestamptzAdd##TYPE = \
+ FunctionDateOrDateTimeComputation<Add##TYPE##Impl<TYPE_TIMESTAMPTZ>>; \
+ using FunctionTimestamptzSub##TYPE = \
FunctionDateOrDateTimeComputation<Subtract##TYPE##Impl<TYPE_TIMESTAMPTZ>>;
FUNCTION_TIME_UNION_CAL(SecondMicrosecond);
@@ -174,19 +207,24 @@ using FunctionTimestamptzSubYears =
FunctionDateOrDateTimeComputation<SubtractYearsImpl<TYPE_TIMESTAMPTZ>>;
using FunctionAddTimeDatetime = FunctionNeedsToHandleNull<AddTimeDatetimeImpl, TYPE_DATETIMEV2>;
+using FunctionAddTimeTimeStampNs =
+ FunctionNeedsToHandleNull<AddTimeTimeStampNsImpl, TYPE_TIMESTAMP_NS>;
using FunctionAddTimeTime = FunctionNeedsToHandleNull<AddTimeTimeImpl, TYPE_TIMEV2>;
using FunctionAddTimeTimestampTz =
FunctionNeedsToHandleNull<AddTimeTimestamptzImpl, TYPE_TIMESTAMPTZ>;
using FunctionSubTimeDatetime = FunctionNeedsToHandleNull<SubTimeDatetimeImpl, TYPE_DATETIMEV2>;
+using FunctionSubTimeTimeStampNs =
+ FunctionNeedsToHandleNull<SubTimeTimeStampNsImpl, TYPE_TIMESTAMP_NS>;
using FunctionSubTimeTime = FunctionNeedsToHandleNull<SubTimeTimeImpl, TYPE_TIMEV2>;
using FunctionSubTimeTimestampTz =
FunctionNeedsToHandleNull<SubTimeTimestamptzImpl, TYPE_TIMESTAMPTZ>;
#define FUNCTION_TIME_DIFF(NAME, IMPL, TYPE) using NAME##_##TYPE = FunctionTimeDiff<IMPL<TYPE>>;
-#define ALL_FUNCTION_TIME_DIFF(NAME, IMPL) \
- FUNCTION_TIME_DIFF(NAME, IMPL, TYPE_DATETIMEV2) \
- FUNCTION_TIME_DIFF(NAME, IMPL, TYPE_DATEV2) \
+#define ALL_FUNCTION_TIME_DIFF(NAME, IMPL) \
+ FUNCTION_TIME_DIFF(NAME, IMPL, TYPE_DATETIMEV2) \
+ FUNCTION_TIME_DIFF(NAME, IMPL, TYPE_TIMESTAMP_NS) \
+ FUNCTION_TIME_DIFF(NAME, IMPL, TYPE_DATEV2) \
FUNCTION_TIME_DIFF(NAME, IMPL, TYPE_TIMESTAMPTZ)
// these diff functions accept all v2 types. but for v1 only datetime.
ALL_FUNCTION_TIME_DIFF(FunctionDatetimeDateDiff, DateDiffImpl)
@@ -201,12 +239,41 @@ ALL_FUNCTION_TIME_DIFF(FunctionDatetimeSecondsDiff, SecondsDiffImpl)
ALL_FUNCTION_TIME_DIFF(FunctionDatetimeDaysDiff, DaysDiffImpl)
ALL_FUNCTION_TIME_DIFF(FunctionDatetimeMilliSecondsDiff, MilliSecondsDiffImpl)
ALL_FUNCTION_TIME_DIFF(FunctionDatetimeMicroSecondsDiff, MicroSecondsDiffImpl)
+using FunctionTimeStampNsNanoSecondsDiff = FunctionTimeDiff<NanoSecondsDiffImpl>;
+
+#define MIXED_FUNCTION_TIME_DIFF(NAME, IMPL) \
+ using NAME##TimeStampNsDateTimeV2 = \
+ FunctionTimeDiff<MixedDateTimeFunctionImpl<TYPE_TIMESTAMP_NS, TYPE_DATETIMEV2, IMPL>>; \
+ using NAME##DateTimeV2TimeStampNs = \
+ FunctionTimeDiff<MixedDateTimeFunctionImpl<TYPE_DATETIMEV2, TYPE_TIMESTAMP_NS, IMPL>>;
+
+MIXED_FUNCTION_TIME_DIFF(FunctionDateDiff, DateDiffImpl)
+using FunctionTimeDiffTimeStampNsDateTimeV2 =
+ FunctionTimeDiff<MixedTimeDiffImpl<TYPE_TIMESTAMP_NS, TYPE_DATETIMEV2>>;
+using FunctionTimeDiffDateTimeV2TimeStampNs =
+ FunctionTimeDiff<MixedTimeDiffImpl<TYPE_DATETIMEV2, TYPE_TIMESTAMP_NS>>;
+MIXED_FUNCTION_TIME_DIFF(FunctionYearsDiff, YearsDiffImpl)
+MIXED_FUNCTION_TIME_DIFF(FunctionQuartersDiff, QuartersDiffImpl)
+MIXED_FUNCTION_TIME_DIFF(FunctionMonthsDiff, MonthsDiffImpl)
+MIXED_FUNCTION_TIME_DIFF(FunctionWeeksDiff, WeeksDiffImpl)
+MIXED_FUNCTION_TIME_DIFF(FunctionHoursDiff, HoursDiffImpl)
+MIXED_FUNCTION_TIME_DIFF(FunctionMinutesDiff, MintuesDiffImpl)
+MIXED_FUNCTION_TIME_DIFF(FunctionSecondsDiff, SecondsDiffImpl)
+MIXED_FUNCTION_TIME_DIFF(FunctionDaysDiff, DaysDiffImpl)
+MIXED_FUNCTION_TIME_DIFF(FunctionMilliSecondsDiff, MilliSecondsDiffImpl)
+MIXED_FUNCTION_TIME_DIFF(FunctionMicroSecondsDiff, MicroSecondsDiffImpl)
+using FunctionNanoSecondsDiffTimeStampNsDateTimeV2 =
+ FunctionTimeDiff<MixedNanoSecondsDiffImpl<TYPE_TIMESTAMP_NS, TYPE_DATETIMEV2>>;
+using FunctionNanoSecondsDiffDateTimeV2TimeStampNs =
+ FunctionTimeDiff<MixedNanoSecondsDiffImpl<TYPE_DATETIMEV2, TYPE_TIMESTAMP_NS>>;
using FunctionDatetimeToYearWeekTwoArgs =
FunctionDateOrDateTimeComputation<ToYearWeekTwoArgsImpl<TYPE_DATETIMEV2>>;
using FunctionDatetimeToWeekTwoArgs =
FunctionDateOrDateTimeComputation<ToWeekTwoArgsImpl<TYPE_DATETIMEV2>>;
+// Date/time functions are deliberately registered together to keep the factory surface auditable.
+// NOLINTNEXTLINE(readability-function-size)
void register_function_date_time_computation(SimpleFunctionFactory& factory) {
factory.register_function<FunctionNextDay>();
factory.register_function<FunctionPreviousDay>();
@@ -257,6 +324,17 @@ void register_function_date_time_computation(SimpleFunctionFactory& factory) {
factory.register_function<FunctionDatetimeAddMonths>();
factory.register_function<FunctionDatetimeAddYears>();
factory.register_function<FunctionDatetimeAddQuarters>();
+ factory.register_function<FunctionTimeStampNsAddMicroseconds>();
+ factory.register_function<FunctionTimeStampNsAddNanoseconds>();
+ factory.register_function<FunctionTimeStampNsAddMilliseconds>();
+ factory.register_function<FunctionTimeStampNsAddSeconds>();
+ factory.register_function<FunctionTimeStampNsAddMinutes>();
+ factory.register_function<FunctionTimeStampNsAddHours>();
+ factory.register_function<FunctionTimeStampNsAddDays>();
+ factory.register_function<FunctionTimeStampNsAddWeeks>();
+ factory.register_function<FunctionTimeStampNsAddMonths>();
+ factory.register_function<FunctionTimeStampNsAddYears>();
+ factory.register_function<FunctionTimeStampNsAddQuarters>();
factory.register_function<FunctionTimestamptzAddMicroseconds>();
factory.register_function<FunctionTimestamptzAddMilliseconds>();
factory.register_function<FunctionTimestamptzAddSeconds>();
@@ -271,6 +349,8 @@ void register_function_date_time_computation(SimpleFunctionFactory& factory) {
#define REGISTER_TIME_UNION_CAL(TYPE) \
factory.register_function<FunctionDatetimeAdd##TYPE>(); \
factory.register_function<FunctionDatetimeSub##TYPE>(); \
+ factory.register_function<FunctionTimeStampNsAdd##TYPE>(); \
+ factory.register_function<FunctionTimeStampNsSub##TYPE>(); \
factory.register_function<FunctionTimestamptzAdd##TYPE>(); \
factory.register_function<FunctionTimestamptzSub##TYPE>();
@@ -302,6 +382,17 @@ void register_function_date_time_computation(SimpleFunctionFactory& factory) {
factory.register_function<FunctionDatetimeSubYears>();
factory.register_function<FunctionDatetimeSubQuarters>();
factory.register_function<FunctionDatetimeSubWeeks>();
+ factory.register_function<FunctionTimeStampNsSubMicroseconds>();
+ factory.register_function<FunctionTimeStampNsSubNanoseconds>();
+ factory.register_function<FunctionTimeStampNsSubMilliseconds>();
+ factory.register_function<FunctionTimeStampNsSubSeconds>();
+ factory.register_function<FunctionTimeStampNsSubMinutes>();
+ factory.register_function<FunctionTimeStampNsSubHours>();
+ factory.register_function<FunctionTimeStampNsSubDays>();
+ factory.register_function<FunctionTimeStampNsSubMonths>();
+ factory.register_function<FunctionTimeStampNsSubYears>();
+ factory.register_function<FunctionTimeStampNsSubQuarters>();
+ factory.register_function<FunctionTimeStampNsSubWeeks>();
factory.register_function<FunctionTimestamptzSubMicroseconds>();
factory.register_function<FunctionTimestamptzSubMilliseconds>();
@@ -315,17 +406,20 @@ void register_function_date_time_computation(SimpleFunctionFactory& factory) {
factory.register_function<FunctionTimestamptzSubWeeks>();
factory.register_function<FunctionAddTimeDatetime>();
+ factory.register_function<FunctionAddTimeTimeStampNs>();
factory.register_function<FunctionAddTimeTime>();
factory.register_function<FunctionAddTimeTimestampTz>();
factory.register_function<FunctionSubTimeDatetime>();
+ factory.register_function<FunctionSubTimeTimeStampNs>();
factory.register_function<FunctionSubTimeTime>();
factory.register_function<FunctionSubTimeTimestampTz>();
#define REGISTER_DATEV2_FUNCTIONS_DIFF(NAME, TYPE) factory.register_function<NAME##_##TYPE>();
-#define REGISTER_ALL_DATEV2_FUNCTIONS_DIFF(NAME) \
- REGISTER_DATEV2_FUNCTIONS_DIFF(NAME, TYPE_DATETIMEV2) \
- REGISTER_DATEV2_FUNCTIONS_DIFF(NAME, TYPE_DATEV2) \
+#define REGISTER_ALL_DATEV2_FUNCTIONS_DIFF(NAME) \
+ REGISTER_DATEV2_FUNCTIONS_DIFF(NAME, TYPE_DATETIMEV2) \
+ REGISTER_DATEV2_FUNCTIONS_DIFF(NAME, TYPE_TIMESTAMP_NS) \
+ REGISTER_DATEV2_FUNCTIONS_DIFF(NAME, TYPE_DATEV2) \
REGISTER_DATEV2_FUNCTIONS_DIFF(NAME, TYPE_TIMESTAMPTZ)
REGISTER_ALL_DATEV2_FUNCTIONS_DIFF(FunctionDatetimeDateDiff)
@@ -340,11 +434,31 @@ void register_function_date_time_computation(SimpleFunctionFactory& factory) {
REGISTER_ALL_DATEV2_FUNCTIONS_DIFF(FunctionDatetimeDaysDiff)
REGISTER_ALL_DATEV2_FUNCTIONS_DIFF(FunctionDatetimeMilliSecondsDiff)
REGISTER_ALL_DATEV2_FUNCTIONS_DIFF(FunctionDatetimeMicroSecondsDiff)
+ factory.register_function<FunctionTimeStampNsNanoSecondsDiff>();
+#define REGISTER_MIXED_FUNCTION_TIME_DIFF(NAME) \
+ factory.register_function<NAME##TimeStampNsDateTimeV2>(); \
+ factory.register_function<NAME##DateTimeV2TimeStampNs>();
+
+ REGISTER_MIXED_FUNCTION_TIME_DIFF(FunctionDateDiff)
+ REGISTER_MIXED_FUNCTION_TIME_DIFF(FunctionTimeDiff)
+ REGISTER_MIXED_FUNCTION_TIME_DIFF(FunctionYearsDiff)
+ REGISTER_MIXED_FUNCTION_TIME_DIFF(FunctionQuartersDiff)
+ REGISTER_MIXED_FUNCTION_TIME_DIFF(FunctionMonthsDiff)
+ REGISTER_MIXED_FUNCTION_TIME_DIFF(FunctionWeeksDiff)
+ REGISTER_MIXED_FUNCTION_TIME_DIFF(FunctionHoursDiff)
+ REGISTER_MIXED_FUNCTION_TIME_DIFF(FunctionMinutesDiff)
+ REGISTER_MIXED_FUNCTION_TIME_DIFF(FunctionSecondsDiff)
+ REGISTER_MIXED_FUNCTION_TIME_DIFF(FunctionDaysDiff)
+ REGISTER_MIXED_FUNCTION_TIME_DIFF(FunctionMilliSecondsDiff)
+ REGISTER_MIXED_FUNCTION_TIME_DIFF(FunctionMicroSecondsDiff)
+ REGISTER_MIXED_FUNCTION_TIME_DIFF(FunctionNanoSecondsDiff)
factory.register_function<FunctionToYearWeekTwoArgs>();
factory.register_function<FunctionToWeekTwoArgs>();
factory.register_function<FunctionDatetimeToYearWeekTwoArgs>();
factory.register_function<FunctionDatetimeToWeekTwoArgs>();
+ factory.register_function<FunctionTimeStampNsToYearWeekTwoArgs>();
+ factory.register_function<FunctionTimeStampNsToWeekTwoArgs>();
}
-} // namespace doris
\ No newline at end of file
+} // namespace doris
diff --git a/be/src/exprs/function/function_date_or_datetime_computation.h b/be/src/exprs/function/function_date_or_datetime_computation.h
index 00cdae8de13..87e5d930443 100644
--- a/be/src/exprs/function/function_date_or_datetime_computation.h
+++ b/be/src/exprs/function/function_date_or_datetime_computation.h
@@ -50,6 +50,7 @@
#include "core/data_type/data_type_nullable.h"
#include "core/data_type/data_type_number.h"
#include "core/data_type/data_type_time.h"
+#include "core/data_type/data_type_timestamp_ns.h"
#include "core/data_type/define_primitive_type.h"
#include "core/data_type/primitive_type.h"
#include "core/pod_array_fwd.h"
@@ -139,6 +140,47 @@ ADD_TIME_FUNCTION_IMPL(AddWeeksImpl, weeks_add, WEEK);
ADD_TIME_FUNCTION_IMPL(AddMonthsImpl, months_add, MONTH);
ADD_TIME_FUNCTION_IMPL(AddYearsImpl, years_add, YEAR);
+struct AddNanosecondsImpl {
+ static constexpr PrimitiveType ArgPType = TYPE_TIMESTAMP_NS;
+ static constexpr PrimitiveType ReturnType = TYPE_TIMESTAMP_NS;
+ static constexpr PrimitiveType IntervalPType = TYPE_BIGINT;
+ using InputValueType = typename PrimitiveTypeTraits<ArgPType>::DataType::FieldType;
+ using ReturnValueType = InputValueType;
+ using IntervalValueType = typename PrimitiveTypeTraits<IntervalPType>::DataType::FieldType;
+
+ static constexpr auto name = "nanoseconds_add";
+ static constexpr auto is_nullable = false;
+
+ static inline ReturnValueType execute(const InputValueType& timestamp,
+ IntervalValueType delta) {
+ int64_t result;
+ if (__builtin_add_overflow(timestamp.epoch_nanos(), delta, &result)) [[unlikely]] {
+ throw Exception(ErrorCode::OUT_OF_BOUND, "Operation {} of {}, {} out of range", name,
+ timestamp.to_string(), delta);
+ }
+ return TimeStampNsValue(result);
+ }
+
+ static DataTypes get_variadic_argument_types() {
+ return {std::make_shared<typename PrimitiveTypeTraits<ArgPType>::DataType>(),
+ std::make_shared<typename PrimitiveTypeTraits<IntervalPType>::DataType>()};
+ }
+};
+
+struct SubtractNanosecondsImpl : AddNanosecondsImpl {
+ static constexpr auto name = "nanoseconds_sub";
+
+ static inline ReturnValueType execute(const InputValueType& timestamp,
+ IntervalValueType delta) {
+ int64_t result;
+ if (__builtin_sub_overflow(timestamp.epoch_nanos(), delta, &result)) [[unlikely]] {
+ throw Exception(ErrorCode::OUT_OF_BOUND, "Operation {} of {}, {} out of range", name,
+ timestamp.to_string(), delta);
+ }
+ return TimeStampNsValue(result);
+ }
+};
+
template <TimeUnit UnionType, PrimitiveType PType>
class AddUnionTypeImpl {
static_assert(UnionType == SECOND_MICROSECOND || UnionType == MINUTE_MICROSECOND ||
@@ -195,11 +237,15 @@ private:
bool has_digit = false;
auto advance_to_digit = [&](const char*& p) {
- while (p < end && (*p < '0' || *p > '9')) ++p;
+ while (p < end && (*p < '0' || *p > '9')) {
+ ++p;
+ }
};
// Skip leading spaces
- while (cur < end && (*cur == ' ' || *cur == '\t')) ++cur;
+ while (cur < end && (*cur == ' ' || *cur == '\t')) {
+ ++cur;
+ }
// Check for negative sign
if (cur < end && *cur == '-') {
@@ -245,6 +291,8 @@ private:
return (cur != end) || !has_digit;
}
+ // Parsing all compound interval layouts together keeps their shared sign/fraction rules aligned.
+ // NOLINTNEXTLINE(readability-function-size)
static void get_interval_value(const IntervalValueType& delta, TimeInterval& interval) {
// the longest expr is 'DAYS HOURS:MINUTES:SECONDS.MICROSECONDS'
// which contains 5 parts
@@ -557,39 +605,68 @@ struct SubtractSecondMicrosecondImpl
static constexpr auto name = "second_microsecond_sub";
};
-#define DECLARE_DATE_FUNCTIONS(NAME, FN_NAME, RETURN_TYPE, STMT) \
- template <PrimitiveType DateType> \
- struct NAME { \
- using ValueType = typename PrimitiveTypeTraits<DateType>::DataType::FieldType; \
- using DateValueType = typename PrimitiveTypeTraits<DateType>::CppType; \
- static constexpr PrimitiveType ArgPType = DateType; \
- static constexpr PrimitiveType ReturnType = PrimitiveType::RETURN_TYPE; \
- \
- static constexpr auto name = #FN_NAME; \
- static constexpr auto is_nullable = false; \
- static inline typename PrimitiveTypeTraits<RETURN_TYPE>::DataType::FieldType execute( \
- const ValueType& t0, const ValueType& t1) { \
- const auto& ts0 = reinterpret_cast<const DateValueType&>(t0); \
- const auto& ts1 = reinterpret_cast<const DateValueType&>(t1); \
- return (STMT); \
- } \
- static DataTypes get_variadic_argument_types() { \
- return {std::make_shared<typename PrimitiveTypeTraits<DateType>::DataType>(), \
- std::make_shared<typename PrimitiveTypeTraits<DateType>::DataType>()}; \
- } \
+#define DECLARE_DATE_FUNCTIONS(NAME, FN_NAME, RETURN_TYPE, STMT) \
+ template <PrimitiveType DateType> \
+ struct NAME { \
+ using ValueType = typename PrimitiveTypeTraits<DateType>::DataType::FieldType; \
+ using DateValueType = typename PrimitiveTypeTraits<DateType>::CppType; \
+ static constexpr PrimitiveType ArgPType = DateType; \
+ static constexpr PrimitiveType IntervalPType = DateType; \
+ static constexpr PrimitiveType ReturnType = PrimitiveType::RETURN_TYPE; \
+ \
+ static constexpr auto name = #FN_NAME; \
+ static constexpr auto is_nullable = false; \
+ static inline typename PrimitiveTypeTraits<RETURN_TYPE>::DataType::FieldType execute( \
+ const ValueType& t0, const ValueType& t1) { \
+ return execute_mixed<DateType>(t0, t1); \
+ } \
+ template <PrimitiveType OtherType> \
+ static inline typename PrimitiveTypeTraits<RETURN_TYPE>::DataType::FieldType \
+ execute_mixed(const ValueType& t0, \
+ const typename PrimitiveTypeTraits<OtherType>::DataType::FieldType& t1) { \
+ using OtherValueType = typename PrimitiveTypeTraits<OtherType>::CppType; \
+ const auto& ts0 = reinterpret_cast<const DateValueType&>(t0); \
+ const auto& ts1 = reinterpret_cast<const OtherValueType&>(t1); \
+ return (STMT); \
+ } \
+ static DataTypes get_variadic_argument_types() { \
+ return {std::make_shared<typename PrimitiveTypeTraits<DateType>::DataType>(), \
+ std::make_shared<typename PrimitiveTypeTraits<DateType>::DataType>()}; \
+ } \
};
DECLARE_DATE_FUNCTIONS(DateDiffImpl, datediff, TYPE_INT, (ts0.daynr() - ts1.daynr()));
+
+template <PrimitiveType LeftType, PrimitiveType RightType,
+ template <PrimitiveType> typename Transform>
+struct MixedDateTimeFunctionImpl : Transform<LeftType> {
+ static constexpr PrimitiveType ArgPType = LeftType;
+ static constexpr PrimitiveType IntervalPType = RightType;
+
+ using LeftFieldType = typename PrimitiveTypeTraits<LeftType>::DataType::FieldType;
+ using RightFieldType = typename PrimitiveTypeTraits<RightType>::DataType::FieldType;
+
+ static inline auto execute(const LeftFieldType& left, const RightFieldType& right) {
+ return Transform<LeftType>::template execute_mixed<RightType>(left, right);
+ }
+
+ static DataTypes get_variadic_argument_types() {
+ return {std::make_shared<typename PrimitiveTypeTraits<LeftType>::DataType>(),
+ std::make_shared<typename PrimitiveTypeTraits<RightType>::DataType>()};
+ }
+};
// DECLARE_DATE_FUNCTIONS(TimeDiffImpl, timediff, DataTypeTime, ts0.datetime_diff_in_seconds(ts1));
// Expands to below here because it use Time type which need some special deal.
template <PrimitiveType DateType>
struct TimeDiffImpl {
static constexpr PrimitiveType ArgPType = DateType;
+ static constexpr PrimitiveType IntervalPType = DateType;
using ValueType = typename PrimitiveTypeTraits<DateType>::CppType;
using ArgType = typename PrimitiveTypeTraits<DateType>::DataType::FieldType;
//TODO: remove V1 since FE already removed it.
- static constexpr bool UsingTimev2 =
- is_date_v2_or_datetime_v2(DateType) || DateType == TYPE_TIMESTAMPTZ;
+ static constexpr bool UsingTimev2 = is_date_v2_or_datetime_v2(DateType) ||
+ DateType == TYPE_TIMESTAMP_NS ||
+ DateType == TYPE_TIMESTAMPTZ;
static constexpr PrimitiveType ReturnType = TYPE_TIMEV2;
static constexpr auto name = "timediff";
@@ -611,10 +688,35 @@ struct TimeDiffImpl {
}
static DataTypePtr get_return_type_impl(const ColumnsWithTypeAndName& arguments) {
+ const auto scale =
+ std::min(std::max(arguments[0].type->get_scale(), arguments[1].type->get_scale()),
+ DataTypeTimeV2::MAX_SCALE);
if (arguments[0].type->is_nullable() || arguments[1].type->is_nullable()) {
- return make_nullable(std::make_shared<DataTypeTimeV2>(arguments[0].type->get_scale()));
+ return make_nullable(std::make_shared<DataTypeTimeV2>(scale));
}
- return std::make_shared<DataTypeTimeV2>(arguments[0].type->get_scale());
+ return std::make_shared<DataTypeTimeV2>(scale);
+ }
+};
+
+template <PrimitiveType LeftType, PrimitiveType RightType>
+struct MixedTimeDiffImpl : TimeDiffImpl<LeftType> {
+ static constexpr PrimitiveType ArgPType = LeftType;
+ static constexpr PrimitiveType IntervalPType = RightType;
+ using LeftFieldType = typename PrimitiveTypeTraits<LeftType>::DataType::FieldType;
+ using RightFieldType = typename PrimitiveTypeTraits<RightType>::DataType::FieldType;
+ using LeftValueType = typename PrimitiveTypeTraits<LeftType>::CppType;
+ using RightValueType = typename PrimitiveTypeTraits<RightType>::CppType;
+
+ static inline DataTypeTimeV2::FieldType execute(const LeftFieldType& left,
+ const RightFieldType& right) {
+ const auto& left_value = reinterpret_cast<const LeftValueType&>(left);
+ const auto& right_value = reinterpret_cast<const RightValueType&>(right);
+ return TimeValue::limit_with_bound(left_value.datetime_diff_in_microseconds(right_value));
+ }
+
+ static DataTypes get_variadic_argument_types() {
+ return {std::make_shared<typename PrimitiveTypeTraits<LeftType>::DataType>(),
+ std::make_shared<typename PrimitiveTypeTraits<RightType>::DataType>()};
}
};
@@ -643,6 +745,86 @@ TIME_DIFF_FUNCTION_IMPL(SecondsDiffImpl, seconds_diff, SECOND);
TIME_DIFF_FUNCTION_IMPL(MilliSecondsDiffImpl, milliseconds_diff, MILLISECOND);
TIME_DIFF_FUNCTION_IMPL(MicroSecondsDiffImpl, microseconds_diff, MICROSECOND);
+struct NanoSecondsDiffImpl {
+ static constexpr PrimitiveType ArgPType = TYPE_TIMESTAMP_NS;
+ static constexpr PrimitiveType IntervalPType = TYPE_TIMESTAMP_NS;
+ static constexpr PrimitiveType ReturnType = TYPE_BIGINT;
+ using ValueType = typename PrimitiveTypeTraits<ArgPType>::DataType::FieldType;
+
+ static constexpr auto name = "nanoseconds_diff";
+ static constexpr auto is_nullable = false;
+
+ static inline int64_t execute(const ValueType& left, const ValueType& right) {
+ int64_t result;
+ if (__builtin_sub_overflow(left.epoch_nanos(), right.epoch_nanos(), &result)) [[unlikely]] {
+ throw Exception(ErrorCode::OUT_OF_BOUND, "Operation {} of {}, {} out of range", name,
+ left.to_string(), right.to_string());
+ }
+ return result;
+ }
+
+ static DataTypes get_variadic_argument_types() {
+ return {std::make_shared<typename PrimitiveTypeTraits<ArgPType>::DataType>(),
+ std::make_shared<typename PrimitiveTypeTraits<ArgPType>::DataType>()};
+ }
+};
+
+template <PrimitiveType LeftType, PrimitiveType RightType>
+struct MixedNanoSecondsDiffImpl : NanoSecondsDiffImpl {
+ static_assert(LeftType == TYPE_TIMESTAMP_NS || RightType == TYPE_TIMESTAMP_NS);
+ static constexpr PrimitiveType ArgPType = LeftType;
+ static constexpr PrimitiveType IntervalPType = RightType;
+ using LeftFieldType = typename PrimitiveTypeTraits<LeftType>::DataType::FieldType;
+ using RightFieldType = typename PrimitiveTypeTraits<RightType>::DataType::FieldType;
+ using LeftValueType = typename PrimitiveTypeTraits<LeftType>::CppType;
+ using RightValueType = typename PrimitiveTypeTraits<RightType>::CppType;
+
+ static inline int64_t execute(const LeftFieldType& left, const RightFieldType& right) {
+ const auto& left_value = reinterpret_cast<const LeftValueType&>(left);
+ const auto& right_value = reinterpret_cast<const RightValueType&>(right);
+ const auto left_datetime = [&] {
+ if constexpr (LeftType == TYPE_TIMESTAMP_NS) {
+ return left_value.to_datetime();
+ } else {
+ return left_value;
+ }
+ }();
+ const auto right_datetime = [&] {
+ if constexpr (RightType == TYPE_TIMESTAMP_NS) {
+ return right_value.to_datetime();
+ } else {
+ return right_value;
+ }
+ }();
+ const uint32_t left_nanosecond = [&] {
+ if constexpr (LeftType == TYPE_TIMESTAMP_NS) {
+ return left_value.nanosecond();
+ } else {
+ return left_value.microsecond() * 1000;
+ }
+ }();
+ const uint32_t right_nanosecond = [&] {
+ if constexpr (RightType == TYPE_TIMESTAMP_NS) {
+ return right_value.nanosecond();
+ } else {
+ return right_value.microsecond() * 1000;
+ }
+ }();
+ const __int128 difference = datetime_diff_in_nanoseconds(right_datetime, right_nanosecond,
+ left_datetime, left_nanosecond);
+ if (difference < std::numeric_limits<int64_t>::min() ||
+ difference > std::numeric_limits<int64_t>::max()) [[unlikely]] {
+ throw Exception(ErrorCode::OUT_OF_BOUND, "Operation {} out of range", name);
+ }
+ return static_cast<int64_t>(difference);
+ }
+
+ static DataTypes get_variadic_argument_types() {
+ return {std::make_shared<typename PrimitiveTypeTraits<LeftType>::DataType>(),
+ std::make_shared<typename PrimitiveTypeTraits<RightType>::DataType>()};
+ }
+};
+
#define TIME_FUNCTION_TWO_ARGS_IMPL(CLASS, NAME, FUNCTION, RETURN_TYPE) \
template <PrimitiveType DateType> \
struct CLASS { \
@@ -763,11 +945,15 @@ public:
RETURN_REAL_TYPE_FOR_DATEV2_FUNCTION(Transform::ReturnType);
}
+ // Vector/constant/null dispatch stays centralized for every date computation transform.
+ // NOLINTNEXTLINE(readability-function-size)
Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments,
uint32_t result, size_t input_rows_count) const override {
using ResFieldType =
typename PrimitiveTypeTraits<Transform::ReturnType>::DataType::FieldType;
- using Op = DateTimeOp<Transform::ArgPType, Transform::ArgPType, ResFieldType, Transform>;
+ using Op =
+ DateTimeOp<Transform::ArgPType, Transform::IntervalPType, ResFieldType, Transform>;
+ using RightColumnType = typename PrimitiveTypeTraits<Transform::IntervalPType>::ColumnType;
//ATTN: those null maps may be nullmap of ColumnConst(only 1 row)
// src column is always datelike type.
@@ -791,13 +977,11 @@ public:
if (const auto* nest_col1_const = check_and_get_column<ColumnConst>(*nest_col1)) {
rconst = true;
const auto& col1_inside_const =
- assert_cast<const ColumnVector<Transform::ArgPType>&>(
- nest_col1_const->get_data_column());
+ assert_cast<const RightColumnType&>(nest_col1_const->get_data_column());
Op::vector_constant(sources->get_data(), res_col->get_data(),
Op::get_element(col1_inside_const, 0), nullmap0, nullmap1);
} else { // vector-vector
- const auto& concrete_col1 =
- assert_cast<const ColumnVector<Transform::ArgPType>&>(*nest_col1);
+ const auto& concrete_col1 = assert_cast<const RightColumnType&>(*nest_col1);
Op::vector_vector(sources->get_data(), concrete_col1, res_col->get_data(), nullmap0,
nullmap1);
}
@@ -824,8 +1008,7 @@ public:
const auto& col0_inside_const = assert_cast<const ColumnVector<Transform::ArgPType>&>(
sources_const->get_data_column());
const ColumnPtr nested_col1 = remove_nullable(col1);
- const auto& concrete_col1 =
- assert_cast<const ColumnVector<Transform::ArgPType>&>(*nested_col1);
+ const auto& concrete_col1 = assert_cast<const RightColumnType&>(*nested_col1);
Op::constant_vector(col0_inside_const.get_data()[0], res_col->get_data(), concrete_col1,
nullmap0, nullmap1);
@@ -884,6 +1067,8 @@ public:
RETURN_REAL_TYPE_FOR_DATEV2_FUNCTION(Transform::ReturnType);
}
+ // Vector/constant/null dispatch stays centralized for every nullable date computation.
+ // NOLINTNEXTLINE(readability-function-size)
Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments,
uint32_t result, size_t input_rows_count) const override {
using ResFieldType =
@@ -1005,6 +1190,13 @@ public:
return {};
}
+ bool skip_return_type_check() const override {
+ if constexpr (requires { FunctionImpl::skip_return_type_check(); }) {
+ return FunctionImpl::skip_return_type_check();
+ }
+ return false;
+ }
+
Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments,
uint32_t result, size_t input_rows_count) const override {
return FunctionImpl::execute(context, block, arguments, result, input_rows_count);
@@ -1015,6 +1207,9 @@ template <typename FunctionName, bool WithPrecision>
struct CurrentDateTimeImpl {
static constexpr auto name = FunctionName::name;
static constexpr PrimitiveType ReturnType = TYPE_DATETIMEV2;
+ static constexpr int DATETIMEV2_MAX_SCALE = 6;
+
+ static bool skip_return_type_check() { return WithPrecision; }
static DataTypes get_variadic_argument_types() {
if constexpr (WithPrecision) {
@@ -1028,10 +1223,15 @@ struct CurrentDateTimeImpl {
uint32_t result, size_t input_rows_count) {
if constexpr (WithPrecision) {
DCHECK(block.get_by_position(result).type->get_primitive_type() == TYPE_DATETIMEV2 ||
- block.get_by_position(result).type->get_primitive_type() == TYPE_DATEV2);
+ block.get_by_position(result).type->get_primitive_type() == TYPE_DATEV2 ||
+ block.get_by_position(result).type->get_primitive_type() == TYPE_TIMESTAMP_NS);
if (block.get_by_position(result).type->get_primitive_type() == TYPE_DATETIMEV2) {
return executeImpl<TYPE_DATETIMEV2>(context, block, arguments, result,
input_rows_count);
+ } else if (block.get_by_position(result).type->get_primitive_type() ==
+ TYPE_TIMESTAMP_NS) {
+ return executeImpl<TYPE_TIMESTAMP_NS>(context, block, arguments, result,
+ input_rows_count);
} else {
return executeImpl<TYPE_DATEV2>(context, block, arguments, result,
input_rows_count);
@@ -1047,6 +1247,27 @@ struct CurrentDateTimeImpl {
}
}
+ template <PrimitiveType PType>
+ static void set_current_value(FunctionContext* context, int64_t scale,
+ typename PrimitiveTypeTraits<PType>::CppType* value) {
+ if constexpr (PType == TYPE_TIMESTAMP_NS) {
+ DCHECK_GT(scale, DATETIMEV2_MAX_SCALE);
+ DCHECK_LE(scale, TimeStampNsValue::FRACTIONAL_DIGITS);
+ const int32_t nanos = context->state()->nano_seconds();
+ const int64_t factor = common::exp10_i64(TimeStampNsValue::FRACTIONAL_DIGITS - scale);
+ const int32_t truncated_nanos = cast_set<int32_t>(nanos / factor * factor);
+ DateV2Value<DateTimeV2ValueType> local_datetime;
+ local_datetime.from_unixtime(context->state()->timestamp_ms() / 1000, nanos,
+ context->state()->timezone_obj(), DATETIMEV2_MAX_SCALE);
+ DCHECK(value->from_datetime(local_datetime,
+ cast_set<uint16_t>(truncated_nanos % 1000)));
+ } else {
+ value->from_unixtime(context->state()->timestamp_ms() / 1000,
+ context->state()->nano_seconds(), context->state()->timezone_obj(),
+ scale);
+ }
+ }
+
template <PrimitiveType PType>
static Status executeImpl(FunctionContext* context, Block& block,
const ColumnNumbers& arguments, uint32_t result,
@@ -1059,9 +1280,7 @@ struct CurrentDateTimeImpl {
if (const auto* const_column = check_and_get_column<ColumnConst>(
block.get_by_position(arguments[0]).column.get())) {
int64_t scale = const_column->get_int(0);
- dtv.from_unixtime(context->state()->timestamp_ms() / 1000,
- context->state()->nano_seconds(),
- context->state()->timezone_obj(), scale);
+ set_current_value<PType>(context, scale, &dtv);
if constexpr (std::is_same_v<DateValueType, VecDateTimeValue>) {
reinterpret_cast<DateValueType*>(&dtv)->set_type(TIME_DATETIME);
}
@@ -1075,10 +1294,7 @@ struct CurrentDateTimeImpl {
nullable_column->get_nested_column_ptr().get());
for (int i = 0; i < input_rows_count; i++) {
if (!null_map[i]) {
- dtv.from_unixtime(context->state()->timestamp_ms() / 1000,
- context->state()->nano_seconds(),
- context->state()->timezone_obj(),
- nested_column->get_element(i));
+ set_current_value<PType>(context, nested_column->get_element(i), &dtv);
if constexpr (std::is_same_v<DateValueType, VecDateTimeValue>) {
reinterpret_cast<DateValueType*>(&dtv)->set_type(TIME_DATETIME);
}
@@ -1095,9 +1311,7 @@ struct CurrentDateTimeImpl {
const auto* int_column = assert_cast<const ColumnInt32*>(
block.get_by_position(arguments[0]).column.get());
for (int i = 0; i < input_rows_count; i++) {
- dtv.from_unixtime(context->state()->timestamp_ms() / 1000,
- context->state()->nano_seconds(),
- context->state()->timezone_obj(), int_column->get_element(i));
+ set_current_value<PType>(context, int_column->get_element(i), &dtv);
if constexpr (std::is_same_v<DateValueType, VecDateTimeValue>) {
reinterpret_cast<DateValueType*>(&dtv)->set_type(TIME_DATETIME);
}
@@ -1198,13 +1412,22 @@ struct TimeToSecImpl {
uint32_t result, size_t input_rows_count) {
auto res_col = ColumnInt32::create(input_rows_count);
const auto& arg_col = block.get_by_position(arguments[0]).column;
- const auto& column_data = assert_cast<const ColumnTimeV2&>(*arg_col);
-
auto& res_data = res_col->get_data();
- for (int i = 0; i < input_rows_count; ++i) {
- res_data[i] =
- cast_set<int, int64_t, false>(static_cast<int64_t>(column_data.get_element(i)) /
- (TimeValue::ONE_SECOND_MICROSECONDS));
+ if (block.get_by_position(arguments[0]).type->get_primitive_type() == TYPE_TIMESTAMP_NS) {
+ const auto& column_data = assert_cast<const ColumnTimeStampNs&>(*arg_col);
+ for (int i = 0; i < input_rows_count; ++i) {
+ res_data[i] = cast_set<int, int64_t, false>(
+ column_data.get_element(i).time_part_to_seconds());
+ }
+ } else {
+ DORIS_CHECK_EQ(block.get_by_position(arguments[0]).type->get_primitive_type(),
+ TYPE_TIMEV2);
+ const auto& column_data = assert_cast<const ColumnTimeV2&>(*arg_col);
+ for (int i = 0; i < input_rows_count; ++i) {
+ res_data[i] = cast_set<int, int64_t, false>(
+ static_cast<int64_t>(column_data.get_element(i)) /
+ TimeValue::ONE_SECOND_MICROSECONDS);
+ }
}
block.replace_by_position(result, std::move(res_col));
@@ -1321,6 +1544,9 @@ struct TimestampToDateTime : IFunction {
template <PrimitiveType UTCType>
struct UtcImpl {
static constexpr PrimitiveType ReturnType = UTCType;
+ static constexpr int DATETIMEV2_MAX_SCALE = 6;
+
+ static bool skip_return_type_check() { return ReturnType == TYPE_DATETIMEV2; }
static constexpr const char* get_function_name() {
if constexpr (ReturnType == TYPE_DATETIMEV2 || ReturnType == TYPE_DATETIME) {
@@ -1343,6 +1569,25 @@ struct UtcImpl {
block.get_by_position(arguments[0]).column.get());
scale = col->get_element(0);
}
+ if constexpr (ReturnType == TYPE_DATETIMEV2) {
+ if (block.get_by_position(result).type->get_primitive_type() == TYPE_TIMESTAMP_NS) {
+ auto col_to = ColumnTimeStampNs::create();
+ const int32_t nanos = context->state()->nano_seconds();
+ const int64_t factor =
+ common::exp10_i64(TimeStampNsValue::FRACTIONAL_DIGITS - scale);
+ const int32_t truncated_nanos = cast_set<int32_t>(nanos / factor * factor);
+ DateV2Value<DateTimeV2ValueType> utc_datetime;
+ utc_datetime.from_unixtime(context->state()->timestamp_ms() / 1000, nanos, "+00:00",
+ DATETIMEV2_MAX_SCALE);
+ TimeStampNsValue timestamp;
+ DCHECK(timestamp.from_datetime(utc_datetime,
+ cast_set<uint16_t>(truncated_nanos % 1000)));
+ col_to->insert_value(timestamp);
+ block.get_by_position(result).column =
+ ColumnConst::create(std::move(col_to), input_rows_count);
+ return Status::OK();
+ }
+ }
auto col_to = PrimitiveTypeTraits<ReturnType>::ColumnType::create();
DateV2Value<DateTimeV2ValueType> dtv;
if (dtv.from_unixtime(context->state()->timestamp_ms() / 1000,
@@ -1412,6 +1657,17 @@ protected:
}
};
+template <typename DateValue>
+DateV2Value<DateV2ValueType> date_v2_from_date_like(const DateValue& value) {
+ if constexpr (std::is_same_v<DateValue, TimeStampNsValue>) {
+ return value.to_date();
+ } else {
+ DateV2Value<DateV2ValueType> date;
+ date.unchecked_set_time(value.year(), value.month(), value.day(), 0, 0, 0, 0);
+ return date;
+ }
+}
+
class FunctionMonthsBetween : public IFunction {
public:
static constexpr auto name = "months_between";
@@ -1437,20 +1693,17 @@ public:
const auto& [col3, col3_const] =
unpack_if_const(block.get_by_position(arguments[2]).column);
-
- const auto& date1_col = *assert_cast<const ColumnDateV2*>(date_cols[0].get());
- const auto& date2_col = *assert_cast<const ColumnDateV2*>(date_cols[1].get());
const auto& round_off_col = *assert_cast<const ColumnBool*>(col3.get());
- if (date_consts[0] && date_consts[1]) {
- execute_vector<true, false>(input_rows_count, date1_col, date2_col, round_off_col,
- *res);
- } else if (col3_const) {
- execute_vector<false, true>(input_rows_count, date1_col, date2_col, round_off_col,
- *res);
+ auto date_type = block.get_by_position(arguments[0]).type->get_primitive_type();
+ DORIS_CHECK_EQ(date_type, block.get_by_position(arguments[1]).type->get_primitive_type());
+ if (date_type == TYPE_TIMESTAMP_NS) {
+ execute_typed<ColumnTimeStampNs>(input_rows_count, date_cols, date_consts, col3_const,
+ round_off_col, *res);
} else {
- execute_vector<false, false>(input_rows_count, date1_col, date2_col, round_off_col,
- *res);
+ DORIS_CHECK_EQ(date_type, TYPE_DATEV2);
+ execute_typed<ColumnDateV2>(input_rows_count, date_cols, date_consts, col3_const,
+ round_off_col, *res);
}
block.replace_by_position(result, std::move(res));
@@ -1458,17 +1711,33 @@ public:
}
private:
- template <bool is_date_const, bool is_round_off_const>
- static void execute_vector(const size_t input_rows_count, const ColumnDateV2& date1_col,
- const ColumnDateV2& date2_col, const ColumnBool& round_off_col,
+ template <typename DateColumn>
+ static void execute_typed(size_t input_rows_count, const ColumnPtr (&date_cols)[2],
+ const bool (&date_consts)[2], bool round_off_const,
+ const ColumnBool& round_off_col, ColumnFloat64& res) {
+ const auto& date1_col = *assert_cast<const DateColumn*>(date_cols[0].get());
+ const auto& date2_col = *assert_cast<const DateColumn*>(date_cols[1].get());
+ if (date_consts[0] && date_consts[1]) {
+ execute_vector<true, false>(input_rows_count, date1_col, date2_col, round_off_col, res);
+ } else if (round_off_const) {
+ execute_vector<false, true>(input_rows_count, date1_col, date2_col, round_off_col, res);
+ } else {
+ execute_vector<false, false>(input_rows_count, date1_col, date2_col, round_off_col,
+ res);
+ }
+ }
+
+ template <bool is_date_const, bool is_round_off_const, typename DateColumn>
+ static void execute_vector(const size_t input_rows_count, const DateColumn& date1_col,
+ const DateColumn& date2_col, const ColumnBool& round_off_col,
ColumnFloat64& res) {
res.reserve(input_rows_count);
double months_between;
bool round_off;
if constexpr (is_date_const) {
- auto dtv1 = date1_col.get_element(0);
- auto dtv2 = date2_col.get_element(0);
+ auto dtv1 = date_v2_from_date_like(date1_col.get_element(0));
+ auto dtv2 = date_v2_from_date_like(date2_col.get_element(0));
months_between = calc_months_between(dtv1, dtv2);
}
@@ -1478,8 +1747,8 @@ private:
for (int i = 0; i < input_rows_count; ++i) {
if constexpr (!is_date_const) {
- auto dtv1 = date1_col.get_element(i);
- auto dtv2 = date2_col.get_element(i);
+ auto dtv1 = date_v2_from_date_like(date1_col.get_element(i));
+ auto dtv2 = date_v2_from_date_like(date2_col.get_element(i));
months_between = calc_months_between(dtv1, dtv2);
}
if constexpr (!is_round_off_const) {
@@ -1541,15 +1810,16 @@ public:
unpack_if_const(block.get_by_position(arguments[0]).column);
const auto& [right_col, right_const] =
unpack_if_const(block.get_by_position(arguments[1]).column);
- const auto& date_col = *assert_cast<const ColumnDateV2*>(left_col.get());
const auto& week_col = *assert_cast<const ColumnString*>(right_col.get());
Status status;
- if (left_const) {
- status = execute_vector<true, false>(input_rows_count, date_col, week_col, *res);
- } else if (right_const) {
- status = execute_vector<false, true>(input_rows_count, date_col, week_col, *res);
+ auto date_type = block.get_by_position(arguments[0]).type->get_primitive_type();
+ if (date_type == TYPE_TIMESTAMP_NS) {
+ status = execute_typed<ColumnTimeStampNs>(input_rows_count, left_col, left_const,
+ right_const, week_col, *res);
} else {
- status = execute_vector<false, false>(input_rows_count, date_col, week_col, *res);
+ DORIS_CHECK_EQ(date_type, TYPE_DATEV2);
+ status = execute_typed<ColumnDateV2>(input_rows_count, left_col, left_const,
+ right_const, week_col, *res);
}
if (!status.ok()) {
return status;
@@ -1582,13 +1852,26 @@ private:
return Status::OK();
}
- template <bool left_const, bool right_const>
- static Status execute_vector(size_t input_rows_count, const ColumnDateV2& left_col,
+ template <typename DateColumn>
+ static Status execute_typed(size_t input_rows_count, const ColumnPtr& left_col, bool left_const,
+ bool right_const, const ColumnString& week_col,
+ ColumnDateV2& res_col) {
+ const auto& date_col = *assert_cast<const DateColumn*>(left_col.get());
+ if (left_const) {
+ return execute_vector<true, false>(input_rows_count, date_col, week_col, res_col);
+ } else if (right_const) {
+ return execute_vector<false, true>(input_rows_count, date_col, week_col, res_col);
+ }
+ return execute_vector<false, false>(input_rows_count, date_col, week_col, res_col);
+ }
+
+ template <bool left_const, bool right_const, typename DateColumn>
+ static Status execute_vector(size_t input_rows_count, const DateColumn& left_col,
const ColumnString& right_col, ColumnDateV2& res_col) {
DateV2Value<DateV2ValueType> dtv;
int week_day;
if constexpr (left_const) {
- dtv = left_col.get_element(0);
+ dtv = date_v2_from_date_like(left_col.get_element(0));
}
if constexpr (right_const) {
auto week = right_col.get_data_at(0);
@@ -1601,7 +1884,7 @@ private:
for (size_t i = 0; i < input_rows_count; ++i) {
if constexpr (!left_const) {
- dtv = left_col.get_element(i);
+ dtv = date_v2_from_date_like(left_col.get_element(i));
}
if constexpr (!right_const) {
auto week = right_col.get_data_at(i);
@@ -1629,20 +1912,38 @@ public:
size_t get_number_of_arguments() const override { return 1; }
DataTypePtr get_return_type_impl(const DataTypes& arguments) const override {
- return std::make_shared<DataTypeTimeV2>(arguments[0]->get_scale());
+ return std::make_shared<DataTypeTimeV2>(
+ std::min(arguments[0]->get_scale(), DataTypeTimeV2::MAX_SCALE));
}
Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments,
uint32_t result, size_t input_rows_count) const override {
DCHECK_EQ(arguments.size(), 1);
- ColumnPtr col = block.get_by_position(arguments[0]).column;
- const auto& arg = assert_cast<const ColumnDateTimeV2&>(*col.get());
+ const auto& argument = block.get_by_position(arguments[0]);
+ if (argument.type->get_primitive_type() == TYPE_TIMESTAMP_NS) {
+ return execute_typed<TYPE_TIMESTAMP_NS>(block, arguments[0], result, input_rows_count);
+ }
+ DORIS_CHECK_EQ(argument.type->get_primitive_type(), TYPE_DATETIMEV2);
+ return execute_typed<TYPE_DATETIMEV2>(block, arguments[0], result, input_rows_count);
+ }
+
+private:
+ template <PrimitiveType PType>
+ Status execute_typed(Block& block, uint32_t argument, uint32_t result,
+ size_t input_rows_count) const {
+ const auto& arg = assert_cast<const typename PrimitiveTypeTraits<PType>::ColumnType&>(
+ *block.get_by_position(argument).column);
ColumnTimeV2::MutablePtr res = ColumnTimeV2::create(input_rows_count);
auto& res_data = res->get_data();
- for (int i = 0; i < arg.size(); i++) {
+ for (size_t i = 0; i < arg.size(); ++i) {
const auto& v = arg.get_element(i);
- // the arg is datetimev2 type, it's store as uint64, so we need to get arg's hour minute second part
- res_data[i] = TimeValue::make_time(v.hour(), v.minute(), v.second(), v.microsecond());
+ // TIMEV2 has microsecond precision, so TIMESTAMP_NS intentionally drops digits 7-9.
+ if constexpr (PType == TYPE_TIMESTAMP_NS) {
+ res_data[i] = static_cast<TimeValue::TimeType>(v.time_part_to_microsecond());
+ } else {
+ res_data[i] =
+ TimeValue::make_time(v.hour(), v.minute(), v.second(), v.microsecond());
+ }
}
block.replace_by_position(result, std::move(res));
return Status::OK();
@@ -1887,7 +2188,16 @@ public:
const auto& arg1 = left_data[index_check_const(i, cols_info[0].is_const)];
const auto& arg2 = right_data[index_check_const(i, cols_info[1].is_const)];
- if constexpr (PType == TYPE_DATETIMEV2 || PType == TYPE_TIMESTAMPTZ) {
+ if constexpr (PType == TYPE_TIMESTAMP_NS) {
+ auto result = arg1;
+ auto tv2 = static_cast<TimeValue::TimeType>(arg2);
+ TimeInterval interval(TimeUnit::MICROSECOND, tv2, IsNegative);
+ if (!result.template date_add_interval<TimeUnit::MICROSECOND>(interval))
+ [[unlikely]] {
+ throw_invalid_strings(name, arg1.to_string(), std::to_string(arg2));
+ }
+ res_col->insert_value(result);
+ } else if constexpr (PType == TYPE_DATETIMEV2 || PType == TYPE_TIMESTAMPTZ) {
DateV2Value<DateTimeV2ValueType> dtv1(arg1.to_date_int_val());
auto tv2 = static_cast<TimeValue::TimeType>(arg2);
TimeInterval interval(TimeUnit::MICROSECOND, tv2, IsNegative);
@@ -1909,9 +2219,11 @@ public:
};
using AddTimeDatetimeImpl = AddTimeImplBase<TYPE_DATETIMEV2, false>;
+using AddTimeTimeStampNsImpl = AddTimeImplBase<TYPE_TIMESTAMP_NS, false>;
using AddTimeTimeImpl = AddTimeImplBase<TYPE_TIMEV2, false>;
using AddTimeTimestamptzImpl = AddTimeImplBase<TYPE_TIMESTAMPTZ, false>;
using SubTimeDatetimeImpl = AddTimeImplBase<TYPE_DATETIMEV2, true>;
+using SubTimeTimeStampNsImpl = AddTimeImplBase<TYPE_TIMESTAMP_NS, true>;
using SubTimeTimeImpl = AddTimeImplBase<TYPE_TIMEV2, true>;
using SubTimeTimestamptzImpl = AddTimeImplBase<TYPE_TIMESTAMPTZ, true>;
#include "common/compile_check_avoid_end.h"
diff --git a/be/src/exprs/function/function_date_or_datetime_to_something.h b/be/src/exprs/function/function_date_or_datetime_to_something.h
index 6f0ee6fca26..17ed66e3363 100644
--- a/be/src/exprs/function/function_date_or_datetime_to_something.h
+++ b/be/src/exprs/function/function_date_or_datetime_to_something.h
@@ -49,7 +49,8 @@ struct Transformer {
#endif
auto res = Transform::execute(vec_from[i]);
- if constexpr (is_date_type(ToPType) || ToPType == TYPE_TIMESTAMPTZ) {
+ if constexpr (is_date_type(ToPType) || ToPType == TYPE_TIMESTAMP_NS ||
+ ToPType == TYPE_TIMESTAMPTZ) {
vec_to[i] = res;
} else {
using RESULT_TYPE = std::decay_t<decltype(res)>;
@@ -109,7 +110,8 @@ public:
DataTypePtr get_return_type_impl(const ColumnsWithTypeAndName& arguments) const override {
if (arguments.size() == 1) {
if (!is_date_or_datetime(arguments[0].type->get_primitive_type()) &&
- !is_date_v2_or_datetime_v2(arguments[0].type->get_primitive_type())) {
+ !is_date_v2_or_datetime_v2(arguments[0].type->get_primitive_type()) &&
+ arguments[0].type->get_primitive_type() != TYPE_TIMESTAMP_NS) {
throw doris::Exception(ErrorCode::INVALID_ARGUMENT,
"Illegal type {} of argument of function {}. Should be a "
"date or a date with time",
@@ -117,7 +119,8 @@ public:
}
} else if (arguments.size() == 2) {
if (!is_date_or_datetime(arguments[0].type->get_primitive_type()) &&
- !is_date_v2_or_datetime_v2(arguments[0].type->get_primitive_type())) {
+ !is_date_v2_or_datetime_v2(arguments[0].type->get_primitive_type()) &&
+ arguments[0].type->get_primitive_type() != TYPE_TIMESTAMP_NS) {
throw doris::Exception(ErrorCode::INVALID_ARGUMENT,
"Illegal type {} of argument of function {}. Should be a "
"date or a date with time",
diff --git a/be/src/exprs/function/function_date_or_datetime_to_string.cpp b/be/src/exprs/function/function_date_or_datetime_to_string.cpp
index 9788801e39f..3b259d504b5 100644
--- a/be/src/exprs/function/function_date_or_datetime_to_string.cpp
+++ b/be/src/exprs/function/function_date_or_datetime_to_string.cpp
@@ -214,20 +214,30 @@ using FunctionMonthNameV2 = FunctionDateOrDateTimeToString<MonthNameImpl<TYPE_DA
using FunctionDateTimeV2DayName = FunctionDateOrDateTimeToString<DayNameImpl<TYPE_DATETIMEV2>>;
using FunctionDateTimeV2MonthName = FunctionDateOrDateTimeToString<MonthNameImpl<TYPE_DATETIMEV2>>;
-using FunctionYearMonth = FunctionDateOrDateTimeToString<YearMonthImpl>;
-using FunctionDayHour = FunctionDateOrDateTimeToString<DayHourImpl>;
-using FunctionDayMinute = FunctionDateOrDateTimeToString<DayMinuteImpl>;
-using FunctionDaySecond = FunctionDateOrDateTimeToString<DaySecondImpl>;
-using FunctionDayMicrosecond = FunctionDateOrDateTimeToString<DayMicrosecondImpl>;
-using FunctionHourMinute = FunctionDateOrDateTimeToString<HourMinuteImpl>;
-using FunctionHourSecond = FunctionDateOrDateTimeToString<HourSecondImpl>;
-using FunctionHourMicrosecond = FunctionDateOrDateTimeToString<HourMicrosecondImpl>;
-using FunctionMinuteSecond = FunctionDateOrDateTimeToString<MinuteSecondImpl>;
-using FunctionMinuteMicrosecond = FunctionDateOrDateTimeToString<MinuteMicrosecondImpl>;
-using FunctionSecondMicrosecond = FunctionDateOrDateTimeToString<SecondMicrosecondImpl>;
+using FunctionTimestampNsDayName = FunctionDateOrDateTimeToString<DayNameImpl<TYPE_TIMESTAMP_NS>>;
+using FunctionTimestampNsMonthName =
+ FunctionDateOrDateTimeToString<MonthNameImpl<TYPE_TIMESTAMP_NS>>;
+#define DEFINE_DATETIME_FORMAT_FUNCTION(NAME) \
+ using Function##NAME = FunctionDateOrDateTimeToString<NAME##Impl<TYPE_DATETIMEV2>>; \
+ using FunctionTimestampNs##NAME = FunctionDateOrDateTimeToString<NAME##Impl<TYPE_TIMESTAMP_NS>>
+
+DEFINE_DATETIME_FORMAT_FUNCTION(YearMonth);
+DEFINE_DATETIME_FORMAT_FUNCTION(DayHour);
+DEFINE_DATETIME_FORMAT_FUNCTION(DayMinute);
+DEFINE_DATETIME_FORMAT_FUNCTION(DaySecond);
+DEFINE_DATETIME_FORMAT_FUNCTION(DayMicrosecond);
+DEFINE_DATETIME_FORMAT_FUNCTION(HourMinute);
+DEFINE_DATETIME_FORMAT_FUNCTION(HourSecond);
+DEFINE_DATETIME_FORMAT_FUNCTION(HourMicrosecond);
+DEFINE_DATETIME_FORMAT_FUNCTION(MinuteSecond);
+DEFINE_DATETIME_FORMAT_FUNCTION(MinuteMicrosecond);
+DEFINE_DATETIME_FORMAT_FUNCTION(SecondMicrosecond);
+
+#undef DEFINE_DATETIME_FORMAT_FUNCTION
using FunctionDateIso8601 = FunctionDateOrDateTimeToString<ToIso8601Impl<TYPE_DATEV2>>;
using FunctionDateTimeIso8601 = FunctionDateOrDateTimeToString<ToIso8601Impl<TYPE_DATETIMEV2>>;
+using FunctionTimeStampNsIso8601 = FunctionDateOrDateTimeToString<ToIso8601Impl<TYPE_TIMESTAMP_NS>>;
using FunctionTimestampTzIso8601 = FunctionDateOrDateTimeToString<ToIso8601Impl<TYPE_TIMESTAMPTZ>>;
void register_function_date_time_to_string(SimpleFunctionFactory& factory) {
@@ -235,6 +245,8 @@ void register_function_date_time_to_string(SimpleFunctionFactory& factory) {
factory.register_function<FunctionMonthNameV2>();
factory.register_function<FunctionDateTimeV2DayName>();
factory.register_function<FunctionDateTimeV2MonthName>();
+ factory.register_function<FunctionTimestampNsDayName>();
+ factory.register_function<FunctionTimestampNsMonthName>();
factory.register_function<FunctionYearMonth>();
factory.register_function<FunctionDayHour>();
factory.register_function<FunctionDayMinute>();
@@ -246,8 +258,20 @@ void register_function_date_time_to_string(SimpleFunctionFactory& factory) {
factory.register_function<FunctionMinuteSecond>();
factory.register_function<FunctionMinuteMicrosecond>();
factory.register_function<FunctionSecondMicrosecond>();
+ factory.register_function<FunctionTimestampNsYearMonth>();
+ factory.register_function<FunctionTimestampNsDayHour>();
+ factory.register_function<FunctionTimestampNsDayMinute>();
+ factory.register_function<FunctionTimestampNsDaySecond>();
+ factory.register_function<FunctionTimestampNsDayMicrosecond>();
+ factory.register_function<FunctionTimestampNsHourMinute>();
+ factory.register_function<FunctionTimestampNsHourSecond>();
+ factory.register_function<FunctionTimestampNsHourMicrosecond>();
+ factory.register_function<FunctionTimestampNsMinuteSecond>();
+ factory.register_function<FunctionTimestampNsMinuteMicrosecond>();
+ factory.register_function<FunctionTimestampNsSecondMicrosecond>();
factory.register_function<FunctionDateIso8601>();
factory.register_function<FunctionDateTimeIso8601>();
+ factory.register_function<FunctionTimeStampNsIso8601>();
factory.register_function<FunctionTimestampTzIso8601>();
}
diff --git a/be/src/exprs/function/function_datetime_floor_ceil.cpp b/be/src/exprs/function/function_datetime_floor_ceil.cpp
index 80b128ee6ec..0ab4d4c0c68 100644
--- a/be/src/exprs/function/function_datetime_floor_ceil.cpp
+++ b/be/src/exprs/function/function_datetime_floor_ceil.cpp
@@ -372,9 +372,11 @@ struct DateTimeFloorCeilCore {
static void vector_const_const(const PaddedPODArray<DateValueType>& dates, const Int32 period,
DateValueType origin_date, PaddedPODArray<DateValueType>& res,
const NullMap& result_null_map, FunctionContext* context) {
- if (auto cast_date = origin_date; cast_date == DateValueType::FIRST_DAY) {
- vector_const_period(dates, period, res, result_null_map, context);
- return;
+ if constexpr (!std::is_same_v<DateValueType, TimeStampNsValue>) {
+ if (auto cast_date = origin_date; cast_date == DateValueType::FIRST_DAY) {
+ vector_const_period(dates, period, res, result_null_map, context);
+ return;
+ }
}
// expand codes for const input periods
@@ -521,24 +523,37 @@ struct DateTimeFloorCeilCore {
template <int const_period = 0>
static bool time_round_reinterpret_two_args(DateValueType date, Int32 period,
DateValueType& res, FunctionContext* context) {
- auto ts_arg = date;
- auto& ts_res = res;
-
- if constexpr (const_period == 0) {
- if (can_use_optimize(period)) {
- floor_opt(ts_arg, ts_res, period);
- return true;
- } else {
- ts_res = DateValueType::FIRST_DAY;
- return time_round_two_args(ts_arg, period, ts_res, context);
+ if constexpr (std::is_same_v<DateValueType, TimeStampNsValue>) {
+ auto legacy_date = date.to_datetime();
+ if (date.nanosecond_remainder() != 0 && legacy_date.microsecond() == 0) {
+ legacy_date.template unchecked_set_time_unit<TimeUnit::MICROSECOND>(1);
+ }
+ auto legacy_result = DateV2Value<DateTimeV2ValueType>::FIRST_DAY;
+ if (!DateTimeFloorCeilCore<Flag, TYPE_DATETIMEV2>::template time_round_two_args<
+ const_period>(legacy_date, period, legacy_result, context)) {
+ return false;
}
+ return res.from_datetime(legacy_result);
} else {
- if (can_use_optimize(const_period)) {
- floor_opt(ts_arg, ts_res, const_period);
- return true;
+ auto ts_arg = date;
+ auto& ts_res = res;
+
+ if constexpr (const_period == 0) {
+ if (can_use_optimize(period)) {
+ floor_opt(ts_arg, ts_res, period);
+ return true;
+ } else {
+ ts_res = DateValueType::FIRST_DAY;
+ return time_round_two_args(ts_arg, period, ts_res, context);
+ }
} else {
- ts_res = DateValueType::FIRST_DAY;
- return time_round_two_args<const_period>(ts_arg, const_period, ts_res, context);
+ if (can_use_optimize(const_period)) {
+ floor_opt(ts_arg, ts_res, const_period);
+ return true;
+ } else {
+ ts_res = DateValueType::FIRST_DAY;
+ return time_round_two_args<const_period>(ts_arg, const_period, ts_res, context);
+ }
}
}
}
@@ -757,6 +772,93 @@ struct DateTimeFloorCeilCore {
trivial_part_ts_arg = calc_arg.microsecond();
trivial_part_ts_res = calc_origin.microsecond();
}
+ } else if constexpr (std::is_same_v<DateValueType, TimeStampNsValue>) {
+ const auto nanos_since_midnight = [](const TimeStampNsValue& value) {
+ return value.time_part_to_nanosecond();
+ };
+ const auto nanos_since_month = [&](const TimeStampNsValue& value,
+ const auto& civil_value) {
+ return (civil_value.day() - 1) * TimeStampNsValue::NANOS_PER_DAY +
+ nanos_since_midnight(value);
+ };
+
+ if constexpr (Flag::Unit == YEAR) {
+ const auto arg_date = ts_arg.to_date();
+ const auto origin_date = ts_origin.to_date();
+ diff = arg_date.year() - origin_date.year();
+ const auto calendar_remainder = [&](const TimeStampNsValue& value,
+ const auto& civil_value) {
+ return (static_cast<int64_t>(civil_value.month()) * 32 + civil_value.day()) *
+ TimeStampNsValue::NANOS_PER_DAY +
+ nanos_since_midnight(value);
+ };
+ trivial_part_ts_arg = calendar_remainder(ts_arg, arg_date);
+ trivial_part_ts_res = calendar_remainder(ts_origin, origin_date);
+ }
+ if constexpr (Flag::Unit == QUARTER) {
+ const auto arg_date = ts_arg.to_date();
+ const auto origin_date = ts_origin.to_date();
+ const int64_t total_months = (arg_date.year() - origin_date.year()) * 12 +
+ arg_date.month() - origin_date.month();
+ diff = total_months / 3;
+ const int64_t remaining_months = total_months % 3;
+ if (remaining_months != 0) {
+ trivial_part_ts_arg = remaining_months;
+ trivial_part_ts_res = 0;
+ } else {
+ trivial_part_ts_arg = nanos_since_month(ts_arg, arg_date);
+ trivial_part_ts_res = nanos_since_month(ts_origin, origin_date);
+ }
+ }
+ if constexpr (Flag::Unit == MONTH) {
+ const auto arg_date = ts_arg.to_date();
+ const auto origin_date = ts_origin.to_date();
+ diff = (arg_date.year() - origin_date.year()) * 12 + arg_date.month() -
+ origin_date.month();
+ trivial_part_ts_arg = nanos_since_month(ts_arg, arg_date);
+ trivial_part_ts_res = nanos_since_month(ts_origin, origin_date);
+ }
+ if constexpr (Flag::Unit == WEEK) {
+ const int64_t arg_daynr = ts_arg.daynr();
+ const int64_t origin_daynr = ts_origin.daynr();
+ diff = arg_daynr / 7 - origin_daynr / 7;
+ trivial_part_ts_arg = arg_daynr % 7 * TimeStampNsValue::NANOS_PER_DAY +
+ nanos_since_midnight(ts_arg);
+ trivial_part_ts_res = origin_daynr % 7 * TimeStampNsValue::NANOS_PER_DAY +
+ nanos_since_midnight(ts_origin);
+ }
+ if constexpr (Flag::Unit == DAY) {
+ diff = ts_arg.daynr() - ts_origin.daynr();
+ trivial_part_ts_arg = nanos_since_midnight(ts_arg);
+ trivial_part_ts_res = nanos_since_midnight(ts_origin);
+ }
+ if constexpr (Flag::Unit == HOUR) {
+ const int64_t arg_seconds = ts_arg.time_part_to_seconds();
+ const int64_t origin_seconds = ts_origin.time_part_to_seconds();
+ diff = (ts_arg.daynr() - ts_origin.daynr()) * HOUR_PER_DAY + arg_seconds / 3600 -
+ origin_seconds / 3600;
+ trivial_part_ts_arg = arg_seconds % 3600 * TimeStampNsValue::NANOS_PER_SECOND +
+ ts_arg.nanosecond();
+ trivial_part_ts_res = origin_seconds % 3600 * TimeStampNsValue::NANOS_PER_SECOND +
+ ts_origin.nanosecond();
+ }
+ if constexpr (Flag::Unit == MINUTE) {
+ const int64_t arg_seconds = ts_arg.time_part_to_seconds();
+ const int64_t origin_seconds = ts_origin.time_part_to_seconds();
+ diff = (ts_arg.daynr() - ts_origin.daynr()) * HOUR_PER_DAY * SECOND_PER_MINUTE +
+ arg_seconds / SECOND_PER_MINUTE - origin_seconds / SECOND_PER_MINUTE;
+ trivial_part_ts_arg =
+ arg_seconds % SECOND_PER_MINUTE * TimeStampNsValue::NANOS_PER_SECOND +
+ ts_arg.nanosecond();
+ trivial_part_ts_res =
+ origin_seconds % SECOND_PER_MINUTE * TimeStampNsValue::NANOS_PER_SECOND +
+ ts_origin.nanosecond();
+ }
+ if constexpr (Flag::Unit == SECOND) {
+ diff = ts_arg.epoch_seconds() - ts_origin.epoch_seconds();
+ trivial_part_ts_arg = ts_arg.nanosecond();
+ trivial_part_ts_res = ts_origin.nanosecond();
+ }
}
//round down/up to specific time-unit(HOUR/DAY/MONTH...) by increase/decrease diff variable
@@ -803,6 +905,10 @@ struct DateTimeFloorCeilCore {
/// optimized path
constexpr static bool can_use_optimize(int period) {
+ if constexpr (std::is_same_v<DateValueType, TimeStampNsValue>) {
+ return false;
+ }
+
// For TimestampTzValue on date-based units, disable optimization to ensure timezone conversion
if constexpr (std::is_same_v<DateValueType, TimestampTzValue>) {
if constexpr (Flag::Unit == YEAR || Flag::Unit == QUARTER || Flag::Unit == MONTH ||
@@ -944,6 +1050,12 @@ struct DateTimeFloorCeilCore {
FunctionDateTimeFloorCeil<IMPL, TYPE_DATETIMEV2, 2>; \
using FunctionDateTimeV2ThreeArg##IMPL##DELTA = \
FunctionDateTimeFloorCeil<IMPL, TYPE_DATETIMEV2, 3>; \
+ using FunctionTimestampNsOneArg##IMPL##DELTA = \
+ FunctionDateTimeFloorCeil<IMPL, TYPE_TIMESTAMP_NS, 1>; \
+ using FunctionTimestampNsTwoArg##IMPL##DELTA = \
+ FunctionDateTimeFloorCeil<IMPL, TYPE_TIMESTAMP_NS, 2>; \
+ using FunctionTimestampNsThreeArg##IMPL##DELTA = \
+ FunctionDateTimeFloorCeil<IMPL, TYPE_TIMESTAMP_NS, 3>; \
using FunctionTimestamptzOneArg##IMPL##DELTA = \
FunctionDateTimeFloorCeil<IMPL, TYPE_TIMESTAMPTZ, 1>; \
using FunctionTimestamptzTwoArg##IMPL##DELTA = \
@@ -962,6 +1074,8 @@ struct DateTimeFloorCeilCore {
using FunctionDateV2TwoArg##IMPL = FunctionDateTimeFloorCeil<IMPL, TYPE_DATEV2, 2, true>; \
using FunctionDateTimeV2TwoArg##IMPL = \
FunctionDateTimeFloorCeil<IMPL, TYPE_DATETIMEV2, 2, true>; \
+ using FunctionTimestampNsTwoArg##IMPL = \
+ FunctionDateTimeFloorCeil<IMPL, TYPE_TIMESTAMP_NS, 2, true>; \
using FunctionTimestamptzTwoArg##IMPL = \
FunctionDateTimeFloorCeil<IMPL, TYPE_TIMESTAMPTZ, 2, true>;
@@ -991,10 +1105,14 @@ void register_function_datetime_floor_ceil(SimpleFunctionFactory& factory) {
factory.register_function<FunctionDateTimeV2OneArg##IMPL##DELTA>(); \
factory.register_function<FunctionDateTimeV2TwoArg##IMPL##DELTA>(); \
factory.register_function<FunctionDateTimeV2ThreeArg##IMPL##DELTA>(); \
+ factory.register_function<FunctionTimestampNsOneArg##IMPL##DELTA>(); \
+ factory.register_function<FunctionTimestampNsTwoArg##IMPL##DELTA>(); \
+ factory.register_function<FunctionTimestampNsThreeArg##IMPL##DELTA>(); \
factory.register_function<FunctionTimestamptzOneArg##IMPL##DELTA>(); \
factory.register_function<FunctionTimestamptzTwoArg##IMPL##DELTA>(); \
factory.register_function<FunctionTimestamptzThreeArg##IMPL##DELTA>(); \
factory.register_function<FunctionDateTimeV2TwoArg##IMPL>(); \
+ factory.register_function<FunctionTimestampNsTwoArg##IMPL>(); \
factory.register_function<FunctionDateV2TwoArg##IMPL>(); \
factory.register_function<FunctionTimestamptzTwoArg##IMPL>();
diff --git a/be/src/exprs/function/function_datetime_string_to_string.cpp b/be/src/exprs/function/function_datetime_string_to_string.cpp
index 9fca37931ec..6ba9779c3cc 100644
--- a/be/src/exprs/function/function_datetime_string_to_string.cpp
+++ b/be/src/exprs/function/function_datetime_string_to_string.cpp
@@ -27,6 +27,8 @@ namespace doris {
using FunctionDateFormatV2 = FunctionDateTimeStringToString<DateFormatImpl<TYPE_DATEV2>>;
using FunctionDateTimeV2DateFormat =
FunctionDateTimeStringToString<DateFormatImpl<TYPE_DATETIMEV2>>;
+using FunctionTimestampNsDateFormat =
+ FunctionDateTimeStringToString<DateFormatImpl<TYPE_TIMESTAMP_NS>>;
// old version
using FunctionFromUnixTimeOneArg = FunctionDateTimeStringToString<FromUnixTimeImpl<false, false>>;
using FunctionFromUnixTimeTwoArg = FunctionDateTimeStringToString<FromUnixTimeImpl<true, false>>;
@@ -37,8 +39,11 @@ using FunctionFromUnixTimeNewDecimalOneArg =
FunctionDateTimeStringToString<FromUnixTimeDecimalImpl<false>>;
using FunctionFromUnixTimeNewDecimalTwoArg =
FunctionDateTimeStringToString<FromUnixTimeDecimalImpl<true>>;
+using FunctionFromUnixTimeNewDecimalNanoTwoArg =
+ FunctionDateTimeStringToString<FromUnixTimeDecimalImpl<true, true>>;
using FunctionTimeFormatDate = FunctionTimeFormat<TYPE_DATEV2>;
using FunctionTimeFormatDateTime = FunctionTimeFormat<TYPE_DATETIMEV2>;
+using FunctionTimeFormatTimeStampNs = FunctionTimeFormat<TYPE_TIMESTAMP_NS>;
using FunctionTimeFormatTime = FunctionTimeFormat<TYPE_TIMEV2>;
void register_function_date_time_string_to_string(SimpleFunctionFactory& factory) {
@@ -49,9 +54,12 @@ void register_function_date_time_string_to_string(SimpleFunctionFactory& factory
factory.register_function<FunctionFromUnixTimeNewTwoArg>();
factory.register_function<FunctionFromUnixTimeNewDecimalOneArg>();
factory.register_function<FunctionFromUnixTimeNewDecimalTwoArg>();
+ factory.register_function<FunctionFromUnixTimeNewDecimalNanoTwoArg>();
factory.register_function<FunctionDateTimeV2DateFormat>();
+ factory.register_function<FunctionTimestampNsDateFormat>();
factory.register_function<FunctionTimeFormatDate>();
factory.register_function<FunctionTimeFormatDateTime>();
+ factory.register_function<FunctionTimeFormatTimeStampNs>();
factory.register_function<FunctionTimeFormatTime>();
}
diff --git a/be/src/exprs/function/function_datetime_string_to_string.h b/be/src/exprs/function/function_datetime_string_to_string.h
index 5c3dcd36460..df954ed426b 100644
--- a/be/src/exprs/function/function_datetime_string_to_string.h
+++ b/be/src/exprs/function/function_datetime_string_to_string.h
@@ -230,6 +230,7 @@ public:
std::is_same_v<ColumnType, ColumnDateV2> ||
std::is_same_v<ColumnType, ColumnDateTime> ||
std::is_same_v<ColumnType, ColumnDateTimeV2> ||
+ std::is_same_v<ColumnType, ColumnTimeStampNs> ||
std::is_same_v<ColumnType, ColumnTimeStampTz>) {
char buf[64];
pod_array[i].to_string(buf);
diff --git a/be/src/exprs/function/function_java_udf.cpp b/be/src/exprs/function/function_java_udf.cpp
index 39cfa7bd453..4a0aecff086 100644
--- a/be/src/exprs/function/function_java_udf.cpp
+++ b/be/src/exprs/function/function_java_udf.cpp
@@ -24,7 +24,7 @@
#include "common/exception.h"
#include "core/block/block.h"
-#include "exec/connector/jni_connector.h"
+#include "format/jni/jni_data_bridge.h"
#include "jni.h"
#include "runtime/exec_env.h"
#include "runtime/user_function_cache.h"
@@ -90,8 +90,8 @@ Status JavaFunctionCall::execute_impl(FunctionContext* context, Block& block,
context->get_function_state(FunctionContext::THREAD_LOCAL));
SCOPED_TIMER(context->get_udf_execute_timer());
std::unique_ptr<long[]> input_table;
- RETURN_IF_ERROR(JniConnector::to_java_table(&block, num_rows, arguments, input_table));
- auto input_table_schema = JniConnector::parse_table_schema(&block, arguments, true);
+ RETURN_IF_ERROR(JniDataBridge::to_java_table(&block, num_rows, arguments, input_table));
+ auto input_table_schema = JniDataBridge::parse_table_schema(&block, arguments, true);
std::map<String, String> input_params = {
{"meta_address", std::to_string((long)input_table.get())},
{"required_fields", input_table_schema.first},
@@ -99,7 +99,7 @@ Status JavaFunctionCall::execute_impl(FunctionContext* context, Block& block,
Jni::LocalObject input_map;
RETURN_IF_ERROR(Jni::Util::convert_to_java_map(env, input_params, &input_map));
- auto output_table_schema = JniConnector::parse_table_schema(&block, {result}, true);
+ auto output_table_schema = JniDataBridge::parse_table_schema(&block, {result}, true);
std::string output_nullable =
block.get_by_position(result).type->is_nullable() ? "true" : "false";
std::map<String, String> output_params = {{"is_nullable", output_nullable},
@@ -113,7 +113,7 @@ Status JavaFunctionCall::execute_impl(FunctionContext* context, Block& block,
.with_arg(output_map)
.call(&output_address));
- return JniConnector::fill_block(&block, {result}, output_address);
+ return JniDataBridge::fill_block(&block, {result}, output_address);
}
Status JavaFunctionCall::close(FunctionContext* context,
diff --git a/be/src/exprs/function/function_map.cpp b/be/src/exprs/function/function_map.cpp
index 242a2d62c6b..db9b5c09a52 100644
--- a/be/src/exprs/function/function_map.cpp
+++ b/be/src/exprs/function/function_map.cpp
@@ -571,7 +571,9 @@ private:
return true;
};
- if (!dispatch_switch_all(type, call)) {
+ if (type == TYPE_TIMESTAMP_NS) {
+ call(DispatchDataType<TYPE_TIMESTAMP_NS>());
+ } else if (!dispatch_switch_all(type, call)) {
throw doris::Exception(ErrorCode::INTERNAL_ERROR, "not support type {}",
type_to_string(type));
}
@@ -762,11 +764,11 @@ private:
/*nan_direction_hint=*/1) == 0;
}
- // whether this function supports equality comparison for the given primitive type.
- // Uses dispatch_switch_all as the single source of truth so any type supported
- // by the dispatch layer is automatically accepted here.
+ // TIMESTAMP_NS is dispatched locally because adding it to the shared scalar
+ // dispatcher would instantiate unrelated arithmetic-only templates for this type.
bool is_equality_comparison_supported(PrimitiveType type) const {
- return dispatch_switch_all(type, [](const auto&) { return true; });
+ return type == TYPE_TIMESTAMP_NS ||
+ dispatch_switch_all(type, [](const auto&) { return true; });
}
};
diff --git a/be/src/exprs/function/function_other_types_to_date.cpp b/be/src/exprs/function/function_other_types_to_date.cpp
index 122fdb9338a..04b60c9c78d 100644
--- a/be/src/exprs/function/function_other_types_to_date.cpp
+++ b/be/src/exprs/function/function_other_types_to_date.cpp
@@ -23,6 +23,7 @@
#include <climits>
#include <cstdint>
#include <cstring>
+#include <limits>
#include <memory>
#include <string>
#include <tuple>
@@ -453,10 +454,11 @@ struct DateTrunc {
using DateValueType = typename PrimitiveTypeTraits<PType>::CppType;
struct State {
- using CallbackFunction =
- std::function<void(const ColumnPtr&, ColumnType&, size_t, const cctz::time_zone&)>;
+ using CallbackFunction = std::function<void(const ColumnPtr&, ColumnType&, size_t,
+ const cctz::time_zone&, bool)>;
CallbackFunction callback_function;
cctz::time_zone timezone;
+ bool clamp_to_timestamp_ns_min = false;
};
static bool is_variadic() { return true; }
@@ -491,6 +493,7 @@ struct DateTrunc {
std::shared_ptr<State> state = std::make_shared<State>();
state->timezone = context->state()->timezone_obj();
+ state->clamp_to_timestamp_ns_min = context->is_auto_partition_boundary_context();
if (std::strncmp("year", lower_str.data(), 4) == 0) {
state->callback_function = &execute_impl_right_const<TimeUnit::YEAR>;
} else if (std::strncmp("quarter", lower_str.data(), 7) == 0) {
@@ -526,7 +529,8 @@ struct DateTrunc {
auto* state = reinterpret_cast<State*>(
context->get_function_state(FunctionContext::THREAD_LOCAL));
DCHECK(state != nullptr);
- state->callback_function(datetime_column, *res, input_rows_count, state->timezone);
+ state->callback_function(datetime_column, *res, input_rows_count, state->timezone,
+ state->clamp_to_timestamp_ns_min);
block.replace_by_position(result, std::move(res));
return Status::OK();
}
@@ -535,13 +539,15 @@ private:
template <TimeUnit Unit>
static void execute_impl_right_const(const ColumnPtr& datetime_column,
ColumnType& result_column, size_t input_rows_count,
- const cctz::time_zone& timezone) {
+ const cctz::time_zone& timezone,
+ bool clamp_to_timestamp_ns_min) {
auto& data = static_cast<const ColumnType*>(datetime_column.get())->get_data();
auto& res = result_column.get_data();
for (size_t i = 0; i < input_rows_count; ++i) {
auto dt = data[i];
- // datetime_trunc only raise only when dt invalid which is impossible. so we dont throw error better.
- // then we can use default implementation for nulls with no worry of invalid nested value.
+ // TimestampTz truncation is evaluated in the session time zone. Other date values use
+ // their civil representation directly; nano values may additionally fail when the
+ // truncated result is outside the signed epoch-nanosecond range.
if constexpr (PType == TYPE_TIMESTAMPTZ) {
DateV2Value<DateTimeV2ValueType> local_dt;
dt.convert_utc_to_local(timezone, local_dt);
@@ -554,7 +560,17 @@ private:
dt.convert_local_to_utc(timezone, local_dt);
}
} else {
- dt.template datetime_trunc<Unit>();
+ if constexpr (PType == TYPE_TIMESTAMP_NS) {
+ if (!dt.template datetime_trunc<Unit>()) {
+ if (clamp_to_timestamp_ns_min) {
+ dt = DateValueType(std::numeric_limits<int64_t>::min());
+ } else {
+ throw_out_of_bound_one_date<DateValueType>("date_trunc", data[i]);
+ }
+ }
+ } else {
+ dt.template datetime_trunc<Unit>();
+ }
}
res[i] = dt;
}
@@ -695,15 +711,18 @@ template <typename DateType, bool NewVersion = false>
struct UnixTimeStampDateImpl {
static DataTypes get_variadic_argument_types() { return {std::make_shared<DateType>()}; }
- using ResultDataType =
+ using ResultDataType = std::conditional_t<
+ std::is_same_v<DateType, DataTypeTimeStampNs>, DataTypeDecimal128,
std::conditional_t<std::is_same_v<DateType, DataTypeDateTimeV2>, DataTypeDecimal64,
- std::conditional_t<NewVersion, DataTypeInt64, DataTypeInt32>>;
- using ResultColumnType =
+ std::conditional_t<NewVersion, DataTypeInt64, DataTypeInt32>>>;
+ using ResultColumnType = std::conditional_t<
+ std::is_same_v<DateType, DataTypeTimeStampNs>, ColumnDecimal128V3,
std::conditional_t<std::is_same_v<DateType, DataTypeDateTimeV2>, ColumnDecimal64,
- std::conditional_t<NewVersion, ColumnInt64, ColumnInt32>>;
+ std::conditional_t<NewVersion, ColumnInt64, ColumnInt32>>>;
static DataTypePtr get_return_type_impl(const ColumnsWithTypeAndName& arguments) {
- if constexpr (std::is_same_v<DateType, DataTypeDateTimeV2>) {
+ if constexpr (std::is_same_v<DateType, DataTypeDateTimeV2> ||
+ std::is_same_v<DateType, DataTypeTimeStampNs>) {
UInt32 scale = arguments[0].type->get_scale();
return std::make_shared<ResultDataType>(12 + scale, scale);
} else {
@@ -711,6 +730,9 @@ struct UnixTimeStampDateImpl {
}
}
+ // The compile-time branches keep the four date storage representations type-safe and avoid
+ // runtime type dispatch in this hot path.
+ // NOLINTNEXTLINE(readability-function-size)
static Status execute_impl(FunctionContext* context, Block& block,
const ColumnNumbers& arguments, uint32_t result,
size_t input_rows_count) {
@@ -747,7 +769,7 @@ struct UnixTimeStampDateImpl {
col_result_data[i] = trim_timestamp(timestamp, NewVersion);
}
block.replace_by_position(result, std::move(col_result));
- } else { // DatetimeV2
+ } else if constexpr (std::is_same_v<DateType, DataTypeDateTimeV2>) {
const auto* col_source = assert_cast<const ColumnDateTimeV2*>(col.get());
UInt32 scale = block.get_by_position(arguments[0]).type->get_scale();
auto col_result = ColumnDecimal64::create(input_rows_count, scale);
@@ -768,6 +790,30 @@ struct UnixTimeStampDateImpl {
.value;
}
block.replace_by_position(result, std::move(col_result));
+ } else {
+ const auto* col_source = assert_cast<const ColumnTimeStampNs*>(col.get());
+ UInt32 scale = block.get_by_position(arguments[0]).type->get_scale();
+ auto col_result = ColumnDecimal128V3::create(input_rows_count, scale);
+ auto& col_result_data = col_result->get_data();
+ col_result->resize(input_rows_count);
+ const Int128 scale_multiplier = common::exp10_i128(scale);
+
+ for (int i = 0; i < input_rows_count; i++) {
+ const auto& value = col_source->get_data()[i];
+ auto datetime = value.to_datetime();
+ int64_t timestamp = 0;
+ datetime.unix_timestamp(×tamp, context->state()->timezone_obj());
+ const int64_t original_timestamp = timestamp;
+ timestamp = trim_timestamp(timestamp, NewVersion);
+ if (timestamp == 0 && original_timestamp != 0) {
+ col_result_data[i] = Decimal128V3(0);
+ continue;
+ }
+ const Int128 fraction = value.nanosecond() / common::exp10_i64(9 - scale);
+ col_result_data[i] =
+ Decimal128V3(static_cast<Int128>(timestamp) * scale_multiplier + fraction);
+ }
+ block.replace_by_position(result, std::move(col_result));
}
return Status::OK();
@@ -940,18 +986,26 @@ struct Sec {
static constexpr auto name = "second_timestamp";
static constexpr Int64 ratio = 1;
};
-template <typename Impl>
+template <typename Impl, PrimitiveType PType>
class DateTimeToTimestamp : public IFunction {
public:
+ static_assert(PType == TYPE_DATETIMEV2 || PType == TYPE_TIMESTAMP_NS);
using ReturnType = Int64;
static constexpr Int64 ratio_to_micro = (1000 * 1000) / Impl::ratio;
static constexpr auto name = Impl::name;
- static FunctionPtr create() { return std::make_shared<DateTimeToTimestamp<Impl>>(); }
+ static FunctionPtr create() { return std::make_shared<DateTimeToTimestamp<Impl, PType>>(); }
String get_name() const override { return name; }
size_t get_number_of_arguments() const override { return 1; }
+ DataTypes get_variadic_argument_types_impl() const override {
+ if constexpr (PType == TYPE_TIMESTAMP_NS) {
+ return {std::make_shared<DataTypeTimeStampNs>()};
+ }
+ return {};
+ }
+
DataTypePtr get_return_type_impl(const ColumnsWithTypeAndName& arguments) const override {
if (arguments[0].type->is_nullable()) {
return make_nullable(std::make_shared<DataTypeInt64>());
@@ -962,6 +1016,23 @@ public:
Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments,
uint32_t result, size_t input_rows_count) const override {
const auto& arg_col = block.get_by_position(arguments[0]).column;
+ if constexpr (PType == TYPE_TIMESTAMP_NS) {
+ const auto& column_data = assert_cast<const ColumnTimeStampNs&>(*arg_col);
+ auto res_col = ColumnInt64::create();
+ auto& res_data = res_col->get_data();
+ res_data.resize_fill(input_rows_count, 0);
+ for (size_t i = 0; i < input_rows_count; ++i) {
+ const auto& source = column_data.get_data()[i];
+ auto dt = source.to_datetime();
+ int64_t timestamp {0};
+ dt.unix_timestamp(×tamp, context->state()->timezone_obj());
+ timestamp = timestamp * Impl::ratio + source.microsecond() / ratio_to_micro;
+ res_data[i] = timestamp;
+ }
+ block.replace_by_position(result, std::move(res_col));
+ return Status::OK();
+ }
+
const auto& column_data = assert_cast<const ColumnDateTimeV2&>(*arg_col);
auto res_col = ColumnInt64::create();
auto& res_data = res_col->get_data();
@@ -1074,12 +1145,17 @@ struct LastDayImpl {
if (!ts_value.is_valid_date()) {
throw_out_of_bound_one_date<DateValueType>("last_day", cur_data);
}
- int day = get_last_month_day(ts_value.year(), ts_value.month());
- // day is definitely legal
- if constexpr (std::is_same_v<DateType, DataTypeDateV2>) {
+ if constexpr (PType == TYPE_TIMESTAMP_NS) {
+ const auto civil_value = ts_value.to_date();
+ const int day = get_last_month_day(civil_value.year(), civil_value.month());
+ res_data[i].unchecked_set_time(civil_value.year(), civil_value.month(), day, 0, 0,
+ 0, 0);
+ } else if constexpr (std::is_same_v<DateType, DataTypeDateV2>) {
+ const int day = get_last_month_day(ts_value.year(), ts_value.month());
ts_value.template unchecked_set_time_unit<TimeUnit::DAY>(day);
res_data[i] = ts_value;
- } else { // datetimev2
+ } else if constexpr (std::is_same_v<DateType, DataTypeDateTimeV2>) {
+ const int day = get_last_month_day(ts_value.year(), ts_value.month());
ts_value.template unchecked_set_time_unit<TimeUnit::DAY>(day);
ts_value.unchecked_set_time(ts_value.year(), ts_value.month(), day, 0, 0, 0, 0);
DataTypeDateTimeV2::cast_to_date_v2(ts_value, res_data[i]);
@@ -1153,20 +1229,17 @@ struct ToMondayImpl {
res_data[i] = ts_value;
continue;
}
-
- // day_of_week, from 1(Mon) to 7(Sun)
int day_of_week = ts_value.weekday() + 1;
int gap_of_monday = day_of_week - 1;
TimeInterval interval(DAY, gap_of_monday, true);
ts_value.template date_add_interval<DAY>(interval);
res_data[i] = ts_value;
- } else { // datetimev2
+ } else if constexpr (std::is_same_v<DateType, DataTypeDateTimeV2>) {
if (is_special_day(ts_value.year(), ts_value.month(), ts_value.day())) {
ts_value.unchecked_set_time(ts_value.year(), ts_value.month(), 1, 0, 0, 0, 0);
DataTypeDateTimeV2::cast_to_date_v2(ts_value, res_data[i]);
continue;
}
- // day_of_week, from 1(Mon) to 7(Sun)
int day_of_week = ts_value.weekday() + 1;
int gap_of_monday = day_of_week - 1;
TimeInterval interval(DAY, gap_of_monday, true);
@@ -1174,6 +1247,18 @@ struct ToMondayImpl {
ts_value.unchecked_set_time(ts_value.year(), ts_value.month(), ts_value.day(), 0, 0,
0, 0);
DataTypeDateTimeV2::cast_to_date_v2(ts_value, res_data[i]);
+ } else if constexpr (PType == TYPE_TIMESTAMP_NS) {
+ const int64_t daynr = ts_value.daynr();
+ if (daynr >= TimeStampNsValue::UNIX_EPOCH_DAYNR &&
+ daynr <= TimeStampNsValue::UNIX_EPOCH_DAYNR + 3) {
+ res_data[i].unchecked_set_time(1970, 1, 1, 0, 0, 0, 0);
+ continue;
+ }
+ int day_of_week = ts_value.weekday() + 1;
+ int gap_of_monday = day_of_week - 1;
+ if (!res_data[i].get_date_from_daynr(daynr - gap_of_monday)) {
+ throw_out_of_bound_one_date<DateValueType>("to_monday", cur_data);
+ }
}
}
}
@@ -1224,9 +1309,11 @@ public:
Status open(FunctionContext* context, FunctionContext::FunctionStateScope scope) override {
if constexpr (std::is_same_v<Impl, DateTrunc<TYPE_DATEV2, true>> ||
std::is_same_v<Impl, DateTrunc<TYPE_DATETIMEV2, true>> ||
+ std::is_same_v<Impl, DateTrunc<TYPE_TIMESTAMP_NS, true>> ||
std::is_same_v<Impl, DateTrunc<TYPE_TIMESTAMPTZ, true>> ||
std::is_same_v<Impl, DateTrunc<TYPE_DATEV2, false>> ||
std::is_same_v<Impl, DateTrunc<TYPE_DATETIMEV2, false>> ||
+ std::is_same_v<Impl, DateTrunc<TYPE_TIMESTAMP_NS, false>> ||
std::is_same_v<Impl, DateTrunc<TYPE_TIMESTAMPTZ, false>>) {
return Impl::open(context, scope);
} else {
@@ -1474,12 +1561,16 @@ using FunctionMakeDate = FunctionOtherTypesToDateType<MakeDateImpl>;
using FunctionDateTruncDateV2 = FunctionOtherTypesToDateType<DateTrunc<TYPE_DATEV2, true>>;
using FunctionDateTruncDatetimeV2 = FunctionOtherTypesToDateType<DateTrunc<TYPE_DATETIMEV2, true>>;
+using FunctionDateTruncTimestampNs =
+ FunctionOtherTypesToDateType<DateTrunc<TYPE_TIMESTAMP_NS, true>>;
using FunctionDateTruncTimestamptz =
FunctionOtherTypesToDateType<DateTrunc<TYPE_TIMESTAMPTZ, true>>;
using FunctionDateTruncDateV2WithCommonOrder =
FunctionOtherTypesToDateType<DateTrunc<TYPE_DATEV2, false>>;
using FunctionDateTruncDatetimeV2WithCommonOrder =
FunctionOtherTypesToDateType<DateTrunc<TYPE_DATETIMEV2, false>>;
+using FunctionDateTruncTimestampNsWithCommonOrder =
+ FunctionOtherTypesToDateType<DateTrunc<TYPE_TIMESTAMP_NS, false>>;
using FunctionDateTruncTimestamptzWithCommonOrder =
FunctionOtherTypesToDateType<DateTrunc<TYPE_TIMESTAMPTZ, false>>;
using FunctionFromIso8601DateV2 = FunctionOtherTypesToDateType<FromIso8601DateV2>;
@@ -1492,9 +1583,11 @@ void register_function_timestamp(SimpleFunctionFactory& factory) {
factory.register_function<FromDays>();
factory.register_function<FunctionDateTruncDateV2>();
factory.register_function<FunctionDateTruncDatetimeV2>();
+ factory.register_function<FunctionDateTruncTimestampNs>();
factory.register_function<FunctionDateTruncTimestamptz>();
factory.register_function<FunctionDateTruncDateV2WithCommonOrder>();
factory.register_function<FunctionDateTruncDatetimeV2WithCommonOrder>();
+ factory.register_function<FunctionDateTruncTimestampNsWithCommonOrder>();
factory.register_function<FunctionDateTruncTimestamptzWithCommonOrder>();
factory.register_function<FunctionFromIso8601DateV2>();
@@ -1503,6 +1596,7 @@ void register_function_timestamp(SimpleFunctionFactory& factory) {
factory.register_function<FunctionUnixTimestamp<UnixTimeStampDateImpl<DataTypeDateV2>>>();
factory.register_function<FunctionUnixTimestamp<UnixTimeStampDateImpl<DataTypeDateTime>>>();
factory.register_function<FunctionUnixTimestamp<UnixTimeStampDateImpl<DataTypeDateTimeV2>>>();
+ factory.register_function<FunctionUnixTimestamp<UnixTimeStampDateImpl<DataTypeTimeStampNs>>>();
factory.register_function<FunctionUnixTimestamp<UnixTimeStampStrImpl<>>>();
factory.register_function<FunctionUnixTimestampNew<UnixTimeStampImpl<true>>>();
factory.register_function<
@@ -1513,16 +1607,23 @@ void register_function_timestamp(SimpleFunctionFactory& factory) {
FunctionUnixTimestampNew<UnixTimeStampDateImpl<DataTypeDateTime, true>>>();
factory.register_function<
FunctionUnixTimestampNew<UnixTimeStampDateImpl<DataTypeDateTimeV2, true>>>();
+ factory.register_function<
+ FunctionUnixTimestampNew<UnixTimeStampDateImpl<DataTypeTimeStampNs, true>>>();
factory.register_function<FunctionUnixTimestampNew<UnixTimeStampStrImpl<true>>>();
factory.register_function<FunctionDateOrDateTimeToDate<LastDayImpl, TYPE_DATEV2>>();
factory.register_function<FunctionDateOrDateTimeToDate<LastDayImpl, TYPE_DATETIMEV2>>();
+ factory.register_function<FunctionDateOrDateTimeToDate<LastDayImpl, TYPE_TIMESTAMP_NS>>();
factory.register_function<FunctionDateOrDateTimeToDate<ToMondayImpl, TYPE_DATEV2>>();
factory.register_function<FunctionDateOrDateTimeToDate<ToMondayImpl, TYPE_DATETIMEV2>>();
-
- factory.register_function<DateTimeToTimestamp<MicroSec>>();
- factory.register_function<DateTimeToTimestamp<MilliSec>>();
- factory.register_function<DateTimeToTimestamp<Sec>>();
+ factory.register_function<FunctionDateOrDateTimeToDate<ToMondayImpl, TYPE_TIMESTAMP_NS>>();
+
+ factory.register_function<DateTimeToTimestamp<MicroSec, TYPE_DATETIMEV2>>();
+ factory.register_function<DateTimeToTimestamp<MilliSec, TYPE_DATETIMEV2>>();
+ factory.register_function<DateTimeToTimestamp<Sec, TYPE_DATETIMEV2>>();
+ factory.register_function<DateTimeToTimestamp<MicroSec, TYPE_TIMESTAMP_NS>>();
+ factory.register_function<DateTimeToTimestamp<MilliSec, TYPE_TIMESTAMP_NS>>();
+ factory.register_function<DateTimeToTimestamp<Sec, TYPE_TIMESTAMP_NS>>();
}
#include "common/compile_check_avoid_end.h"
} // namespace doris
diff --git a/be/src/exprs/function/functions_comparison.h b/be/src/exprs/function/functions_comparison.h
index 96ff5f10664..6ae6e5a7b62 100644
--- a/be/src/exprs/function/functions_comparison.h
+++ b/be/src/exprs/function/functions_comparison.h
@@ -41,6 +41,7 @@
#include "core/decimal_comparison.h"
#include "core/field.h"
#include "core/memcmp_small.h"
+#include "core/value/timestamptz_value.h"
#include "core/value/vdatetime_value.h"
#include "exprs/expr_zonemap_filter.h"
#include "exprs/function/function.h"
@@ -643,6 +644,90 @@ private:
return Status::OK();
}
+ template <typename TemporalValue>
+ static int compare_timestamp_ns_with_temporal(const TimeStampNsValue& timestamp,
+ const TemporalValue& temporal) {
+ const auto timestamp_datetime = timestamp.to_datetime();
+ const auto compare_part = [](auto left, auto right) {
+ if (left < right) {
+ return -1;
+ }
+ if (left > right) {
+ return 1;
+ }
+ return 0;
+ };
+ int comparison = compare_part(timestamp_datetime.year(), temporal.year());
+ if (comparison == 0) {
+ comparison = compare_part(timestamp_datetime.month(), temporal.month());
+ }
+ if (comparison == 0) {
+ comparison = compare_part(timestamp_datetime.day(), temporal.day());
+ }
+ if (comparison == 0) {
+ comparison = compare_part(timestamp_datetime.hour(), temporal.hour());
+ }
+ if (comparison == 0) {
+ comparison = compare_part(timestamp_datetime.minute(), temporal.minute());
+ }
+ if (comparison == 0) {
+ comparison = compare_part(timestamp_datetime.second(), temporal.second());
+ }
+ if (comparison == 0) {
+ comparison = compare_part(timestamp.nanosecond(), temporal.microsecond() * 1000);
+ }
+ return comparison;
+ }
+
+ template <PrimitiveType TemporalPType, bool timestamp_ns_on_left>
+ Status execute_timestamp_ns_temporal(FunctionContext* context, Block& block, uint32_t result,
+ const ColumnPtr& left_column,
+ const ColumnPtr& right_column,
+ const DataTypePtr& temporal_type,
+ size_t input_rows_count) const {
+ static_assert(TemporalPType == TYPE_DATE || TemporalPType == TYPE_DATEV2 ||
+ TemporalPType == TYPE_DATETIME || TemporalPType == TYPE_DATETIMEV2 ||
+ TemporalPType == TYPE_TIMESTAMPTZ);
+ const auto& timestamp_column = timestamp_ns_on_left ? left_column : right_column;
+ const auto& temporal_column = timestamp_ns_on_left ? right_column : left_column;
+ auto [timestamp_unpacked, timestamp_is_const] = unpack_if_const(timestamp_column);
+ auto [temporal_unpacked, temporal_is_const] = unpack_if_const(temporal_column);
+ const auto& timestamps =
+ assert_cast<const ColumnTimeStampNs&>(*timestamp_unpacked).get_data();
+ using TemporalColumn = typename PrimitiveTypeTraits<TemporalPType>::ColumnType;
+ using TemporalValue = typename PrimitiveTypeTraits<TemporalPType>::CppType;
+ const auto& temporals = assert_cast<const TemporalColumn&>(*temporal_unpacked).get_data();
+
+ auto result_column = ColumnUInt8::create(input_rows_count);
+ auto& result_data = result_column->get_data();
+ for (size_t row = 0; row < input_rows_count; ++row) {
+ const auto& timestamp = timestamps[timestamp_is_const ? 0 : row];
+ const auto& temporal =
+ reinterpret_cast<const TemporalValue&>(temporals[temporal_is_const ? 0 : row]);
+ int comparison;
+ if constexpr (TemporalPType == TYPE_TIMESTAMPTZ) {
+ DateV2Value<DateTimeV2ValueType> local_datetime;
+ const auto scale = temporal_type->get_scale();
+ if (!temporal.to_datetime(local_datetime, context->state()->timezone_obj(), scale,
+ scale)) [[unlikely]] {
+ return Status::InvalidArgument(
+ "can not compare timestamptz {} with TIMESTAMP_NS in timezone {}",
+ temporal.to_string(context->state()->timezone_obj(), scale),
+ context->state()->timezone());
+ }
+ comparison = compare_timestamp_ns_with_temporal(timestamp, local_datetime);
+ } else {
+ comparison = compare_timestamp_ns_with_temporal(timestamp, temporal);
+ }
+ if constexpr (!timestamp_ns_on_left) {
+ comparison = -comparison;
+ }
+ result_data[row] = Op<TYPE_INT>::apply(comparison, 0);
+ }
+ block.replace_by_position(result, std::move(result_column));
+ return Status::OK();
+ }
+
public:
String get_name() const override { return name; }
@@ -761,6 +846,9 @@ public:
return Status::OK();
}
+ // Keep all primitive-type pairs in one dispatch point so every comparison operator shares the
+ // same mixed TIMESTAMP_NS semantics.
+ // NOLINTNEXTLINE(readability-function-size)
Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments,
uint32_t result, size_t input_rows_count) const override {
const auto& col_with_type_and_name_left = block.get_by_position(arguments[0]);
@@ -774,6 +862,27 @@ public:
const DataTypePtr& left_type = col_with_type_and_name_left.type;
const DataTypePtr& right_type = col_with_type_and_name_right.type;
+#define EXECUTE_TIMESTAMP_NS_TEMPORAL_COMPARISON(TYPE) \
+ if (left_type->get_primitive_type() == TYPE_TIMESTAMP_NS && \
+ right_type->get_primitive_type() == TYPE) { \
+ return execute_timestamp_ns_temporal<TYPE, true>(context, block, result, col_left_ptr, \
+ col_right_ptr, right_type, \
+ input_rows_count); \
+ } \
+ if (left_type->get_primitive_type() == TYPE && \
+ right_type->get_primitive_type() == TYPE_TIMESTAMP_NS) { \
+ return execute_timestamp_ns_temporal<TYPE, false>( \
+ context, block, result, col_left_ptr, col_right_ptr, left_type, input_rows_count); \
+ }
+
+ EXECUTE_TIMESTAMP_NS_TEMPORAL_COMPARISON(TYPE_DATE)
+ EXECUTE_TIMESTAMP_NS_TEMPORAL_COMPARISON(TYPE_DATEV2)
+ EXECUTE_TIMESTAMP_NS_TEMPORAL_COMPARISON(TYPE_DATETIME)
+ EXECUTE_TIMESTAMP_NS_TEMPORAL_COMPARISON(TYPE_DATETIMEV2)
+ EXECUTE_TIMESTAMP_NS_TEMPORAL_COMPARISON(TYPE_TIMESTAMPTZ)
+
+#undef EXECUTE_TIMESTAMP_NS_TEMPORAL_COMPARISON
+
/// The case when arguments are the same (tautological comparison). Return constant.
/// NOTE: Nullable types are special case. (BTW, this function use default implementation for Nullable, so Nullable types cannot be here. Check just in case.)
/// NOTE: We consider NaN comparison to be implementation specific (and in our implementation NaNs are sometimes equal sometimes not).
@@ -801,7 +910,8 @@ public:
}
auto can_compare = [](PrimitiveType t) -> bool {
- return is_int_or_bool(t) || is_float_or_double(t) || is_ip(t) || is_date_type(t);
+ return is_int_or_bool(t) || is_float_or_double(t) || is_ip(t) || is_date_type(t) ||
+ is_timestamp_ns_type(t);
};
if (can_compare(left_type->get_primitive_type()) &&
@@ -822,6 +932,8 @@ public:
return execute_num_type<TYPE_DATEV2>(block, result, col_left_ptr, col_right_ptr);
case TYPE_DATETIMEV2:
return execute_num_type<TYPE_DATETIMEV2>(block, result, col_left_ptr, col_right_ptr);
+ case TYPE_TIMESTAMP_NS:
+ return execute_num_type<TYPE_TIMESTAMP_NS>(block, result, col_left_ptr, col_right_ptr);
case TYPE_TIMESTAMPTZ:
return execute_num_type<TYPE_TIMESTAMPTZ>(block, result, col_left_ptr, col_right_ptr);
case TYPE_TINYINT:
diff --git a/be/src/exprs/function/least_greast.cpp b/be/src/exprs/function/least_greast.cpp
index 13558035636..d8005d3df7a 100644
--- a/be/src/exprs/function/least_greast.cpp
+++ b/be/src/exprs/function/least_greast.cpp
@@ -113,7 +113,13 @@ struct CompareMultiImpl {
return true;
};
- if (!dispatch_switch_scalar(data_type->get_primitive_type(), call)) {
+ bool supported = false;
+ if (data_type->get_primitive_type() == TYPE_TIMESTAMP_NS) {
+ supported = call(DispatchDataType<TYPE_TIMESTAMP_NS>());
+ } else {
+ supported = dispatch_switch_scalar(data_type->get_primitive_type(), call);
+ }
+ if (!supported) {
throw doris::Exception(ErrorCode::INTERNAL_ERROR, "not support type {}",
data_type->get_name());
}
@@ -304,6 +310,13 @@ struct FunctionFieldImpl {
}
break;
}
+ case PrimitiveType::TYPE_TIMESTAMP_NS: {
+ for (int col = 1; col < arguments.size(); ++col) {
+ insert_result_data<TYPE_TIMESTAMP_NS>(res_data, argument_columns[0],
+ argument_columns[col], input_rows_count, col);
+ }
+ break;
+ }
case PrimitiveType::TYPE_TIMESTAMPTZ: {
for (int col = 1; col < arguments.size(); ++col) {
insert_result_data<TYPE_TIMESTAMPTZ>(res_data, argument_columns[0],
@@ -367,4 +380,4 @@ void register_function_least_greast(SimpleFunctionFactory& factory) {
factory.register_function<FunctionGreaest>();
factory.register_function<FunctionField>();
}
-}; // namespace doris
\ No newline at end of file
+}; // namespace doris
diff --git a/be/src/exprs/function/time_of_function.cpp b/be/src/exprs/function/time_of_function.cpp
index 6162477f385..5a1bf52cfca 100644
--- a/be/src/exprs/function/time_of_function.cpp
+++ b/be/src/exprs/function/time_of_function.cpp
@@ -51,6 +51,19 @@ using FunctionDateTimeV2YearWeek =
using FunctionDateTimeV2WeekDay =
FunctionDateOrDateTimeToSomething<DataTypeInt8, WeekDayImpl<TYPE_DATETIMEV2>>;
+using FunctionTimestampNsWeekOfYear =
+ FunctionDateOrDateTimeToSomething<DataTypeInt8, WeekOfYearImpl<TYPE_TIMESTAMP_NS>>;
+using FunctionTimestampNsDayOfYear =
+ FunctionDateOrDateTimeToSomething<DataTypeInt16, DayOfYearImpl<TYPE_TIMESTAMP_NS>>;
+using FunctionTimestampNsDayOfWeek =
+ FunctionDateOrDateTimeToSomething<DataTypeInt8, DayOfWeekImpl<TYPE_TIMESTAMP_NS>>;
+using FunctionTimestampNsDayOfMonth =
+ FunctionDateOrDateTimeToSomething<DataTypeInt8, DayOfMonthImpl<TYPE_TIMESTAMP_NS>>;
+using FunctionTimestampNsYearWeek =
+ FunctionDateOrDateTimeToSomething<DataTypeInt32, ToYearWeekOneArgImpl<TYPE_TIMESTAMP_NS>>;
+using FunctionTimestampNsWeekDay =
+ FunctionDateOrDateTimeToSomething<DataTypeInt8, WeekDayImpl<TYPE_TIMESTAMP_NS>>;
+
void register_function_time_of_function(SimpleFunctionFactory& factory) {
factory.register_function<FunctionDayOfWeekV2>();
factory.register_function<FunctionDayOfMonthV2>();
@@ -64,5 +77,11 @@ void register_function_time_of_function(SimpleFunctionFactory& factory) {
factory.register_function<FunctionDateTimeV2DayOfMonth>();
factory.register_function<FunctionDateTimeV2YearWeek>();
factory.register_function<FunctionDateTimeV2WeekDay>();
+ factory.register_function<FunctionTimestampNsWeekOfYear>();
+ factory.register_function<FunctionTimestampNsDayOfYear>();
+ factory.register_function<FunctionTimestampNsDayOfWeek>();
+ factory.register_function<FunctionTimestampNsDayOfMonth>();
+ factory.register_function<FunctionTimestampNsYearWeek>();
+ factory.register_function<FunctionTimestampNsWeekDay>();
}
} // namespace doris
diff --git a/be/src/exprs/function/to_time_function.cpp b/be/src/exprs/function/to_time_function.cpp
index 2f4b6491ceb..557988dc069 100644
--- a/be/src/exprs/function/to_time_function.cpp
+++ b/be/src/exprs/function/to_time_function.cpp
@@ -38,6 +38,8 @@ using FunctionWeekV2 =
using FunctionHourV2 = FunctionDateOrDateTimeToSomething<DataTypeInt8, ToHourImpl<TYPE_DATEV2>>;
using FunctionMinuteV2 = FunctionDateOrDateTimeToSomething<DataTypeInt8, ToMinuteImpl<TYPE_DATEV2>>;
using FunctionSecondV2 = FunctionDateOrDateTimeToSomething<DataTypeInt8, ToSecondImpl<TYPE_DATEV2>>;
+using FunctionNanoSecondV2 =
+ FunctionDateOrDateTimeToSomething<DataTypeInt32, ToNanoSecondImpl<TYPE_DATEV2>>;
using FunctionToDaysV2 = FunctionDateOrDateTimeToSomething<DataTypeInt32, ToDaysImpl<TYPE_DATEV2>>;
using FunctionToDateV2 = FunctionDateOrDateTimeToSomething<DataTypeDateV2, ToDateImpl<TYPE_DATEV2>>;
using FunctionDateV2 = FunctionDateOrDateTimeToSomething<DataTypeDateV2, DateImpl<TYPE_DATEV2>>;
@@ -62,6 +64,8 @@ using FunctionDateTimeV2Second =
FunctionDateOrDateTimeToSomething<DataTypeInt8, ToSecondImpl<TYPE_DATETIMEV2>>;
using FunctionDateTimeV2MicroSecond =
FunctionDateOrDateTimeToSomething<DataTypeInt32, ToMicroSecondImpl<TYPE_DATETIMEV2>>;
+using FunctionDateTimeV2NanoSecond =
+ FunctionDateOrDateTimeToSomething<DataTypeInt32, ToNanoSecondImpl<TYPE_DATETIMEV2>>;
using FunctionDateTimeV2ToDate =
FunctionDateOrDateTimeToSomething<DataTypeDateV2, ToDateImpl<TYPE_DATETIMEV2>>;
using FunctionDateTimeV2Date =
@@ -72,9 +76,46 @@ using FunctionCenturyV2 =
FunctionDateOrDateTimeToSomething<DataTypeInt16, ToCenturyImpl<TYPE_DATEV2>>;
using FunctionDateTimeV2Century =
FunctionDateOrDateTimeToSomething<DataTypeInt16, ToCenturyImpl<TYPE_DATETIMEV2>>;
+
+using FunctionTimestampNsYear =
+ FunctionDateOrDateTimeToSomething<DataTypeInt16, ToYearImpl<TYPE_TIMESTAMP_NS>>;
+using FunctionTimestampNsYearOfWeek =
+ FunctionDateOrDateTimeToSomething<DataTypeInt16, ToYearOfWeekImpl<TYPE_TIMESTAMP_NS>>;
+using FunctionTimestampNsQuarter =
+ FunctionDateOrDateTimeToSomething<DataTypeInt8, ToQuarterImpl<TYPE_TIMESTAMP_NS>>;
+using FunctionTimestampNsMonth =
+ FunctionDateOrDateTimeToSomething<DataTypeInt8, ToMonthImpl<TYPE_TIMESTAMP_NS>>;
+using FunctionTimestampNsDay =
+ FunctionDateOrDateTimeToSomething<DataTypeInt8, ToDayImpl<TYPE_TIMESTAMP_NS>>;
+using FunctionTimestampNsWeek =
+ FunctionDateOrDateTimeToSomething<DataTypeInt8, ToWeekOneArgImpl<TYPE_TIMESTAMP_NS>>;
+using FunctionTimestampNsHour =
+ FunctionDateOrDateTimeToSomething<DataTypeInt8, ToHourImpl<TYPE_TIMESTAMP_NS>>;
+using FunctionTimestampNsMinute =
+ FunctionDateOrDateTimeToSomething<DataTypeInt8, ToMinuteImpl<TYPE_TIMESTAMP_NS>>;
+using FunctionTimestampNsSecond =
+ FunctionDateOrDateTimeToSomething<DataTypeInt8, ToSecondImpl<TYPE_TIMESTAMP_NS>>;
+using FunctionTimestampNsToDays =
+ FunctionDateOrDateTimeToSomething<DataTypeInt32, ToDaysImpl<TYPE_TIMESTAMP_NS>>;
+using FunctionTimestampNsMicroSecond =
+ FunctionDateOrDateTimeToSomething<DataTypeInt32, ToMicroSecondImpl<TYPE_TIMESTAMP_NS>>;
+using FunctionTimestampNsNanoSecond =
+ FunctionDateOrDateTimeToSomething<DataTypeInt32, ToNanoSecondImpl<TYPE_TIMESTAMP_NS>>;
+using FunctionTimestampNsToDate =
+ FunctionDateOrDateTimeToSomething<DataTypeDateV2, ToDateImpl<TYPE_TIMESTAMP_NS>>;
+using FunctionTimestampNsDate =
+ FunctionDateOrDateTimeToSomething<DataTypeDateV2, DateImpl<TYPE_TIMESTAMP_NS>>;
+using FunctionTimestampNsTimestamp =
+ FunctionDateOrDateTimeToSomething<DataTypeTimeStampNs, TimeStampImpl<TYPE_TIMESTAMP_NS>>;
+using FunctionTimestampNsCentury =
+ FunctionDateOrDateTimeToSomething<DataTypeInt16, ToCenturyImpl<TYPE_TIMESTAMP_NS>>;
+using FunctionTimestampNsToSeconds =
+ FunctionDateOrDateTimeToSomething<DataTypeInt64, ToSecondsImpl<TYPE_TIMESTAMP_NS>>;
+
void register_function_to_time_function(SimpleFunctionFactory& factory) {
factory.register_function<FunctionTimeStampV2>();
factory.register_function<FunctionSecondV2>();
+ factory.register_function<FunctionNanoSecondV2>();
factory.register_function<FunctionMinuteV2>();
factory.register_function<FunctionHourV2>();
factory.register_function<FunctionDayV2>();
@@ -87,6 +128,7 @@ void register_function_to_time_function(SimpleFunctionFactory& factory) {
factory.register_function<FunctionToDateV2>();
factory.register_function<FunctionDateV2>();
factory.register_function<FunctionDateTimeV2MicroSecond>();
+ factory.register_function<FunctionDateTimeV2NanoSecond>();
factory.register_function<FunctionDateTimeV2Second>();
factory.register_function<FunctionDateTimeV2Minute>();
factory.register_function<FunctionDateTimeV2Hour>();
@@ -100,6 +142,23 @@ void register_function_to_time_function(SimpleFunctionFactory& factory) {
factory.register_function<FunctionCenturyV2>();
factory.register_function<FunctionDateTimeV2Century>();
factory.register_function<FunctionToSeconds>();
+ factory.register_function<FunctionTimestampNsTimestamp>();
+ factory.register_function<FunctionTimestampNsMicroSecond>();
+ factory.register_function<FunctionTimestampNsNanoSecond>();
+ factory.register_function<FunctionTimestampNsSecond>();
+ factory.register_function<FunctionTimestampNsToDays>();
+ factory.register_function<FunctionTimestampNsMinute>();
+ factory.register_function<FunctionTimestampNsHour>();
+ factory.register_function<FunctionTimestampNsDay>();
+ factory.register_function<FunctionTimestampNsWeek>();
+ factory.register_function<FunctionTimestampNsMonth>();
+ factory.register_function<FunctionTimestampNsYear>();
+ factory.register_function<FunctionTimestampNsYearOfWeek>();
+ factory.register_function<FunctionTimestampNsQuarter>();
+ factory.register_function<FunctionTimestampNsToDate>();
+ factory.register_function<FunctionTimestampNsDate>();
+ factory.register_function<FunctionTimestampNsCentury>();
+ factory.register_function<FunctionTimestampNsToSeconds>();
factory.register_alias("date", "datev2");
factory.register_alias("to_date", "to_datev2");
}
diff --git a/be/src/exprs/function_context.cpp b/be/src/exprs/function_context.cpp
index 84ad5509ca5..6be7b6f8c3b 100644
--- a/be/src/exprs/function_context.cpp
+++ b/be/src/exprs/function_context.cpp
@@ -57,6 +57,7 @@ std::unique_ptr<FunctionContext> FunctionContext::clone() {
new_context->_fragment_local_fn_state = _fragment_local_fn_state;
new_context->_check_overflow_for_decimal = _check_overflow_for_decimal;
new_context->_enable_strict_mode = _enable_strict_mode;
+ new_context->_is_auto_partition_boundary_context = _is_auto_partition_boundary_context;
new_context->_string_as_jsonb_string = _string_as_jsonb_string;
new_context->_jsonb_string_as_string = _jsonb_string_as_string;
return new_context;
diff --git a/be/src/exprs/function_context.h b/be/src/exprs/function_context.h
index ca0f1ed4610..69d6dfa6187 100644
--- a/be/src/exprs/function_context.h
+++ b/be/src/exprs/function_context.h
@@ -84,6 +84,8 @@ public:
bool enable_strict_mode() const { return _enable_strict_mode; }
+ bool is_auto_partition_boundary_context() const { return _is_auto_partition_boundary_context; }
+
bool set_check_overflow_for_decimal(bool check_overflow_for_decimal) {
return _check_overflow_for_decimal = check_overflow_for_decimal;
}
@@ -92,6 +94,8 @@ public:
return _enable_strict_mode = enable_strict_mode;
}
+ void set_auto_partition_boundary_context() { _is_auto_partition_boundary_context = true; }
+
void set_string_as_jsonb_string(bool string_as_jsonb_string) {
_string_as_jsonb_string = string_as_jsonb_string;
}
@@ -198,6 +202,7 @@ private:
RuntimeProfile::Counter* _udf_execute_timer = nullptr;
bool _check_overflow_for_decimal = false;
bool _enable_strict_mode = false;
+ bool _is_auto_partition_boundary_context = false;
bool _string_as_jsonb_string = false;
bool _jsonb_string_as_string = false;
diff --git a/be/src/exprs/lambda_function/varray_sort_function.cpp b/be/src/exprs/lambda_function/varray_sort_function.cpp
index c8a3d6ef1f4..9ae72ad2c63 100644
--- a/be/src/exprs/lambda_function/varray_sort_function.cpp
+++ b/be/src/exprs/lambda_function/varray_sort_function.cpp
@@ -53,7 +53,7 @@ using ConstColumnVariant =
const ColumnDecimal32*, const ColumnDecimal64*, const ColumnDecimal128V2*,
const ColumnDecimal128V3*, const ColumnDecimal256*, const ColumnDate*,
const ColumnDateTime*, const ColumnDateV2*, const ColumnDateTimeV2*,
- const ColumnTime*, const ColumnTimeV2*>;
+ const ColumnTime*, const ColumnTimeStampNs*, const ColumnTimeV2*>;
template <typename T>
struct is_column_vector : std::false_type {};
@@ -431,6 +431,7 @@ private:
DISPATCH_PRIMITIVE_TYPE(TYPE_DATEV2, ColumnDateV2)
DISPATCH_PRIMITIVE_TYPE(TYPE_DATETIMEV2, ColumnDateTimeV2)
DISPATCH_PRIMITIVE_TYPE(TYPE_TIME, ColumnTime)
+ DISPATCH_PRIMITIVE_TYPE(TYPE_TIMESTAMP_NS, ColumnTimeStampNs)
DISPATCH_PRIMITIVE_TYPE(TYPE_TIMEV2, ColumnTimeV2)
default:
return Status::InternalError("Unsupported type in array_sort");
diff --git a/be/src/exprs/vcase_expr.h b/be/src/exprs/vcase_expr.h
index 026715d9aff..465a879fdc8 100644
--- a/be/src/exprs/vcase_expr.h
+++ b/be/src/exprs/vcase_expr.h
@@ -143,6 +143,7 @@ private:
CASE_TYPE(TYPE_DATETIME, ColumnDateTime)
CASE_TYPE(TYPE_DATEV2, ColumnDateV2)
CASE_TYPE(TYPE_DATETIMEV2, ColumnDateTimeV2)
+ CASE_TYPE(TYPE_TIMESTAMP_NS, ColumnTimeStampNs)
CASE_TYPE(TYPE_TIMESTAMPTZ, ColumnTimeStampTz)
CASE_TYPE(TYPE_IPV6, ColumnIPv6)
CASE_TYPE(TYPE_IPV4, ColumnIPv4)
@@ -231,6 +232,7 @@ private:
std::is_same_v<ColumnType, ColumnDateTime> ||
std::is_same_v<ColumnType, ColumnDateV2> ||
std::is_same_v<ColumnType, ColumnDateTimeV2> ||
+ std::is_same_v<ColumnType, ColumnTimeStampNs> ||
std::is_same_v<ColumnType, ColumnTimeStampTz>) {
result_raw_data[i] = ColumnType::default_value();
} else {
@@ -251,6 +253,7 @@ private:
std::is_same_v<ColumnType, ColumnDateTime> ||
std::is_same_v<ColumnType, ColumnDateV2> ||
std::is_same_v<ColumnType, ColumnDateTimeV2> ||
+ std::is_same_v<ColumnType, ColumnTimeStampNs> ||
std::is_same_v<ColumnType, ColumnTimeStampTz>) {
for (int row_idx = 0; row_idx < rows_count; row_idx++) {
result_raw_data[row_idx] = (then_idx[row_idx] == i) ? column_raw_data[row_idx]
diff --git a/be/src/exprs/vcast_expr.cpp b/be/src/exprs/vcast_expr.cpp
index 723a398e2bd..755be44b545 100644
--- a/be/src/exprs/vcast_expr.cpp
+++ b/be/src/exprs/vcast_expr.cpp
@@ -91,6 +91,9 @@ doris::Status VCastExpr::open(doris::RuntimeState* state, VExprContext* context,
for (auto& i : _children) {
RETURN_IF_ERROR(i->open(state, context, scope));
}
+ if (_is_strict_cast) {
+ context->fn_context(_fn_context_index)->set_enable_strict_mode(true);
+ }
RETURN_IF_ERROR(VExpr::init_function_context(state, context, scope, _function));
if (scope == FunctionContext::FRAGMENT_LOCAL) {
RETURN_IF_ERROR(VExpr::get_const_col(context, nullptr));
diff --git a/be/src/exprs/vcast_expr.h b/be/src/exprs/vcast_expr.h
index a9e01980797..abed5e3d3b1 100644
--- a/be/src/exprs/vcast_expr.h
+++ b/be/src/exprs/vcast_expr.h
@@ -44,7 +44,8 @@ public:
#ifdef BE_TEST
VCastExpr() = default;
#endif
- VCastExpr(const TExprNode& node) : VExpr(node) {}
+ VCastExpr(const TExprNode& node)
+ : VExpr(node), _is_strict_cast(node.__isset.is_strict_cast && node.is_strict_cast) {}
~VCastExpr() override = default;
Status execute_column(VExprContext* context, const Block* block, Selector* selector,
size_t count, ColumnPtr& result_column) const override;
@@ -67,8 +68,9 @@ public:
uint64_t get_digest(uint64_t seed) const override {
auto res = VExpr::get_digest(seed);
if (res) {
- return HashUtil::hash64(_target_data_type_name.data(), _target_data_type_name.size(),
- res);
+ res = HashUtil::hash64(_target_data_type_name.data(), _target_data_type_name.size(),
+ res);
+ return HashUtil::hash64(&_is_strict_cast, sizeof(_is_strict_cast), res);
}
return 0;
}
@@ -83,6 +85,8 @@ private:
DataTypePtr _cast_param_data_type;
+ bool _is_strict_cast = false;
+
static const constexpr char* function_name = "CAST";
};
diff --git a/be/src/exprs/vcondition_expr.cpp b/be/src/exprs/vcondition_expr.cpp
index d126511b033..f4066b06940 100644
--- a/be/src/exprs/vcondition_expr.cpp
+++ b/be/src/exprs/vcondition_expr.cpp
@@ -575,6 +575,11 @@ void insert_result_data(MutableColumnPtr& result_column, ColumnPtr& argument_col
result_raw_data[row].to_date_int_val() +
column_raw_data[row].to_date_int_val() *
uint64_t(!(null_map_data[row] | filled_flag[row])));
+ } else if constexpr (std::is_same_v<ColumnType, ColumnTimeStampNs>) {
+ result_raw_data[row] =
+ TimeStampNsValue(result_raw_data[row].epoch_nanos() +
+ column_raw_data[row].epoch_nanos() *
+ int64_t(!(null_map_data[row] | filled_flag[row])));
} else if constexpr (std::is_same_v<ColumnType, ColumnTimeStampTz>) {
result_raw_data[row] = binary_cast<uint64_t, TimestampTzValue>(
result_raw_data[row].to_date_int_val() +
diff --git a/be/src/exprs/vdirect_in_predicate.h b/be/src/exprs/vdirect_in_predicate.h
index ace2ac513a2..0eaba5b3437 100644
--- a/be/src/exprs/vdirect_in_predicate.h
+++ b/be/src/exprs/vdirect_in_predicate.h
@@ -267,6 +267,7 @@ private:
RETURN_RAW_FIXED_SIZE(TYPE_DATETIME);
RETURN_RAW_FIXED_SIZE(TYPE_DATEV2);
RETURN_RAW_FIXED_SIZE(TYPE_DATETIMEV2);
+ RETURN_RAW_FIXED_SIZE(TYPE_TIMESTAMP_NS);
RETURN_RAW_FIXED_SIZE(TYPE_TIMESTAMPTZ);
RETURN_RAW_FIXED_SIZE(TYPE_TIMEV2);
RETURN_RAW_FIXED_SIZE(TYPE_DECIMAL32);
diff --git a/be/src/exprs/vectorized_fn_call.cpp b/be/src/exprs/vectorized_fn_call.cpp
index 160679d3563..57d3b8006d6 100644
--- a/be/src/exprs/vectorized_fn_call.cpp
+++ b/be/src/exprs/vectorized_fn_call.cpp
@@ -198,6 +198,7 @@ size_t raw_comparison_value_size(PrimitiveType primitive_type) {
RETURN_RAW_COMPARISON_SIZE(TYPE_DATETIME);
RETURN_RAW_COMPARISON_SIZE(TYPE_DATEV2);
RETURN_RAW_COMPARISON_SIZE(TYPE_DATETIMEV2);
+ RETURN_RAW_COMPARISON_SIZE(TYPE_TIMESTAMP_NS);
RETURN_RAW_COMPARISON_SIZE(TYPE_TIMESTAMPTZ);
RETURN_RAW_COMPARISON_SIZE(TYPE_TIMEV2);
RETURN_RAW_COMPARISON_SIZE(TYPE_DECIMAL32);
@@ -539,6 +540,7 @@ Status VectorizedFnCall::execute_on_raw_fixed_values(const uint8_t* values, size
EXECUTE_RAW_SCALAR_COMPARISON(TYPE_DATETIME);
EXECUTE_RAW_SCALAR_COMPARISON(TYPE_DATEV2);
EXECUTE_RAW_SCALAR_COMPARISON(TYPE_DATETIMEV2);
+ EXECUTE_RAW_SCALAR_COMPARISON(TYPE_TIMESTAMP_NS);
EXECUTE_RAW_SCALAR_COMPARISON(TYPE_TIMESTAMPTZ);
EXECUTE_RAW_SCALAR_COMPARISON(TYPE_TIMEV2);
EXECUTE_RAW_SCALAR_COMPARISON(TYPE_DECIMAL32);
diff --git a/be/src/exprs/vexpr.cpp b/be/src/exprs/vexpr.cpp
index c7773074630..6a4d3410266 100644
--- a/be/src/exprs/vexpr.cpp
+++ b/be/src/exprs/vexpr.cpp
@@ -138,6 +138,10 @@ TExprNode create_texpr_node_from(const void* data, const PrimitiveType& type, in
THROW_IF_ERROR(create_texpr_literal_node<TYPE_DATETIMEV2>(data, &node, precision, scale));
break;
}
+ case TYPE_TIMESTAMP_NS: {
+ THROW_IF_ERROR(create_texpr_literal_node<TYPE_TIMESTAMP_NS>(data, &node));
+ break;
+ }
case TYPE_DATE: {
THROW_IF_ERROR(create_texpr_literal_node<TYPE_DATE>(data, &node));
break;
@@ -257,6 +261,11 @@ TExprNode create_texpr_node_from(const Field& field, const PrimitiveType& type,
create_texpr_literal_node<TYPE_DATETIMEV2>(&storage, &node, precision, scale));
break;
}
+ case TYPE_TIMESTAMP_NS: {
+ const auto& storage = field.get<TYPE_TIMESTAMP_NS>();
+ THROW_IF_ERROR(create_texpr_literal_node<TYPE_TIMESTAMP_NS>(&storage, &node));
+ break;
+ }
case TYPE_TIMESTAMPTZ: {
const auto& storage = field.get<TYPE_TIMESTAMPTZ>();
diff --git a/be/src/exprs/vexpr.h b/be/src/exprs/vexpr.h
index 5cd73d77f4b..59c67d30c55 100644
--- a/be/src/exprs/vexpr.h
+++ b/be/src/exprs/vexpr.h
@@ -630,6 +630,15 @@ Status create_texpr_literal_node(const void* data, TExprNode* node, int precisio
(*node).__set_date_literal(date_literal);
(*node).__set_node_type(TExprNodeType::DATE_LITERAL);
(*node).__set_type(create_type_desc(PrimitiveType::TYPE_DATETIMEV2, precision, scale));
+ } else if constexpr (T == TYPE_TIMESTAMP_NS) {
+ const auto* origin_value = reinterpret_cast<const TimeStampNsValue*>(data);
+ TDateLiteral date_literal;
+ char convert_buffer[30];
+ origin_value->to_string(convert_buffer);
+ date_literal.__set_value(convert_buffer);
+ (*node).__set_date_literal(date_literal);
+ (*node).__set_node_type(TExprNodeType::DATE_LITERAL);
+ (*node).__set_type(create_type_desc(PrimitiveType::TYPE_TIMESTAMP_NS));
} else if constexpr (T == TYPE_TIMESTAMPTZ) {
const auto* origin_value = reinterpret_cast<const TimestampTzValue*>(data);
TDateLiteral date_literal;
diff --git a/be/src/exprs/vexpr_context.h b/be/src/exprs/vexpr_context.h
index abaee0c5b0d..5433f9db9ba 100644
--- a/be/src/exprs/vexpr_context.h
+++ b/be/src/exprs/vexpr_context.h
@@ -299,6 +299,12 @@ public:
return _fn_contexts[i].get();
}
+ void set_auto_partition_boundary_context() {
+ for (auto& fn_context : _fn_contexts) {
+ fn_context->set_auto_partition_boundary_context();
+ }
+ }
+
// execute expr with inverted index which column a, b has inverted indexes
// but some situation although column b has indexes, but apply index is not useful, we should
// skip this expr, just do not apply index anymore.
diff --git a/be/src/exprs/vin_predicate.cpp b/be/src/exprs/vin_predicate.cpp
index 73a14f053e4..fde81ba10e8 100644
--- a/be/src/exprs/vin_predicate.cpp
+++ b/be/src/exprs/vin_predicate.cpp
@@ -65,6 +65,7 @@ size_t raw_in_value_size(PrimitiveType primitive_type) {
RETURN_RAW_IN_SIZE(TYPE_DATETIME);
RETURN_RAW_IN_SIZE(TYPE_DATEV2);
RETURN_RAW_IN_SIZE(TYPE_DATETIMEV2);
+ RETURN_RAW_IN_SIZE(TYPE_TIMESTAMP_NS);
RETURN_RAW_IN_SIZE(TYPE_TIMESTAMPTZ);
RETURN_RAW_IN_SIZE(TYPE_TIMEV2);
RETURN_RAW_IN_SIZE(TYPE_DECIMAL32);
diff --git a/be/src/exprs/vtopn_pred.cpp b/be/src/exprs/vtopn_pred.cpp
index 973f8db1886..e99c309387b 100644
--- a/be/src/exprs/vtopn_pred.cpp
+++ b/be/src/exprs/vtopn_pred.cpp
@@ -45,6 +45,7 @@ size_t topn_raw_value_size(PrimitiveType type) {
RETURN_TOPN_RAW_SIZE(TYPE_DATETIME);
RETURN_TOPN_RAW_SIZE(TYPE_DATEV2);
RETURN_TOPN_RAW_SIZE(TYPE_DATETIMEV2);
+ RETURN_TOPN_RAW_SIZE(TYPE_TIMESTAMP_NS);
RETURN_TOPN_RAW_SIZE(TYPE_TIMESTAMPTZ);
RETURN_TOPN_RAW_SIZE(TYPE_TIME);
RETURN_TOPN_RAW_SIZE(TYPE_TIMEV2);
@@ -159,6 +160,7 @@ Status VTopNPred::execute_on_raw_fixed_values(const uint8_t* values, size_t num_
EXECUTE_TOPN_RAW_SCALAR(TYPE_DATETIME);
EXECUTE_TOPN_RAW_SCALAR(TYPE_DATEV2);
EXECUTE_TOPN_RAW_SCALAR(TYPE_DATETIMEV2);
+ EXECUTE_TOPN_RAW_SCALAR(TYPE_TIMESTAMP_NS);
EXECUTE_TOPN_RAW_SCALAR(TYPE_TIMESTAMPTZ);
EXECUTE_TOPN_RAW_SCALAR(TYPE_TIME);
EXECUTE_TOPN_RAW_SCALAR(TYPE_TIMEV2);
diff --git a/be/src/format/arrow/arrow_row_batch.cpp b/be/src/format/arrow/arrow_row_batch.cpp
index 9589f6e59ac..200261ffab7 100644
--- a/be/src/format/arrow/arrow_row_batch.cpp
+++ b/be/src/format/arrow/arrow_row_batch.cpp
@@ -113,6 +113,10 @@ Status DorisArrowSchemaConvertor::convert_to_arrow_type(
case TYPE_DATEV2:
*result = std::make_shared<arrow::Date32Type>();
break;
+ case TYPE_TIMESTAMP_NS:
+ // TIMESTAMP_NS is stored as signed epoch nanoseconds, but its SQL type has no timezone.
+ *result = std::make_shared<arrow::TimestampType>(arrow::TimeUnit::NANO);
+ break;
case TYPE_TIMESTAMPTZ:
case TYPE_DATETIMEV2: {
arrow::TimeUnit::type time_unit;
diff --git a/be/src/format/jni/jni_data_bridge.cpp b/be/src/format/jni/jni_data_bridge.cpp
index cd4f20b0bbd..9899901e38a 100644
--- a/be/src/format/jni/jni_data_bridge.cpp
+++ b/be/src/format/jni/jni_data_bridge.cpp
@@ -61,6 +61,7 @@ namespace doris {
M(PrimitiveType::TYPE_DATEV2, ColumnDateV2, UInt32) \
M(PrimitiveType::TYPE_DATETIME, ColumnDateTime, Int64) \
M(PrimitiveType::TYPE_DATETIMEV2, ColumnDateTimeV2, UInt64) \
+ M(PrimitiveType::TYPE_TIMESTAMP_NS, ColumnTimeStampNs, Int64) \
M(PrimitiveType::TYPE_TIMESTAMPTZ, ColumnTimeStampTz, UInt64) \
M(PrimitiveType::TYPE_IPV4, ColumnIPv4, IPv4) \
M(PrimitiveType::TYPE_IPV6, ColumnIPv6, IPv6)
@@ -335,6 +336,8 @@ std::string JniDataBridge::get_jni_type(const DataTypePtr& data_type) {
buffer << "datetimev2(" << type->get_scale() << ")";
return buffer.str();
}
+ case TYPE_TIMESTAMP_NS:
+ return "timestamp_ns";
case TYPE_TIMESTAMPTZ: {
buffer << "timestamptz(" << type->get_scale() << ")";
return buffer.str();
@@ -441,6 +444,8 @@ std::string JniDataBridge::get_jni_type_with_different_string(const DataTypePtr&
buffer << "datetimev2(" << data_type->get_scale() << ")";
return buffer.str();
}
+ case TYPE_TIMESTAMP_NS:
+ return "timestamp_ns";
case TYPE_TIMESTAMPTZ: {
buffer << "timestamptz(" << data_type->get_scale() << ")";
return buffer.str();
diff --git a/be/src/format/jni/jni_data_bridge.h b/be/src/format/jni/jni_data_bridge.h
index 339dd600f07..00698f967f9 100644
--- a/be/src/format/jni/jni_data_bridge.h
+++ b/be/src/format/jni/jni_data_bridge.h
@@ -179,6 +179,7 @@ private:
!std::is_same_v<COLUMN_TYPE, ColumnDateTime> &&
!std::is_same_v<COLUMN_TYPE, ColumnDateV2> &&
!std::is_same_v<COLUMN_TYPE, ColumnDateTimeV2> &&
+ !std::is_same_v<COLUMN_TYPE, ColumnTimeStampNs> &&
!std::is_same_v<COLUMN_TYPE, ColumnTimeStampTz>)
static Status _fill_fixed_length_column(MutableColumnPtr& doris_column, CPP_TYPE* ptr,
size_t num_rows) {
@@ -191,7 +192,8 @@ private:
template <typename COLUMN_TYPE, typename CPP_TYPE>
requires(std::is_same_v<COLUMN_TYPE, ColumnDate> ||
- std::is_same_v<COLUMN_TYPE, ColumnDateTime>)
+ std::is_same_v<COLUMN_TYPE, ColumnDateTime> ||
+ std::is_same_v<COLUMN_TYPE, ColumnTimeStampNs>)
static Status _fill_fixed_length_column(MutableColumnPtr& doris_column, CPP_TYPE* ptr,
size_t num_rows) {
auto& column_data = assert_cast<COLUMN_TYPE&>(*doris_column).get_data();
diff --git a/be/src/information_schema/schema_columns_scanner.cpp b/be/src/information_schema/schema_columns_scanner.cpp
index d72ace6b56e..c5fa05b0ffa 100644
--- a/be/src/information_schema/schema_columns_scanner.cpp
+++ b/be/src/information_schema/schema_columns_scanner.cpp
@@ -135,6 +135,8 @@ std::string SchemaColumnsScanner::_to_mysql_data_type_string(TColumnDesc& desc)
case TPrimitiveType::DATETIME:
case TPrimitiveType::DATETIMEV2:
return "datetime";
+ case TPrimitiveType::TIMESTAMP_NS:
+ return "timestamp_ns";
case TPrimitiveType::TIMESTAMPTZ:
return "timestamp";
case TPrimitiveType::DECIMAL32:
@@ -238,6 +240,8 @@ std::string SchemaColumnsScanner::_type_to_string(TColumnDesc& desc) {
}
return fmt::to_string(debug_string_buffer);
}
+ case TPrimitiveType::TIMESTAMP_NS:
+ return "timestamp_ns";
case TPrimitiveType::TIMESTAMPTZ: {
fmt::memory_buffer debug_string_buffer;
if (!desc.__isset.columnScale || desc.columnScale == 0) {
@@ -549,7 +553,8 @@ Status SchemaColumnsScanner::_fill_block_impl(Block* block) {
for (int i = 0; i < columns_num; ++i) {
int data_type = _desc_result.columns[i].columnDesc.columnType;
if (_desc_result.columns[i].columnDesc.__isset.columnPrecision &&
- data_type != TPrimitiveType::DATETIMEV2) {
+ data_type != TPrimitiveType::DATETIMEV2 &&
+ data_type != TPrimitiveType::TIMESTAMP_NS) {
srcs[i] = _desc_result.columns[i].columnDesc.columnPrecision;
datas[i] = srcs.data() + i;
} else {
@@ -564,7 +569,8 @@ Status SchemaColumnsScanner::_fill_block_impl(Block* block) {
for (int i = 0; i < columns_num; ++i) {
int data_type = _desc_result.columns[i].columnDesc.columnType;
if (_desc_result.columns[i].columnDesc.__isset.columnScale &&
- data_type != TPrimitiveType::DATETIMEV2) {
+ data_type != TPrimitiveType::DATETIMEV2 &&
+ data_type != TPrimitiveType::TIMESTAMP_NS) {
srcs[i] = _desc_result.columns[i].columnDesc.columnScale;
datas[i] = srcs.data() + i;
} else {
@@ -580,6 +586,7 @@ Status SchemaColumnsScanner::_fill_block_impl(Block* block) {
int data_type = _desc_result.columns[i].columnDesc.columnType;
if (_desc_result.columns[i].columnDesc.__isset.columnScale &&
(data_type == TPrimitiveType::DATETIMEV2 ||
+ data_type == TPrimitiveType::TIMESTAMP_NS ||
data_type == TPrimitiveType::TIMESTAMPTZ)) {
srcs[i] = _desc_result.columns[i].columnDesc.columnScale;
datas[i] = srcs.data() + i;
@@ -660,7 +667,8 @@ Status SchemaColumnsScanner::_fill_block_impl(Block* block) {
for (int i = 0; i < columns_num; ++i) {
int data_type = _desc_result.columns[i].columnDesc.columnType;
if (_desc_result.columns[i].columnDesc.__isset.columnScale &&
- data_type != TPrimitiveType::DATETIMEV2) {
+ data_type != TPrimitiveType::DATETIMEV2 &&
+ data_type != TPrimitiveType::TIMESTAMP_NS) {
srcs[i] = _desc_result.columns[i].columnDesc.columnScale;
datas[i] = srcs.data() + i;
} else {
diff --git a/be/src/runtime/fold_constant_executor.cpp b/be/src/runtime/fold_constant_executor.cpp
index 97efd2d72de..3bbbd999b79 100644
--- a/be/src/runtime/fold_constant_executor.cpp
+++ b/be/src/runtime/fold_constant_executor.cpp
@@ -67,10 +67,11 @@
namespace doris {
static std::unordered_set<PrimitiveType> PRIMITIVE_TYPE_SET {
- TYPE_BOOLEAN, TYPE_TINYINT, TYPE_SMALLINT, TYPE_INT, TYPE_BIGINT,
- TYPE_LARGEINT, TYPE_FLOAT, TYPE_DOUBLE, TYPE_TIMEV2, TYPE_CHAR,
- TYPE_VARCHAR, TYPE_STRING, TYPE_HLL, TYPE_BITMAP, TYPE_DATE,
- TYPE_DATETIME, TYPE_DATEV2, TYPE_DATETIMEV2, TYPE_DECIMALV2, TYPE_TIMESTAMPTZ};
+ TYPE_BOOLEAN, TYPE_TINYINT, TYPE_SMALLINT, TYPE_INT, TYPE_BIGINT,
+ TYPE_LARGEINT, TYPE_FLOAT, TYPE_DOUBLE, TYPE_TIMEV2, TYPE_CHAR,
+ TYPE_VARCHAR, TYPE_STRING, TYPE_HLL, TYPE_BITMAP, TYPE_DATE,
+ TYPE_DATETIME, TYPE_DATEV2, TYPE_DATETIMEV2, TYPE_DECIMALV2, TYPE_TIMESTAMPTZ,
+ TYPE_TIMESTAMP_NS};
Status FoldConstantExecutor::fold_constant_vexpr(const TFoldConstantParams& params,
PConstantExprResult* response) {
@@ -265,6 +266,11 @@ Status FoldConstantExecutor::_get_result(void* src, size_t size, const DataTypeP
result = CastToString::from_datetimev2(value, type->get_scale());
break;
}
+ case TYPE_TIMESTAMP_NS: {
+ const auto value = TimeStampNsValue(*reinterpret_cast<int64_t*>(src));
+ result = value.to_string();
+ break;
+ }
case TYPE_TIMESTAMPTZ: {
auto value = binary_cast<uint64_t, TimestampTzValue>(*(int64_t*)src);
result = CastToString::from_timestamptz(value, type->get_scale(), options.timezone);
@@ -298,4 +304,4 @@ Status FoldConstantExecutor::_get_result(void* src, size_t size, const DataTypeP
return Status::OK();
}
-} // namespace doris
\ No newline at end of file
+} // namespace doris
diff --git a/be/src/runtime/runtime_predicate.cpp b/be/src/runtime/runtime_predicate.cpp
index 8c9e040f7a5..bac72d2060d 100644
--- a/be/src/runtime/runtime_predicate.cpp
+++ b/be/src/runtime/runtime_predicate.cpp
@@ -79,7 +79,8 @@ Status RuntimePredicate::init_target(
bool RuntimePredicate::_init(PrimitiveType type) {
return is_int_or_bool(type) || is_decimal(type) || is_string_type(type) || is_date_type(type) ||
- is_time_type(type) || is_timestamptz_type(type) || is_ip(type) || is_varbinary(type);
+ is_timestamp_ns_type(type) || is_time_type(type) || is_timestamptz_type(type) ||
+ is_ip(type) || is_varbinary(type);
}
Status RuntimePredicate::update(const Field& value) {
diff --git a/be/src/storage/delete/delete_handler.cpp b/be/src/storage/delete/delete_handler.cpp
index 1781c7734de..c80f9f5cbcd 100644
--- a/be/src/storage/delete/delete_handler.cpp
+++ b/be/src/storage/delete/delete_handler.cpp
@@ -88,6 +88,7 @@ Status convert(const DataTypePtr& data_type, const std::list<std::string>& str,
FROM_FE_STRING_CASE(TYPE_DATETIME);
FROM_FE_STRING_CASE(TYPE_DATEV2);
FROM_FE_STRING_CASE(TYPE_DATETIMEV2);
+ FROM_FE_STRING_CASE(TYPE_TIMESTAMP_NS);
FROM_FE_STRING_CASE(TYPE_TIMESTAMPTZ);
FROM_FE_STRING_CASE(TYPE_BOOLEAN);
FROM_FE_STRING_CASE(TYPE_IPV4);
@@ -357,6 +358,7 @@ bool DeleteHandler::is_condition_value_valid(const TabletColumn& column,
case FieldType::OLAP_FIELD_TYPE_DATETIME:
case FieldType::OLAP_FIELD_TYPE_DATEV2:
case FieldType::OLAP_FIELD_TYPE_DATETIMEV2:
+ case FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS:
case FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ:
return valid_datetime(value_str, column.frac());
case FieldType::OLAP_FIELD_TYPE_BOOL:
diff --git a/be/src/storage/index/bloom_filter/bloom_filter_index_writer.cpp b/be/src/storage/index/bloom_filter/bloom_filter_index_writer.cpp
index 1c2d0fa9089..37b2b790cc9 100644
--- a/be/src/storage/index/bloom_filter/bloom_filter_index_writer.cpp
+++ b/be/src/storage/index/bloom_filter/bloom_filter_index_writer.cpp
@@ -331,6 +331,7 @@ Status BloomFilterIndexWriter::create(const BloomFilterOptions& bf_options, Fiel
M(FieldType::OLAP_FIELD_TYPE_DECIMAL)
M(FieldType::OLAP_FIELD_TYPE_DATEV2)
M(FieldType::OLAP_FIELD_TYPE_DATETIMEV2)
+ M(FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS)
M(FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ)
M(FieldType::OLAP_FIELD_TYPE_DECIMAL32)
M(FieldType::OLAP_FIELD_TYPE_DECIMAL64)
diff --git a/be/src/storage/index/index_writer.cpp b/be/src/storage/index/index_writer.cpp
index 0b7838766a6..88a2c44985c 100644
--- a/be/src/storage/index/index_writer.cpp
+++ b/be/src/storage/index/index_writer.cpp
@@ -127,6 +127,7 @@ Status IndexColumnWriter::create(const TabletColumn* column,
M(FieldType::OLAP_FIELD_TYPE_DECIMAL)
M(FieldType::OLAP_FIELD_TYPE_DATEV2)
M(FieldType::OLAP_FIELD_TYPE_DATETIMEV2)
+ M(FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS)
M(FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ)
M(FieldType::OLAP_FIELD_TYPE_DECIMAL32)
M(FieldType::OLAP_FIELD_TYPE_DECIMAL64)
diff --git a/be/src/storage/index/inverted/inverted_index_writer.cpp b/be/src/storage/index/inverted/inverted_index_writer.cpp
index 404af9c0ba0..ebd8876c22e 100644
--- a/be/src/storage/index/inverted/inverted_index_writer.cpp
+++ b/be/src/storage/index/inverted/inverted_index_writer.cpp
@@ -688,6 +688,7 @@ template class InvertedIndexColumnWriter<FieldType::OLAP_FIELD_TYPE_DATETIME>;
template class InvertedIndexColumnWriter<FieldType::OLAP_FIELD_TYPE_DECIMAL>;
template class InvertedIndexColumnWriter<FieldType::OLAP_FIELD_TYPE_DATEV2>;
template class InvertedIndexColumnWriter<FieldType::OLAP_FIELD_TYPE_DATETIMEV2>;
+template class InvertedIndexColumnWriter<FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS>;
template class InvertedIndexColumnWriter<FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ>;
template class InvertedIndexColumnWriter<FieldType::OLAP_FIELD_TYPE_DECIMAL32>;
template class InvertedIndexColumnWriter<FieldType::OLAP_FIELD_TYPE_DECIMAL64>;
diff --git a/be/src/storage/index/inverted/inverted_index_writer.h b/be/src/storage/index/inverted/inverted_index_writer.h
index e57db04abbc..6a39cc9bb48 100644
--- a/be/src/storage/index/inverted/inverted_index_writer.h
+++ b/be/src/storage/index/inverted/inverted_index_writer.h
@@ -114,4 +114,4 @@ private:
};
} // namespace segment_v2
-} // namespace doris
\ No newline at end of file
+} // namespace doris
diff --git a/be/src/storage/index/snii/bkd/bkd_index_block.cpp b/be/src/storage/index/snii/bkd/bkd_index_block.cpp
index 592f43305e0..db13e4a65a1 100644
--- a/be/src/storage/index/snii/bkd/bkd_index_block.cpp
+++ b/be/src/storage/index/snii/bkd/bkd_index_block.cpp
@@ -41,8 +41,8 @@ constexpr FieldType kIndexableFieldTypes[] = {
FieldType::OLAP_FIELD_TYPE_DECIMAL128I, FieldType::OLAP_FIELD_TYPE_DECIMAL256,
FieldType::OLAP_FIELD_TYPE_DATE, FieldType::OLAP_FIELD_TYPE_DATETIME,
FieldType::OLAP_FIELD_TYPE_DATEV2, FieldType::OLAP_FIELD_TYPE_DATETIMEV2,
- FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ, FieldType::OLAP_FIELD_TYPE_IPV4,
- FieldType::OLAP_FIELD_TYPE_IPV6,
+ FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ, FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS,
+ FieldType::OLAP_FIELD_TYPE_IPV4, FieldType::OLAP_FIELD_TYPE_IPV6,
};
// Resolves a raw on-disk field_type code and the bytes_per_dim it implies.
diff --git a/be/src/storage/index/zone_map/zone_map_index.cpp b/be/src/storage/index/zone_map/zone_map_index.cpp
index cc5f73ee0ab..cedd7a7b85c 100644
--- a/be/src/storage/index/zone_map/zone_map_index.cpp
+++ b/be/src/storage/index/zone_map/zone_map_index.cpp
@@ -415,6 +415,7 @@ ZoneMapIndexReader::~ZoneMapIndexReader() = default;
M(TYPE_DATETIME) \
M(TYPE_DATEV2) \
M(TYPE_DATETIMEV2) \
+ M(TYPE_TIMESTAMP_NS) \
M(TYPE_TIMESTAMPTZ) \
M(TYPE_IPV4) \
M(TYPE_IPV6) \
diff --git a/be/src/storage/iterator/olap_data_convertor.cpp b/be/src/storage/iterator/olap_data_convertor.cpp
index c9e4aca59c5..23b8dc01127 100644
--- a/be/src/storage/iterator/olap_data_convertor.cpp
+++ b/be/src/storage/iterator/olap_data_convertor.cpp
@@ -168,6 +168,9 @@ OlapBlockDataConvertor::create_olap_column_data_convertor(const TabletColumn& co
case FieldType::OLAP_FIELD_TYPE_DATETIMEV2: {
return std::make_unique<OlapColumnDataConvertorDateTimeV2>();
}
+ case FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS: {
+ return std::make_unique<OlapColumnDataConvertorSimple<TYPE_TIMESTAMP_NS>>();
+ }
case FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ: {
return std::make_unique<OlapColumnDataConvertorSimple<TYPE_TIMESTAMPTZ>>();
}
diff --git a/be/src/storage/key_coder.cpp b/be/src/storage/key_coder.cpp
index d7a4d7b1457..e683b299e9b 100644
--- a/be/src/storage/key_coder.cpp
+++ b/be/src/storage/key_coder.cpp
@@ -77,6 +77,7 @@ private:
add_mapping<FieldType::OLAP_FIELD_TYPE_BOOL>();
add_mapping<FieldType::OLAP_FIELD_TYPE_DATEV2>();
add_mapping<FieldType::OLAP_FIELD_TYPE_DATETIMEV2>();
+ add_mapping<FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS>();
add_mapping<FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ>();
add_mapping<FieldType::OLAP_FIELD_TYPE_DECIMAL32>();
add_mapping<FieldType::OLAP_FIELD_TYPE_DECIMAL64>();
diff --git a/be/src/storage/key_coder.h b/be/src/storage/key_coder.h
index 0c4bcf08d17..803e5c9c732 100644
--- a/be/src/storage/key_coder.h
+++ b/be/src/storage/key_coder.h
@@ -469,6 +469,7 @@ class KeyCoderTraits<FieldType::OLAP_FIELD_TYPE_DOUBLE>
M(OLAP_FIELD_TYPE_DATETIME, TYPE_DATETIME) \
M(OLAP_FIELD_TYPE_DATEV2, TYPE_DATEV2) \
M(OLAP_FIELD_TYPE_DATETIMEV2, TYPE_DATETIMEV2) \
+ M(OLAP_FIELD_TYPE_TIMESTAMP_NS, TYPE_TIMESTAMP_NS) \
M(OLAP_FIELD_TYPE_TIMESTAMPTZ, TYPE_TIMESTAMPTZ) \
M(OLAP_FIELD_TYPE_IPV4, TYPE_IPV4) \
M(OLAP_FIELD_TYPE_IPV6, TYPE_IPV6)
diff --git a/be/src/storage/olap_common.h b/be/src/storage/olap_common.h
index 6b37a5b6ede..7e83be2ff93 100644
--- a/be/src/storage/olap_common.h
+++ b/be/src/storage/olap_common.h
@@ -171,6 +171,7 @@ enum class FieldType {
OLAP_FIELD_TYPE_IPV4 = 38,
OLAP_FIELD_TYPE_IPV6 = 39,
OLAP_FIELD_TYPE_TIMESTAMPTZ = 40,
+ OLAP_FIELD_TYPE_TIMESTAMP_NS = 41,
};
// Define all aggregation methods supported by TabletColumn
@@ -227,6 +228,7 @@ constexpr bool field_is_numeric_type(const FieldType& field_type) {
field_type == FieldType::OLAP_FIELD_TYPE_DATEV2 ||
field_type == FieldType::OLAP_FIELD_TYPE_DATETIME ||
field_type == FieldType::OLAP_FIELD_TYPE_DATETIMEV2 ||
+ field_type == FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS ||
field_type == FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ ||
field_type == FieldType::OLAP_FIELD_TYPE_LARGEINT ||
field_type == FieldType::OLAP_FIELD_TYPE_DECIMAL ||
diff --git a/be/src/storage/olap_scan_common.h b/be/src/storage/olap_scan_common.h
index 9ef64b27800..d06f2689515 100644
--- a/be/src/storage/olap_scan_common.h
+++ b/be/src/storage/olap_scan_common.h
@@ -240,6 +240,7 @@ private:
primitive_type == PrimitiveType::TYPE_BOOLEAN ||
primitive_type == PrimitiveType::TYPE_DATETIME ||
primitive_type == PrimitiveType::TYPE_DATETIMEV2 ||
+ primitive_type == PrimitiveType::TYPE_TIMESTAMP_NS ||
primitive_type == PrimitiveType::TYPE_TIMESTAMPTZ ||
primitive_type == PrimitiveType::TYPE_DECIMAL256;
};
@@ -344,10 +345,10 @@ using ColumnValueRangeType = std::variant<
ColumnValueRange<TYPE_IPV6>, ColumnValueRange<TYPE_CHAR>, ColumnValueRange<TYPE_VARCHAR>,
ColumnValueRange<TYPE_STRING>, ColumnValueRange<TYPE_DATE>, ColumnValueRange<TYPE_DATEV2>,
ColumnValueRange<TYPE_DATETIME>, ColumnValueRange<TYPE_DATETIMEV2>,
- ColumnValueRange<TYPE_TIMESTAMPTZ>, ColumnValueRange<TYPE_DECIMALV2>,
- ColumnValueRange<TYPE_BOOLEAN>, ColumnValueRange<TYPE_DECIMAL32>,
- ColumnValueRange<TYPE_DECIMAL64>, ColumnValueRange<TYPE_DECIMAL128I>,
- ColumnValueRange<TYPE_DECIMAL256>>;
+ ColumnValueRange<TYPE_TIMESTAMP_NS>, ColumnValueRange<TYPE_TIMESTAMPTZ>,
+ ColumnValueRange<TYPE_DECIMALV2>, ColumnValueRange<TYPE_BOOLEAN>,
+ ColumnValueRange<TYPE_DECIMAL32>, ColumnValueRange<TYPE_DECIMAL64>,
+ ColumnValueRange<TYPE_DECIMAL128I>, ColumnValueRange<TYPE_DECIMAL256>>;
template <PrimitiveType primitive_type>
const typename ColumnValueRange<primitive_type>::CppType
diff --git a/be/src/storage/partial_update_info.cpp b/be/src/storage/partial_update_info.cpp
index f0c306abf35..17b61c714df 100644
--- a/be/src/storage/partial_update_info.cpp
+++ b/be/src/storage/partial_update_info.cpp
@@ -28,6 +28,8 @@
#include "core/block/block.h"
#include "core/data_type/data_type_number.h" // IWYU pragma: keep
#include "core/value/bitmap_value.h"
+#include "core/value/timestamp_ns_value.h"
+#include "exec/common/int_exp.h"
#include "storage/iterator/olap_data_convertor.h"
#include "storage/olap_common.h"
#include "storage/rowset/rowset.h"
@@ -293,6 +295,32 @@ void PartialUpdateInfo::_generate_default_values_for_missing_cids(
default_value += timezone;
}
}
+ } else if (UNLIKELY(column.type() == FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS &&
+ to_lower(column.has_default_value_expr()
+ ? column.default_value_expr()
+ : column.default_value())
+ .find(to_lower("CURRENT_TIMESTAMP")) !=
+ std::string::npos)) {
+ const auto& default_value_expr = column.has_default_value_expr()
+ ? column.default_value_expr()
+ : column.default_value();
+ auto pos = to_lower(default_value_expr).find('(');
+ DateV2Value<DateTimeV2ValueType> dtv;
+ uint16_t nanosecond_remainder = 0;
+ if (pos == std::string::npos) {
+ dtv.from_unixtime(timestamp_ms / 1000, timezone);
+ } else {
+ int precision = std::stoi(default_value_expr.substr(pos + 1));
+ dtv.from_unixtime(timestamp_ms / 1000, nano_seconds, timezone, precision);
+ if (precision > 6) {
+ const int64_t factor = static_cast<int64_t>(int_exp10(9 - precision));
+ const int64_t truncated_nanos = nano_seconds / factor * factor;
+ nanosecond_remainder = static_cast<uint16_t>(truncated_nanos % 1000);
+ }
+ }
+ TimeStampNsValue timestamp_ns;
+ DORIS_CHECK(timestamp_ns.from_datetime(dtv, nanosecond_remainder));
+ default_value = timestamp_ns.to_string();
} else if (UNLIKELY(column.type() == FieldType::OLAP_FIELD_TYPE_DATEV2 &&
to_lower(column.default_value()).find(to_lower("CURRENT_DATE")) !=
std::string::npos)) {
diff --git a/be/src/storage/predicate/predicate_creator.cpp b/be/src/storage/predicate/predicate_creator.cpp
index ad1f3c288c6..930585fa496 100644
--- a/be/src/storage/predicate/predicate_creator.cpp
+++ b/be/src/storage/predicate/predicate_creator.cpp
@@ -94,6 +94,10 @@ std::shared_ptr<ColumnPredicate> create_bloom_filter_predicate(
return BloomFilterColumnPredicate<TYPE_DATETIMEV2>::create_shared(cid, col_name,
filter_olap);
}
+ case TYPE_TIMESTAMP_NS: {
+ return BloomFilterColumnPredicate<TYPE_TIMESTAMP_NS>::create_shared(cid, col_name,
+ filter_olap);
+ }
case TYPE_TIMESTAMPTZ: {
return BloomFilterColumnPredicate<TYPE_TIMESTAMPTZ>::create_shared(cid, col_name,
filter_olap);
diff --git a/be/src/storage/predicate/predicate_creator_comparison.cpp b/be/src/storage/predicate/predicate_creator_comparison.cpp
index b10a175b016..d444a35a084 100644
--- a/be/src/storage/predicate/predicate_creator_comparison.cpp
+++ b/be/src/storage/predicate/predicate_creator_comparison.cpp
@@ -102,6 +102,10 @@ std::shared_ptr<ColumnPredicate> create_comparison_predicate(const uint32_t cid,
return ComparisonPredicateBase<TYPE_DATETIMEV2, PT>::create_shared(cid, col_name, value,
opposite);
}
+ case TYPE_TIMESTAMP_NS: {
+ return ComparisonPredicateBase<TYPE_TIMESTAMP_NS, PT>::create_shared(cid, col_name, value,
+ opposite);
+ }
case TYPE_TIMESTAMPTZ: {
return ComparisonPredicateBase<TYPE_TIMESTAMPTZ, PT>::create_shared(cid, col_name, value,
opposite);
diff --git a/be/src/storage/predicate/predicate_creator_in_list_in.cpp b/be/src/storage/predicate/predicate_creator_in_list_in.cpp
index f1ad365b4f5..9162d02506f 100644
--- a/be/src/storage/predicate/predicate_creator_in_list_in.cpp
+++ b/be/src/storage/predicate/predicate_creator_in_list_in.cpp
@@ -139,6 +139,10 @@ std::shared_ptr<ColumnPredicate> create_in_list_predicate<PredicateType::IN_LIST
return create_in_list_predicate_impl<TYPE_DATETIMEV2, PredicateType::IN_LIST>(
cid, col_name, set, is_opposite);
}
+ case TYPE_TIMESTAMP_NS: {
+ return create_in_list_predicate_impl<TYPE_TIMESTAMP_NS, PredicateType::IN_LIST>(
+ cid, col_name, set, is_opposite);
+ }
case TYPE_TIMESTAMPTZ: {
return create_in_list_predicate_impl<TYPE_TIMESTAMPTZ, PredicateType::IN_LIST>(
cid, col_name, set, is_opposite);
diff --git a/be/src/storage/predicate/predicate_creator_in_list_not_in.cpp b/be/src/storage/predicate/predicate_creator_in_list_not_in.cpp
index e4cb4731a57..d90d8538dff 100644
--- a/be/src/storage/predicate/predicate_creator_in_list_not_in.cpp
+++ b/be/src/storage/predicate/predicate_creator_in_list_not_in.cpp
@@ -139,6 +139,10 @@ std::shared_ptr<ColumnPredicate> create_in_list_predicate<PredicateType::NOT_IN_
return create_in_list_predicate_impl<TYPE_DATETIMEV2, PredicateType::NOT_IN_LIST>(
cid, col_name, set, is_opposite);
}
+ case TYPE_TIMESTAMP_NS: {
+ return create_in_list_predicate_impl<TYPE_TIMESTAMP_NS, PredicateType::NOT_IN_LIST>(
+ cid, col_name, set, is_opposite);
+ }
case TYPE_TIMESTAMPTZ: {
return create_in_list_predicate_impl<TYPE_TIMESTAMPTZ, PredicateType::NOT_IN_LIST>(
cid, col_name, set, is_opposite);
diff --git a/be/src/storage/segment/encoding_info.cpp b/be/src/storage/segment/encoding_info.cpp
index 752627c2e28..f678512cd69 100644
--- a/be/src/storage/segment/encoding_info.cpp
+++ b/be/src/storage/segment/encoding_info.cpp
@@ -285,6 +285,9 @@ EncodingInfoResolver::EncodingInfoResolver() {
_register_supported_encoding<FieldType::OLAP_FIELD_TYPE_DATETIMEV2, BIT_SHUFFLE>();
_register_supported_encoding<FieldType::OLAP_FIELD_TYPE_DATETIMEV2, PLAIN_ENCODING>();
_register_supported_encoding<FieldType::OLAP_FIELD_TYPE_DATETIMEV2, FOR_ENCODING>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS, BIT_SHUFFLE>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS, PLAIN_ENCODING>();
+ _register_supported_encoding<FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS, FOR_ENCODING>();
_register_supported_encoding<FieldType::OLAP_FIELD_TYPE_DATETIME, BIT_SHUFFLE>();
_register_supported_encoding<FieldType::OLAP_FIELD_TYPE_DATETIME, PLAIN_ENCODING>();
_register_supported_encoding<FieldType::OLAP_FIELD_TYPE_DATETIME, FOR_ENCODING>();
@@ -339,6 +342,7 @@ EncodingInfoResolver::EncodingInfoResolver() {
_set_v2_default<FieldType::OLAP_FIELD_TYPE_DATE, BIT_SHUFFLE>();
_set_v2_default<FieldType::OLAP_FIELD_TYPE_DATEV2, BIT_SHUFFLE>();
_set_v2_default<FieldType::OLAP_FIELD_TYPE_DATETIMEV2, BIT_SHUFFLE>();
+ _set_v2_default<FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS, BIT_SHUFFLE>();
_set_v2_default<FieldType::OLAP_FIELD_TYPE_DATETIME, BIT_SHUFFLE>();
_set_v2_default<FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ, BIT_SHUFFLE>();
_set_v2_default<FieldType::OLAP_FIELD_TYPE_DECIMAL, BIT_SHUFFLE>();
@@ -372,6 +376,7 @@ EncodingInfoResolver::EncodingInfoResolver() {
_set_v3_default<FieldType::OLAP_FIELD_TYPE_DATE, BIT_SHUFFLE>();
_set_v3_default<FieldType::OLAP_FIELD_TYPE_DATEV2, BIT_SHUFFLE>();
_set_v3_default<FieldType::OLAP_FIELD_TYPE_DATETIMEV2, BIT_SHUFFLE>();
+ _set_v3_default<FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS, BIT_SHUFFLE>();
_set_v3_default<FieldType::OLAP_FIELD_TYPE_DATETIME, BIT_SHUFFLE>();
_set_v3_default<FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ, BIT_SHUFFLE>();
_set_v3_default<FieldType::OLAP_FIELD_TYPE_DECIMAL, BIT_SHUFFLE>();
diff --git a/be/src/storage/segment/variant/v2/variant_path_builder.cpp b/be/src/storage/segment/variant/v2/variant_path_builder.cpp
index e054aba7dd8..fb70f6420a7 100644
--- a/be/src/storage/segment/variant/v2/variant_path_builder.cpp
+++ b/be/src/storage/segment/variant/v2/variant_path_builder.cpp
@@ -44,10 +44,12 @@
#include "core/data_type/data_type_nullable.h"
#include "core/data_type/data_type_number.h"
#include "core/data_type/data_type_string.h"
+#include "core/data_type/data_type_timestamp_ns.h"
#include "core/data_type/data_type_timestamptz.h"
#include "core/data_type/get_least_supertype.h"
#include "core/data_type/primitive_type.h"
#include "core/typeid_cast.h"
+#include "core/value/timestamp_ns_value.h"
#include "core/value/timestamptz_value.h"
#include "core/value/vdatetime_value.h"
#include "exec/common/variant_util.h"
@@ -68,6 +70,7 @@ enum class ValueKind : uint8_t {
DECIMAL,
DATE,
TIMESTAMP_NTZ,
+ TIMESTAMP_NTZ_NANOS,
TIMESTAMP_TZ,
STRING,
JSONB_REF,
@@ -193,10 +196,11 @@ ValueKind value_kind(VariantRef value) {
return ValueKind::TIMESTAMP_TZ;
case VariantPrimitiveId::TIMESTAMP_NTZ_MICROS:
return ValueKind::TIMESTAMP_NTZ;
+ case VariantPrimitiveId::TIMESTAMP_NTZ_NANOS:
+ return ValueKind::TIMESTAMP_NTZ_NANOS;
case VariantPrimitiveId::BINARY:
case VariantPrimitiveId::TIME_NTZ_MICROS:
case VariantPrimitiveId::TIMESTAMP_NANOS:
- case VariantPrimitiveId::TIMESTAMP_NTZ_NANOS:
case VariantPrimitiveId::UUID:
return ValueKind::JSONB_REF;
}
@@ -264,6 +268,10 @@ const DataTypePtr& infer_scalar_type(VariantRef value, ValueKind kind) {
static const DataTypePtr type = std::make_shared<DataTypeDateTimeV2>(6);
return type;
}
+ case ValueKind::TIMESTAMP_NTZ_NANOS: {
+ static const DataTypePtr type = std::make_shared<DataTypeTimeStampNs>();
+ return type;
+ }
case ValueKind::TIMESTAMP_TZ: {
if (!timestamp_fits_doris_range(value.get_timestamp_micros())) {
return jsonb_type();
@@ -369,6 +377,10 @@ DataTypePtr path_least_common_type(const DataTypePtr& left, const DataTypePtr& r
}
const PrimitiveType left_primitive = left->get_primitive_type();
const PrimitiveType right_primitive = right->get_primitive_type();
+ if ((left_primitive == TYPE_TIMESTAMP_NS && right_primitive == TYPE_DATETIMEV2) ||
+ (left_primitive == TYPE_DATETIMEV2 && right_primitive == TYPE_TIMESTAMP_NS)) {
+ return left_primitive == TYPE_TIMESTAMP_NS ? left : right;
+ }
const bool left_decimal = left_primitive == TYPE_DECIMAL128I;
const bool right_decimal = right_primitive == TYPE_DECIMAL128I;
// get_least_supertype_jsonb() treats equal primitive ids as the same type. Decimal128(38, 2)
@@ -469,6 +481,15 @@ bool value_is_representable(VariantRef value, const DataTypePtr& target_type) {
case TYPE_DATETIMEV2:
return kind == ValueKind::TIMESTAMP_NTZ &&
timestamp_fits_doris_range(value.get_timestamp_ntz_micros());
+ case TYPE_TIMESTAMP_NS: {
+ if (kind == ValueKind::TIMESTAMP_NTZ_NANOS) {
+ return true;
+ }
+ int64_t nanos = 0;
+ return kind == ValueKind::TIMESTAMP_NTZ &&
+ !__builtin_mul_overflow(value.get_timestamp_ntz_micros(),
+ TimeStampNsValue::NANOS_PER_MICROSECOND, &nanos);
+ }
case TYPE_TIMESTAMPTZ:
return kind == ValueKind::TIMESTAMP_TZ &&
timestamp_fits_doris_range(value.get_timestamp_micros());
@@ -540,6 +561,7 @@ void write_path_jsonb(VariantRef value, JsonbWriter* writer) {
case ValueKind::BOOL:
case ValueKind::DATE:
case ValueKind::TIMESTAMP_NTZ:
+ case ValueKind::TIMESTAMP_NTZ_NANOS:
case ValueKind::TIMESTAMP_TZ:
case ValueKind::STRING:
case ValueKind::JSONB_REF: {
@@ -560,6 +582,7 @@ void append_jsonb(VariantRef value, ColumnString* column) {
case ValueKind::BOOL:
case ValueKind::DATE:
case ValueKind::TIMESTAMP_NTZ:
+ case ValueKind::TIMESTAMP_NTZ_NANOS:
case ValueKind::TIMESTAMP_TZ:
case ValueKind::STRING:
case ValueKind::JSONB_REF:
@@ -676,10 +699,25 @@ void append_date(VariantRef value, IColumn* target) {
void append_timestamp(VariantRef value, PrimitiveType target_type, IColumn* target) {
const ValueKind kind = value_kind(value);
- if (kind != ValueKind::TIMESTAMP_NTZ && kind != ValueKind::TIMESTAMP_TZ) {
+ if (kind != ValueKind::TIMESTAMP_NTZ && kind != ValueKind::TIMESTAMP_NTZ_NANOS &&
+ kind != ValueKind::TIMESTAMP_TZ) {
throw Exception(ErrorCode::INVALID_ARGUMENT,
"Cannot append Variant value to timestamp path builder");
}
+ if (target_type == TYPE_TIMESTAMP_NS) {
+ int64_t nanos = 0;
+ if (kind == ValueKind::TIMESTAMP_NTZ_NANOS) {
+ nanos = value.get_timestamp_ntz_nanos();
+ } else if (kind != ValueKind::TIMESTAMP_NTZ ||
+ __builtin_mul_overflow(value.get_timestamp_ntz_micros(),
+ TimeStampNsValue::NANOS_PER_MICROSECOND, &nanos)) {
+ throw Exception(ErrorCode::INVALID_ARGUMENT,
+ "Variant timestamp cannot be represented as TIMESTAMP_NS");
+ }
+ assert_cast<ColumnTimeStampNs&>(*target).insert_value(TimeStampNsValue(nanos));
+ return;
+ }
+ DORIS_CHECK(kind != ValueKind::TIMESTAMP_NTZ_NANOS);
const int64_t micros = kind == ValueKind::TIMESTAMP_NTZ ? value.get_timestamp_ntz_micros()
: value.get_timestamp_micros();
cctz::civil_second civil;
@@ -838,6 +876,7 @@ void append_value(VariantRef value, const DataTypePtr& target_type, IColumn* tar
append_date(value, target);
return;
case TYPE_DATETIMEV2:
+ case TYPE_TIMESTAMP_NS:
case TYPE_TIMESTAMPTZ:
append_timestamp(value, target_type->get_primitive_type(), target);
return;
diff --git a/be/src/storage/segment/variant/v2/variant_storage_cell.cpp b/be/src/storage/segment/variant/v2/variant_storage_cell.cpp
index bd9f3ca3adf..311e59fc465 100644
--- a/be/src/storage/segment/variant/v2/variant_storage_cell.cpp
+++ b/be/src/storage/segment/variant/v2/variant_storage_cell.cpp
@@ -32,6 +32,7 @@
#include "core/data_type/data_type_factory.hpp"
#include "core/data_type/storage_field_type.h"
#include "core/data_type_serde/data_type_serde.h"
+#include "core/value/timestamp_ns_value.h"
#include "core/value/variant/variant_parquet_encoding.h"
#include "exec/common/format_ip.h"
#include "exprs/function/parse/variant_jsonb_parse.h"
@@ -133,6 +134,9 @@ Status validate_scalar_cell(BinaryCellCursor& cursor, FieldType field_type, uint
case FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ:
RETURN_IF_ERROR(cursor.read(scale, "timestamp scale"));
return cursor.skip(sizeof(UInt64), "timestamp value");
+ case FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS:
+ RETURN_IF_ERROR(cursor.read(scale, "timestamp_ns scale"));
+ return cursor.skip(sizeof(Int64), "timestamp_ns value");
case FieldType::OLAP_FIELD_TYPE_DECIMAL:
RETURN_IF_ERROR(cursor.read(precision, "legacy DecimalV2 precision"));
RETURN_IF_ERROR(cursor.read(scale, "legacy DecimalV2 scale"));
@@ -325,6 +329,15 @@ Status append_binary_value(BinaryCellCursor& cursor, VariantBatchBuilder::Row& o
}
return Status::OK();
}
+ case FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS: {
+ uint8_t scale = 0;
+ Int64 raw = 0;
+ RETURN_IF_ERROR(cursor.read(&scale, "timestamp_ns scale"));
+ RETURN_IF_ERROR(cursor.read(&raw, "timestamp_ns value"));
+ DORIS_CHECK_EQ(scale, TimeStampNsValue::FRACTIONAL_DIGITS);
+ output.add_timestamp_nanos(raw, false);
+ return Status::OK();
+ }
case FieldType::OLAP_FIELD_TYPE_DECIMAL: {
uint8_t precision = 0;
uint8_t scale = 0;
diff --git a/be/src/storage/tablet/tablet_meta.cpp b/be/src/storage/tablet/tablet_meta.cpp
index 21a7eaf673c..93d9b686068 100644
--- a/be/src/storage/tablet/tablet_meta.cpp
+++ b/be/src/storage/tablet/tablet_meta.cpp
@@ -508,6 +508,9 @@ void TabletMeta::init_column_from_tcolumn(uint32_t unique_id, const TColumn& tco
if (tcolumn.__isset.default_value) {
column->set_default_value(tcolumn.default_value);
}
+ if (tcolumn.__isset.default_value_expr) {
+ column->set_default_value_expr(tcolumn.default_value_expr);
+ }
if (tcolumn.__isset.is_bloom_filter_column) {
column->set_is_bf_column(tcolumn.is_bloom_filter_column);
}
diff --git a/be/src/storage/tablet/tablet_schema.cpp b/be/src/storage/tablet/tablet_schema.cpp
index ef543c7a0a3..d3ef3c641a8 100644
--- a/be/src/storage/tablet/tablet_schema.cpp
+++ b/be/src/storage/tablet/tablet_schema.cpp
@@ -98,6 +98,8 @@ FieldType TabletColumn::get_field_type_by_string(const std::string& type_str) {
type = FieldType::OLAP_FIELD_TYPE_DATEV2;
} else if (0 == upper_type_str.compare("DATETIMEV2")) {
type = FieldType::OLAP_FIELD_TYPE_DATETIMEV2;
+ } else if (0 == upper_type_str.compare("TIMESTAMP_NS")) {
+ type = FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS;
} else if (0 == upper_type_str.compare("DATETIME")) {
type = FieldType::OLAP_FIELD_TYPE_DATETIME;
} else if (0 == upper_type_str.compare("TIMESTAMPTZ")) {
@@ -241,6 +243,8 @@ std::string TabletColumn::get_string_by_field_type(FieldType type) {
case FieldType::OLAP_FIELD_TYPE_DATETIMEV2:
return "DATETIMEV2";
+ case FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS:
+ return "TIMESTAMP_NS";
case FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ:
return "TIMESTAMPTZ";
@@ -356,6 +360,7 @@ uint32_t TabletColumn::get_field_length_by_type(TPrimitiveType::type type, uint3
case TPrimitiveType::DATETIME:
return 8;
case TPrimitiveType::DATETIMEV2:
+ case TPrimitiveType::TIMESTAMP_NS:
case TPrimitiveType::TIMESTAMPTZ:
return 8;
case TPrimitiveType::FLOAT:
@@ -469,6 +474,10 @@ void TabletColumn::init_from_pb(const ColumnPB& column) {
if (_has_default_value) {
_default_value = column.default_value();
}
+ _has_default_value_expr = column.has_default_value_expr();
+ if (_has_default_value_expr) {
+ _default_value_expr = column.default_value_expr();
+ }
if (column.has_precision()) {
_is_decimal = true;
@@ -569,6 +578,9 @@ void TabletColumn::to_schema_pb(ColumnPB* column) const {
if (_has_default_value) {
column->set_default_value(_default_value);
}
+ if (_has_default_value_expr) {
+ column->set_default_value_expr(_default_value_expr);
+ }
if (_is_decimal) {
column->set_precision(_precision);
column->set_frac(_frac);
@@ -1689,6 +1701,10 @@ bool operator==(const TabletColumn& a, const TabletColumn& b) {
if (a._has_default_value) {
if (a._default_value != b._default_value) return false;
}
+ if (a._has_default_value_expr != b._has_default_value_expr) return false;
+ if (a._has_default_value_expr) {
+ if (a._default_value_expr != b._default_value_expr) return false;
+ }
if (a._is_decimal != b._is_decimal) return false;
if (a._is_decimal) {
if (a._precision != b._precision) return false;
diff --git a/be/src/storage/tablet/tablet_schema.h b/be/src/storage/tablet/tablet_schema.h
index 125c12f20f8..3d1c99a736f 100644
--- a/be/src/storage/tablet/tablet_schema.h
+++ b/be/src/storage/tablet/tablet_schema.h
@@ -130,12 +130,18 @@ public:
}
bool has_default_value() const { return _has_default_value; }
std::string default_value() const { return _default_value; }
+ bool has_default_value_expr() const { return _has_default_value_expr; }
+ const std::string& default_value_expr() const { return _default_value_expr; }
int32_t length() const { return _length; }
void set_length(int32_t length) { _length = length; }
void set_default_value(const std::string& default_value) {
_default_value = default_value;
_has_default_value = true;
}
+ void set_default_value_expr(const std::string& default_value_expr) {
+ _default_value_expr = default_value_expr;
+ _has_default_value_expr = true;
+ }
int32_t index_length() const { return _index_length; }
void set_index_length(int32_t index_length) { _index_length = index_length; }
void set_is_key(bool is_key) { _is_key = is_key; }
@@ -301,6 +307,8 @@ private:
bool _has_default_value = false;
std::string _default_value;
+ bool _has_default_value_expr = false;
+ std::string _default_value_expr;
bool _is_decimal = false;
int32_t _precision = -1;
diff --git a/be/src/storage/tablet_info.cpp b/be/src/storage/tablet_info.cpp
index 8d0ebc1b39f..f5b3da07a25 100644
--- a/be/src/storage/tablet_info.cpp
+++ b/be/src/storage/tablet_info.cpp
@@ -42,6 +42,7 @@
#include "core/data_type/data_type_factory.hpp"
#include "core/data_type/define_primitive_type.h"
#include "core/data_type/primitive_type.h"
+#include "core/data_type_serde/data_type_timestamp_ns_serde.h"
#include "core/value/large_int_value.h"
#include "runtime/descriptors.h"
#include "runtime/memory/mem_tracker.h"
@@ -586,6 +587,11 @@ static Status _create_partition_key(const TExprNode& t_expr, BlockRow* part_key,
return Status::InternalError(ss.str());
}
column->insert_data(reinterpret_cast<const char*>(&dt), 0);
+ } else if (primitive_type == TYPE_TIMESTAMP_NS) {
+ TimeStampNsValue dt;
+ RETURN_IF_ERROR(parse_timestamp_ns(
+ {t_expr.date_literal.value.data(), t_expr.date_literal.value.size()}, &dt));
+ column->insert_data(reinterpret_cast<const char*>(&dt), 0);
} else if (primitive_type == TYPE_TIMESTAMPTZ) {
TimestampTzValue res;
CastParameters params {.status = Status::OK(), .is_strict = true};
diff --git a/be/src/storage/types.h b/be/src/storage/types.h
index 730a89db8a9..ef5f7f5899c 100644
--- a/be/src/storage/types.h
+++ b/be/src/storage/types.h
@@ -41,7 +41,6 @@
#include "core/value/ipv4_value.h"
#include "core/value/ipv6_value.h"
#include "core/value/vdatetime_value.h"
-#include "exprs/function/cast/cast_to_timestamptz.h"
#include "storage/olap_common.h"
#include "storage/olap_define.h"
#include "util/slice.h"
@@ -174,6 +173,11 @@ struct CppTypeTraits<FieldType::OLAP_FIELD_TYPE_DATETIMEV2> {
using UnsignedCppType = uint64_t;
};
template <>
+struct CppTypeTraits<FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS> {
+ using CppType = int64_t;
+ using UnsignedCppType = uint64_t;
+};
+template <>
struct CppTypeTraits<FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ> {
using CppType = uint64_t;
using UnsignedCppType = uint64_t;
@@ -307,6 +311,9 @@ struct FieldTypeTraits<FieldType::OLAP_FIELD_TYPE_DATEV2>
template <>
struct FieldTypeTraits<FieldType::OLAP_FIELD_TYPE_DATETIMEV2>
: public BaseFieldTypeTraits<FieldType::OLAP_FIELD_TYPE_DATETIMEV2> {};
+template <>
+struct FieldTypeTraits<FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS>
+ : public BaseFieldTypeTraits<FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS> {};
template <>
struct FieldTypeTraits<FieldType::OLAP_FIELD_TYPE_DATETIME>
@@ -422,6 +429,7 @@ inline size_t field_type_size(FieldType field_type) {
DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_DATETIME)
DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_DATEV2)
DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_DATETIMEV2)
+ DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_TIMESTAMP_NS)
DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_TIMEV2)
DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_TIMESTAMPTZ)
DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_IPV4)
diff --git a/be/src/storage/utils.cpp b/be/src/storage/utils.cpp
index b76db32cdff..e13d4e33efd 100644
--- a/be/src/storage/utils.cpp
+++ b/be/src/storage/utils.cpp
@@ -25,7 +25,6 @@
#include <zconf.h>
#include <zlib.h>
-#include <cmath>
#include <cstring>
#include <memory>
#include <regex>
@@ -34,6 +33,7 @@
#include <string>
#include <vector>
+#include "common/check.h"
#include "common/logging.h"
#include "common/status.h"
#include "core/value/ipv4_value.h"
@@ -228,9 +228,13 @@ bool valid_decimal(const std::string& value_str, const uint32_t precision, const
}
bool valid_datetime(const std::string& value_str, const uint32_t scale) {
+ constexpr uint32_t max_datetime_scale = 6;
+ constexpr uint32_t max_timestamp_scale = 9;
+ DORIS_CHECK_LE(scale, max_timestamp_scale);
+
const char* datetime_pattern =
"((?:\\d){4})-((?:\\d){2})-((?:\\d){2})[ ]*"
- "(((?:\\d){2}):((?:\\d){2}):((?:\\d){2})([.]*((?:\\d){0,6})))?"
+ "(((?:\\d){2}):((?:\\d){2}):((?:\\d){2})([.]*((?:\\d){0,9})))?"
"[ ]*([+-](?:\\d){2}:(?:\\d){2})?";
std::regex e(datetime_pattern);
@@ -274,16 +278,18 @@ bool valid_datetime(const std::string& value_str, const uint32_t scale) {
return false;
}
if (what[8].length()) {
- if (what[9].str().size() > 6) {
- LOG(WARNING) << "invalid microsecond. [microsecond=" << what[9].str() << "]";
+ const auto fractional_second = what[9].str();
+ const auto max_fractional_digits =
+ scale > max_datetime_scale ? scale : max_datetime_scale;
+ if (fractional_second.size() > max_fractional_digits) {
+ LOG(WARNING) << "invalid fractional second. [fractional_second="
+ << fractional_second << "]";
return false;
}
- auto s9 = what[9].str();
- s9.resize(6, '0');
- if (const long ms = strtol(s9.c_str(), nullptr, 10);
- ms % static_cast<long>(std::pow(10, 6 - scale)) != 0) {
- LOG(WARNING) << "invalid microsecond. [microsecond=" << what[9].str()
- << ", scale = " << scale << "]";
+ if (fractional_second.size() > scale &&
+ fractional_second.find_first_not_of('0', scale) != std::string::npos) {
+ LOG(WARNING) << "invalid fractional second. [fractional_second="
+ << fractional_second << ", scale = " << scale << "]";
return false;
}
}
diff --git a/be/src/storage/utils.h b/be/src/storage/utils.h
index 6d1b649dbf4..a6c2afa2917 100644
--- a/be/src/storage/utils.h
+++ b/be/src/storage/utils.h
@@ -167,8 +167,7 @@ bool valid_unsigned_number(const std::string& value_str) {
bool valid_decimal(const std::string& value_str, const uint32_t precision, const uint32_t frac);
-// Validate for date/datetime roughly. The format is 'yyyy-MM-dd HH:mm:ss'
-// TODO: support 'yyyy-MM-dd HH:mm:ss.SSS'
+// Validate date/datetime format and fractional-second scale roughly.
bool valid_datetime(const std::string& value_str, const uint32_t scale);
bool valid_bool(const std::string& value_str);
diff --git a/be/src/tools/meta_tool.cpp b/be/src/tools/meta_tool.cpp
index cf5c1b7a312..33749d6d07c 100644
--- a/be/src/tools/meta_tool.cpp
+++ b/be/src/tools/meta_tool.cpp
@@ -364,6 +364,8 @@ std::string get_field_type_string(doris::FieldType type) {
return "DATEV2";
case doris::FieldType::OLAP_FIELD_TYPE_DATETIMEV2:
return "DATETIMEV2";
+ case doris::FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS:
+ return "TIMESTAMP_NS";
case doris::FieldType::OLAP_FIELD_TYPE_BOOL:
return "BOOLEAN";
case doris::FieldType::OLAP_FIELD_TYPE_STRUCT:
@@ -492,6 +494,14 @@ std::string format_column_value(const doris::IColumn& column, size_t row,
}
return "<invalid datetime>";
}
+ case FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS: {
+ const StringRef& data = column.get_data_at(row);
+ if (data.size == sizeof(int64_t)) {
+ int64_t val = *reinterpret_cast<const int64_t*>(data.data);
+ return std::to_string(val);
+ }
+ return "<invalid timestamp_ns>";
+ }
case FieldType::OLAP_FIELD_TYPE_CHAR:
case FieldType::OLAP_FIELD_TYPE_VARCHAR:
case FieldType::OLAP_FIELD_TYPE_STRING:
diff --git a/be/src/udf/python/python_server.py b/be/src/udf/python/python_server.py
index c0683149ea2..cce71bd6942 100644
--- a/be/src/udf/python/python_server.py
+++ b/be/src/udf/python/python_server.py
@@ -310,6 +310,16 @@ def convert_arrow_value_to_python(value, arrow_type):
if value is None:
return None
+ # Pandas reserves int64's minimum value for NaT, while Doris TIMESTAMP_NS uses the same
+ # bit pattern for its valid minimum value. Keep this one value as an Arrow scalar so a
+ # Python UDF can return it without changing it to NULL or losing nanosecond precision.
+ if (
+ pa.types.is_timestamp(arrow_type)
+ and arrow_type.unit == "ns"
+ and value is pd.NaT
+ ):
+ return pa.scalar(value.value, type=arrow_type)
+
if pa.types.is_map(arrow_type):
key_type = arrow_type.key_type
item_type = arrow_type.item_type
diff --git a/be/src/util/raw_value.h b/be/src/util/raw_value.h
index 1e899e776ff..7264990c2f1 100644
--- a/be/src/util/raw_value.h
+++ b/be/src/util/raw_value.h
@@ -92,7 +92,8 @@ inline uint32_t RawValue::zlib_crc32(const void* v, size_t len, const PrimitiveT
return HashUtil::zlib_crc_hash(v, 4, seed);
}
- case TYPE_DATETIMEV2: {
+ case TYPE_DATETIMEV2:
+ case TYPE_TIMESTAMP_NS: {
return HashUtil::zlib_crc_hash(v, 8, seed);
}
diff --git a/be/test/core/column/common_column_test.h b/be/test/core/column/common_column_test.h
index 93c5f966c5c..3e41c9402d5 100644
--- a/be/test/core/column/common_column_test.h
+++ b/be/test/core/column/common_column_test.h
@@ -2766,7 +2766,7 @@ auto assert_column_vector_insert_range_of_integer_callback = [](auto x, const Mu
auto* col_vec_target = assert_cast<ColumnVector<PType>*>(target_column.get());
auto* col_vec_src = assert_cast<ColumnVector<PType>*>(source_column.get());
if constexpr (std::is_same_v<T, Float32> || std::is_same_v<T, Float64> || is_date_type(PType) ||
- PType == TYPE_TIMESTAMPTZ) {
+ is_timestamp_ns_type(PType) || PType == TYPE_TIMESTAMPTZ) {
typename PrimitiveTypeTraits<PType>::CppType begin;
typename PrimitiveTypeTraits<PType>::CppType end;
if constexpr (PType == TYPE_DATE || PType == TYPE_DATETIME) {
@@ -3037,7 +3037,8 @@ auto assert_column_vector_get_int64_callback = [](auto x, const MutableColumnPtr
auto* col_vec_src = assert_cast<ColumnVecType*>(source_column.get());
const auto& data = col_vec_src->get_data();
for (size_t i = 0; i != src_size; ++i) {
- if constexpr (!IsDecimalNumber<T> && !is_date_type(PType) && PType != TYPE_TIMESTAMPTZ) {
+ if constexpr (!IsDecimalNumber<T> && !is_date_type(PType) && !is_timestamp_ns_type(PType) &&
+ PType != TYPE_TIMESTAMPTZ) {
EXPECT_EQ(col_vec_src->get_int(i), (Int64)data[i]);
}
}
diff --git a/be/test/core/data_type/common_data_type_test.h b/be/test/core/data_type/common_data_type_test.h
index 869ab4c33a8..f9efaf7e56d 100644
--- a/be/test/core/data_type/common_data_type_test.h
+++ b/be/test/core/data_type/common_data_type_test.h
@@ -131,6 +131,7 @@ public:
is_int_or_bool(data_type->get_primitive_type()) ||
is_float_or_double(data_type->get_primitive_type()) ||
is_date_type(data_type->get_primitive_type()) ||
+ is_timestamp_ns_type(data_type->get_primitive_type()) ||
is_ip(data_type->get_primitive_type())) {
ASSERT_EQ(data_type->get_size_of_value_in_memory(), meta_info.size_of_value_in_memory);
} else {
diff --git a/be/test/core/data_type/data_type_datetime_v2_test.cpp b/be/test/core/data_type/data_type_datetime_v2_test.cpp
index e9588d6f9fd..87f543af5cc 100644
--- a/be/test/core/data_type/data_type_datetime_v2_test.cpp
+++ b/be/test/core/data_type/data_type_datetime_v2_test.cpp
@@ -120,6 +120,9 @@ TEST_F(DataTypeDateTimeV2Test, simple_func_test) {
test_func(dt_time_v2_6);
EXPECT_THROW(DataTypeDateTimeV2(7), Exception);
EXPECT_THROW(create_datetimev2(7), Exception);
+ EXPECT_THROW(create_datetimev2(8), Exception);
+ EXPECT_THROW(create_datetimev2(9), Exception);
+ EXPECT_THROW(create_datetimev2(10), Exception);
EXPECT_THROW(DataTypeTimeV2(7), Exception);
}
@@ -881,4 +884,4 @@ TEST_F(DataTypeDateTimeV2Test, GetFieldWithDataTypeTest) {
field_datetime_v2);
}
-} // namespace doris
\ No newline at end of file
+} // namespace doris
diff --git a/be/test/core/data_type/data_type_timestamp_ns_test.cpp b/be/test/core/data_type/data_type_timestamp_ns_test.cpp
new file mode 100644
index 00000000000..aed7beb10a0
--- /dev/null
+++ b/be/test/core/data_type/data_type_timestamp_ns_test.cpp
@@ -0,0 +1,685 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "core/data_type/data_type_timestamp_ns.h"
+
+#include <cctz/time_zone.h>
+#include <gtest/gtest.h>
+
+#include <cstring>
+#include <limits>
+#include <memory>
+#include <string>
+#include <string_view>
+#include <vector>
+
+#include "core/assert_cast.h"
+#include "core/column/column_const.h"
+#include "core/column/column_nullable.h"
+#include "core/column/column_string.h"
+#include "core/data_type/data_type_date_or_datetime_v2.h"
+#include "core/data_type_serde/data_type_serde.h"
+#include "core/data_type_serde/data_type_timestamp_ns_serde.h"
+#include "core/string_buffer.hpp"
+#include "core/value/vdatetime_value.h"
+#include "exprs/function/cast/cast_parameters.h"
+#include "storage/olap_common.h"
+#include "util/jsonb_utils.h"
+#include "util/jsonb_writer.h"
+#include "util/mysql_row_buffer.h"
+#include "util/slice.h"
+#include "util/timezone_utils.h"
+
+namespace doris {
+namespace {
+
+TimeStampNsValue timestamp_ns_from_string(std::string_view input) {
+ TimeStampNsValue value;
+ const auto status = parse_timestamp_ns(StringRef(input.data(), input.size()), &value);
+ EXPECT_TRUE(status.ok()) << input << ": " << status.to_string();
+ return value;
+}
+
+template <TimeUnit Unit>
+void expect_interval_add(std::string_view input, std::string_view expected) {
+ auto value = timestamp_ns_from_string(input);
+ ASSERT_TRUE(value.date_add_interval<Unit>(TimeInterval(Unit, 1, false)));
+ EXPECT_EQ(value.to_string(), expected);
+}
+
+template <TimeUnit Unit>
+void expect_datetime_trunc(std::string_view expected) {
+ auto value = timestamp_ns_from_string("2024-05-17 12:34:56.123456789");
+ ASSERT_TRUE(value.datetime_trunc<Unit>());
+ EXPECT_EQ(value.to_string(), expected);
+}
+
+} // namespace
+
+TEST(DataTypeTimeStampNsTest, TypeFamilyClassifiersKeepTimestampNsIndependent) {
+ EXPECT_FALSE(is_date_type(TYPE_TIMESTAMP_NS));
+ EXPECT_TRUE(is_timestamp_ns_type(TYPE_TIMESTAMP_NS));
+ EXPECT_FALSE(is_timestamp_ns_type(TYPE_DATETIMEV2));
+
+ EXPECT_TRUE(is_date_v2_or_datetime_v2(TYPE_DATEV2));
+ EXPECT_TRUE(is_date_v2_or_datetime_v2(TYPE_DATETIMEV2));
+ EXPECT_FALSE(is_date_v2_or_datetime_v2(TYPE_TIMESTAMP_NS));
+
+ EXPECT_TRUE(IsDataTypeDateTimeV2<DataTypeDateTimeV2>);
+ EXPECT_FALSE(IsDataTypeDateTimeV2<DataTypeTimeStampNs>);
+}
+
+TEST(DataTypeTimeStampNsTest, Int64EpochRangeAndOrdering) {
+ const TimeStampNsValue epoch(0);
+ const TimeStampNsValue before_epoch(-1);
+ const TimeStampNsValue minimum(std::numeric_limits<int64_t>::min());
+ const TimeStampNsValue maximum(std::numeric_limits<int64_t>::max());
+
+ EXPECT_EQ(TimeStampNsValue().to_date_int_val(), 0);
+ EXPECT_EQ(epoch.to_date_int_val(), epoch.epoch_nanos());
+ EXPECT_EQ(epoch.to_string(), "1970-01-01 00:00:00.000000000");
+ EXPECT_EQ(before_epoch.to_string(), "1969-12-31 23:59:59.999999999");
+ EXPECT_EQ(minimum.to_string(), "1677-09-21 00:12:43.145224192");
+ EXPECT_EQ(maximum.to_string(), "2262-04-11 23:47:16.854775807");
+ EXPECT_LT(minimum, before_epoch);
+ EXPECT_LT(before_epoch, epoch);
+ EXPECT_LT(epoch, maximum);
+}
+
+TEST(DataTypeTimeStampNsTest, NegativeEpochUsesFloorSecondAndNormalizedFraction) {
+ struct TestCase {
+ int64_t epoch_nanos;
+ int64_t epoch_seconds;
+ uint32_t nanosecond;
+ };
+ const std::vector<TestCase> cases = {
+ {-1000000001, -2, 999999999},
+ {-1000000000, -1, 0},
+ {-999999999, -1, 1},
+ {-1, -1, 999999999},
+ {0, 0, 0},
+ {1, 0, 1},
+ };
+
+ for (const auto& test_case : cases) {
+ const TimeStampNsValue value(test_case.epoch_nanos);
+ EXPECT_EQ(value.epoch_seconds(), test_case.epoch_seconds);
+ EXPECT_EQ(value.nanosecond(), test_case.nanosecond);
+ EXPECT_EQ(
+ static_cast<__int128>(value.epoch_seconds()) * TimeStampNsValue::NANOS_PER_SECOND +
+ value.nanosecond(),
+ test_case.epoch_nanos);
+ }
+}
+
+TEST(DataTypeTimeStampNsTest, EpochDerivedAndCivilAccessorsAgreeAtBoundaries) {
+ struct TestCase {
+ const char* input;
+ uint16_t year;
+ uint8_t month;
+ uint8_t day;
+ uint8_t hour;
+ uint8_t minute;
+ uint8_t second;
+ };
+ const std::vector<TestCase> cases = {
+ {"1677-09-21 00:12:43.145224192", 1677, 9, 21, 0, 12, 43},
+ {"1969-12-31 23:59:59.999999999", 1969, 12, 31, 23, 59, 59},
+ {"1970-01-01 00:00:00.000000000", 1970, 1, 1, 0, 0, 0},
+ {"2000-02-29 12:34:56.123456789", 2000, 2, 29, 12, 34, 56},
+ {"2262-04-11 23:47:16.854775807", 2262, 4, 11, 23, 47, 16},
+ };
+
+ for (const auto& test_case : cases) {
+ TimeStampNsValue value;
+ ASSERT_TRUE(parse_timestamp_ns(StringRef(test_case.input), &value).ok()) << test_case.input;
+ const auto civil_date = value.to_date();
+ const auto civil_value = value.to_datetime();
+ const int64_t expected_daynr = calc_daynr(test_case.year, test_case.month, test_case.day);
+ const int64_t expected_time =
+ test_case.hour * 3600 + test_case.minute * 60 + test_case.second;
+
+ EXPECT_EQ(value.year(), test_case.year);
+ EXPECT_EQ(value.month(), test_case.month);
+ EXPECT_EQ(value.day(), test_case.day);
+ EXPECT_EQ(value.hour(), test_case.hour);
+ EXPECT_EQ(value.minute(), test_case.minute);
+ EXPECT_EQ(value.second(), test_case.second);
+ EXPECT_EQ(value.quarter(), (test_case.month - 1) / 3 + 1);
+ EXPECT_EQ(value.daynr(), expected_daynr);
+ EXPECT_EQ(value.time_part_to_seconds(), expected_time);
+ EXPECT_EQ(value.time_part_to_nanosecond(),
+ expected_time * TimeStampNsValue::NANOS_PER_SECOND + value.nanosecond());
+ EXPECT_EQ(value.time_part_to_microsecond(),
+ value.time_part_to_nanosecond() / TimeStampNsValue::NANOS_PER_MICROSECOND);
+ EXPECT_EQ(value.weekday(), calc_weekday(expected_daynr, false));
+ EXPECT_EQ(value.day_of_week(), (value.weekday() + 1) % 7 + 1);
+ EXPECT_EQ(value.year_of_week(), civil_date.year_of_week());
+ EXPECT_EQ(value.day_of_year(), civil_date.day_of_year());
+ for (uint8_t mode = 0; mode < 8; ++mode) {
+ EXPECT_EQ(value.week(mode), civil_date.week(mode));
+ EXPECT_EQ(value.year_week(mode), civil_date.year_week(mode));
+ }
+ EXPECT_EQ(civil_date.year(), test_case.year);
+ EXPECT_EQ(civil_date.month(), test_case.month);
+ EXPECT_EQ(civil_date.day(), test_case.day);
+ EXPECT_EQ(civil_value.year(), test_case.year);
+ EXPECT_EQ(civil_value.month(), test_case.month);
+ EXPECT_EQ(civil_value.day(), test_case.day);
+ EXPECT_EQ(civil_value.hour(), test_case.hour);
+ EXPECT_EQ(civil_value.minute(), test_case.minute);
+ EXPECT_EQ(civil_value.second(), test_case.second);
+ }
+
+ EXPECT_EQ(TimeStampNsValue(-1).datetime_diff_in_seconds(TimeStampNsValue(0)), -1);
+ EXPECT_EQ(TimeStampNsValue(0).datetime_diff_in_seconds(TimeStampNsValue(-1)), 1);
+}
+
+TEST(DataTypeTimeStampNsTest, DifferenceHelpersHandleNanosecondsAndMixedDateTimeV2) {
+ const TimeStampNsValue positive(1001);
+ const TimeStampNsValue negative(-1001);
+ EXPECT_EQ(positive.datetime_diff_in_microseconds(negative), 2);
+ EXPECT_EQ(negative.datetime_diff_in_microseconds(positive), -2);
+
+ const TimeStampNsValue minimum(std::numeric_limits<int64_t>::min());
+ const TimeStampNsValue maximum(std::numeric_limits<int64_t>::max());
+ const auto expected_extreme_diff =
+ static_cast<int64_t>((static_cast<__int128>(std::numeric_limits<int64_t>::max()) -
+ std::numeric_limits<int64_t>::min()) /
+ TimeStampNsValue::NANOS_PER_MICROSECOND);
+ EXPECT_EQ(maximum.datetime_diff_in_microseconds(minimum), expected_extreme_diff);
+
+ DateV2Value<DateTimeV2ValueType> datetimev2;
+ datetimev2.unchecked_set_time(1970, 1, 1, 0, 0, 0, 1);
+ const TimeStampNsValue timestamp_ns(2500);
+ EXPECT_EQ(timestamp_ns.datetime_diff_in_seconds(datetimev2), 0);
+ EXPECT_EQ(timestamp_ns.datetime_diff_in_microseconds(datetimev2), 1);
+}
+
+TEST(DataTypeTimeStampNsTest, CalendarNamesAndConservativeFormatting) {
+ const auto value = timestamp_ns_from_string("2024-02-29 12:34:56.123456789");
+ const char* const day_names[] = {"Monday", "Tuesday", "Wednesday", "Thursday",
+ "Friday", "Saturday", "Sunday"};
+ const char* const month_names[] = {"", "January", "February", "March", "April",
+ "May", "June", "July", "August", "September",
+ "October", "November", "December"};
+ EXPECT_STREQ(value.day_name_with_locale(day_names), "Thursday");
+ EXPECT_STREQ(value.month_name_with_locale(month_names), "February");
+
+ constexpr std::string_view format = "%Y-%m-%d %H:%i:%s.%n";
+ char formatted[40] = {};
+ ASSERT_TRUE(value.to_format_string_conservative(format.data(), format.size(), formatted,
+ sizeof(formatted)));
+ EXPECT_STREQ(formatted, "2024-02-29 12:34:56.123456789");
+
+ char insufficient_buffer[4] = {};
+ EXPECT_FALSE(value.to_format_string_conservative(
+ format.data(), format.size(), insufficient_buffer, sizeof(insufficient_buffer)));
+}
+
+TEST(DataTypeTimeStampNsTest, ParseAtFixedNanosecondPrecision) {
+ TimeStampNsValue value;
+
+ ASSERT_TRUE(parse_timestamp_ns(StringRef("1970-01-01 00:00:00.12345675"), &value).ok());
+ EXPECT_EQ(value.to_string(), "1970-01-01 00:00:00.123456750");
+
+ ASSERT_TRUE(parse_timestamp_ns(StringRef("1969-12-31 23:59:59.999999999"), &value).ok());
+ EXPECT_EQ(value.epoch_nanos(), -1);
+
+ ASSERT_TRUE(parse_timestamp_ns(StringRef("1970-01-01 00:00:00.999999995"), &value).ok());
+ EXPECT_EQ(value.to_string(), "1970-01-01 00:00:00.999999995");
+
+ ASSERT_TRUE(parse_timestamp_ns(StringRef("1970-01-01 00:00:00.9999999995"), &value).ok());
+ EXPECT_EQ(value.to_string(), "1970-01-01 00:00:01.000000000");
+}
+
+TEST(DataTypeTimeStampNsTest, ParseTimezoneSuffixInSessionTimezone) {
+ TimezoneUtils::load_timezones_to_cache();
+ cctz::time_zone shanghai;
+ ASSERT_TRUE(cctz::load_time_zone("Asia/Shanghai", &shanghai));
+
+ TimeStampNsValue value;
+ auto status =
+ parse_timestamp_ns(StringRef("2023-08-17T01:41:18.123456789Z"), &value, &shanghai);
+ ASSERT_TRUE(status.ok()) << status.to_string();
+ EXPECT_EQ(value.to_string(), "2023-08-17 09:41:18.123456789");
+
+ ASSERT_TRUE(parse_timestamp_ns(StringRef("2023-08-17T01:41:18.123456789America/Los_Angeles"),
+ &value, &shanghai)
+ .ok());
+ EXPECT_EQ(value.to_string(), "2023-08-17 16:41:18.123456789");
+
+ EXPECT_FALSE(
+ parse_timestamp_ns(StringRef("1677-09-21T00:12:43.145224192+14:00"), &value, &shanghai)
+ .ok());
+ EXPECT_FALSE(
+ parse_timestamp_ns(StringRef("2262-04-11T23:47:16.854775807-01:00"), &value, &shanghai)
+ .ok());
+}
+
+TEST(DataTypeTimeStampNsTest, RoundZonedInputBeforeSessionTimezoneConversion) {
+ TimezoneUtils::load_timezones_to_cache();
+ cctz::time_zone new_york;
+ ASSERT_TRUE(cctz::load_time_zone("America/New_York", &new_york));
+
+ TimeStampNsValue value;
+ ASSERT_TRUE(parse_timestamp_ns(StringRef("2024-03-10T06:59:59.9999999995Z"), &value, &new_york)
+ .ok());
+ EXPECT_EQ(value.to_string(), "2024-03-10 03:00:00.000000000");
+
+ ASSERT_TRUE(parse_timestamp_ns(StringRef("2024-11-03T05:59:59.9999999995Z"), &value, &new_york)
+ .ok());
+ EXPECT_EQ(value.to_string(), "2024-11-03 01:00:00.000000000");
+}
+
+TEST(DataTypeTimeStampNsTest, ParseAcceptsFractionalWidthsAndRejectsMalformedValues) {
+ struct ValidCase {
+ const char* input;
+ const char* expected;
+ };
+ const std::vector<ValidCase> valid_cases = {
+ {"2024-02-29 12:34:56.1234567", "2024-02-29 12:34:56.123456700"},
+ {"2024-02-29 12:34:56.12345678", "2024-02-29 12:34:56.123456780"},
+ {"2024-02-29 12:34:56.123456789", "2024-02-29 12:34:56.123456789"},
+ {"2024-02-29 12:34:56.1234567894", "2024-02-29 12:34:56.123456789"},
+ {"2024-02-29 12:34:56.1234567895", "2024-02-29 12:34:56.123456790"},
+ {"2024-02-29 12:34:56", "2024-02-29 12:34:56.000000000"},
+ };
+
+ for (const auto& test_case : valid_cases) {
+ TimeStampNsValue value;
+ ASSERT_TRUE(parse_timestamp_ns(StringRef(test_case.input), &value).ok()) << test_case.input;
+ EXPECT_EQ(value.to_string(), test_case.expected);
+ }
+
+ const std::vector<const char*> invalid_values = {
+ "",
+ "not-a-date",
+ "2023-02-29 00:00:00.000000000",
+ "2024-13-01",
+ "2024-01-01 24:00:00",
+ "2024-01-01 00:00:00.trailing",
+ "2024-01-01 00:00:00.123.456",
+ "2024-01-01.123 00:00:00",
+ "2024.01-01 00:00:00",
+ };
+ for (const char* input : invalid_values) {
+ TimeStampNsValue value;
+ EXPECT_FALSE(parse_timestamp_ns(StringRef(input), &value).ok()) << input;
+ }
+}
+
+TEST(DataTypeTimeStampNsTest, RejectValuesOutsideEpochRange) {
+ TimeStampNsValue value;
+ EXPECT_FALSE(parse_timestamp_ns(StringRef("0000-01-01 00:00:00.000000000"), &value).ok());
+ EXPECT_FALSE(parse_timestamp_ns(StringRef("1677-09-21 00:12:43.145224191"), &value).ok());
+ EXPECT_FALSE(parse_timestamp_ns(StringRef("2262-04-11 23:47:16.854775808"), &value).ok());
+ EXPECT_FALSE(parse_timestamp_ns(StringRef("9999-12-31 23:59:59.999999999"), &value).ok());
+}
+
+TEST(DataTypeTimeStampNsTest, CivilRoundTripPreservesSubMicrosecondDigits) {
+ DateV2Value<DateTimeV2ValueType> civil;
+ civil.unchecked_set_time(2024, 2, 29, 23, 59, 58, 123456);
+
+ TimeStampNsValue value;
+ ASSERT_TRUE(value.from_datetime(civil, 789));
+ EXPECT_EQ(value.to_string(), "2024-02-29 23:59:58.123456789");
+ const auto round_trip = value.to_datetime();
+ EXPECT_EQ(round_trip.year(), 2024);
+ EXPECT_EQ(round_trip.month(), 2);
+ EXPECT_EQ(round_trip.day(), 29);
+ EXPECT_EQ(round_trip.hour(), 23);
+ EXPECT_EQ(round_trip.minute(), 59);
+ EXPECT_EQ(round_trip.second(), 58);
+ EXPECT_EQ(value.microsecond(), 123456);
+ EXPECT_EQ(value.nanosecond_remainder(), 789);
+ EXPECT_EQ(round_trip.to_date_int_val(), civil.to_date_int_val());
+}
+
+TEST(DataTypeTimeStampNsTest, FromDateTimeRejectsOutOfRangeWithoutChangingValue) {
+ DateV2Value<DateTimeV2ValueType> before_minimum;
+ before_minimum.unchecked_set_time(1677, 9, 21, 0, 0, 0, 0);
+
+ TimeStampNsValue value(42);
+ EXPECT_FALSE(value.from_datetime(before_minimum));
+ EXPECT_EQ(value.epoch_nanos(), 42);
+}
+
+TEST(DataTypeTimeStampNsTest, IntervalAdditionSupportsEveryRegisteredUnit) {
+ constexpr std::string_view input = "2024-01-15 12:30:40.123456789";
+ expect_interval_add<TimeUnit::MICROSECOND>(input, "2024-01-15 12:30:40.123457789");
+ expect_interval_add<TimeUnit::MILLISECOND>(input, "2024-01-15 12:30:40.124456789");
+ expect_interval_add<TimeUnit::SECOND>(input, "2024-01-15 12:30:41.123456789");
+ expect_interval_add<TimeUnit::MINUTE>(input, "2024-01-15 12:31:40.123456789");
+ expect_interval_add<TimeUnit::HOUR>(input, "2024-01-15 13:30:40.123456789");
+ expect_interval_add<TimeUnit::DAY>(input, "2024-01-16 12:30:40.123456789");
+ expect_interval_add<TimeUnit::WEEK>(input, "2024-01-22 12:30:40.123456789");
+ expect_interval_add<TimeUnit::MONTH>(input, "2024-02-15 12:30:40.123456789");
+ expect_interval_add<TimeUnit::QUARTER>(input, "2024-04-15 12:30:40.123456789");
+ expect_interval_add<TimeUnit::YEAR>(input, "2025-01-15 12:30:40.123456789");
+}
+
+TEST(DataTypeTimeStampNsTest, DateTimeTruncSupportsEveryRegisteredUnitAndChecksRange) {
+ expect_datetime_trunc<TimeUnit::SECOND>("2024-05-17 12:34:56.000000000");
+ expect_datetime_trunc<TimeUnit::MINUTE>("2024-05-17 12:34:00.000000000");
+ expect_datetime_trunc<TimeUnit::HOUR>("2024-05-17 12:00:00.000000000");
+ expect_datetime_trunc<TimeUnit::DAY>("2024-05-17 00:00:00.000000000");
+ expect_datetime_trunc<TimeUnit::WEEK>("2024-05-13 00:00:00.000000000");
+ expect_datetime_trunc<TimeUnit::MONTH>("2024-05-01 00:00:00.000000000");
+ expect_datetime_trunc<TimeUnit::QUARTER>("2024-04-01 00:00:00.000000000");
+ expect_datetime_trunc<TimeUnit::YEAR>("2024-01-01 00:00:00.000000000");
+
+ TimeStampNsValue minimum(std::numeric_limits<int64_t>::min());
+ EXPECT_FALSE(minimum.datetime_trunc<TimeUnit::DAY>());
+ EXPECT_EQ(minimum.epoch_nanos(), std::numeric_limits<int64_t>::min());
+
+ auto unchanged = timestamp_ns_from_string("2024-05-17 12:34:56.123456789");
+ EXPECT_FALSE(unchanged.datetime_trunc<TimeUnit::MICROSECOND>());
+ EXPECT_EQ(unchanged.to_string(), "2024-05-17 12:34:56.123456789");
+}
+
+TEST(DataTypeTimeStampNsTest, CheckedIntervalAdditionRejectsEpochOverflow) {
+ const TimeStampNsValue minimum(std::numeric_limits<int64_t>::min());
+ const TimeStampNsValue maximum(std::numeric_limits<int64_t>::max());
+ const TimeInterval three_seconds(TimeUnit::SECOND, 3, false);
+ const TimeInterval negative_three_seconds(TimeUnit::SECOND, 3, true);
+
+ auto below_minimum = minimum;
+ EXPECT_FALSE(below_minimum.date_add_interval<TimeUnit::SECOND>(negative_three_seconds));
+ EXPECT_EQ(below_minimum, minimum);
+
+ auto above_maximum = maximum;
+ EXPECT_FALSE(above_maximum.date_add_interval<TimeUnit::SECOND>(three_seconds));
+ EXPECT_EQ(above_maximum, maximum);
+}
+
+TEST(DataTypeTimeStampNsTest, FactoryKeepsTimestampNsSeparateFromDateTimeV2) {
+ const auto microseconds = create_datetimev2(6);
+ const auto timestamp_ns = std::make_shared<DataTypeTimeStampNs>();
+
+ EXPECT_EQ(microseconds->get_primitive_type(), TYPE_DATETIMEV2);
+ EXPECT_THROW(create_datetimev2(7), Exception);
+ EXPECT_THROW(create_datetimev2(8), Exception);
+ EXPECT_THROW(create_datetimev2(9), Exception);
+ EXPECT_EQ(timestamp_ns->get_primitive_type(), TYPE_TIMESTAMP_NS);
+ EXPECT_EQ(timestamp_ns->get_storage_field_type(), FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS);
+ EXPECT_EQ(timestamp_ns->get_scale(), 9);
+ EXPECT_EQ(microseconds->get_family_name(), "DateTimeV2");
+ EXPECT_EQ(timestamp_ns->get_family_name(), "TimeStampNs");
+}
+
+TEST(DataTypeTimeStampNsTest, SerDeRoundTripsTextProtobufAndBinary) {
+ const auto type = std::make_shared<DataTypeTimeStampNs>();
+ const auto serde = type->get_serde();
+ auto source = type->create_column();
+ DataTypeSerDe::FormatOptions options;
+ const std::vector<std::string> inputs = {
+ "1677-09-21 00:12:43.145224192", "1969-12-31 23:59:59.999999999",
+ "1970-01-01 00:00:00.000000000", "2024-02-29 12:34:56.123456789",
+ "2262-04-11 23:47:16.854775807",
+ };
+ for (const auto& input : inputs) {
+ StringRef ref(input);
+ ASSERT_TRUE(serde->from_string(ref, *source, options).ok()) << input;
+ }
+ const auto& source_data = assert_cast<const ColumnTimeStampNs&>(*source).get_data();
+
+ PValues protobuf_values;
+ ASSERT_TRUE(serde->write_column_to_pb(*source, protobuf_values, 0, source->size()).ok());
+ auto protobuf_result = type->create_column();
+ ASSERT_TRUE(serde->read_column_from_pb(*protobuf_result, protobuf_values).ok());
+ const auto& protobuf_data = assert_cast<const ColumnTimeStampNs&>(*protobuf_result).get_data();
+ EXPECT_EQ(protobuf_data, source_data);
+
+ ColumnString::Chars binary;
+ std::vector<size_t> offsets = {0};
+ for (size_t row = 0; row < source->size(); ++row) {
+ serde->write_one_cell_to_binary(*source, binary, row);
+ offsets.push_back(binary.size());
+ }
+ constexpr size_t bytes_per_row = sizeof(uint8_t) + sizeof(uint8_t) + sizeof(int64_t);
+ ASSERT_EQ(binary.size(), source->size() * bytes_per_row);
+ auto binary_result = ColumnNullable::create(type->create_column(), ColumnUInt8::create());
+ for (size_t row = 0; row < source->size(); ++row) {
+ const uint8_t* begin = binary.data() + offsets[row];
+ const uint8_t* end = DataTypeSerDe::deserialize_binary_to_column(begin, *binary_result);
+ EXPECT_EQ(end - begin, bytes_per_row);
+ }
+ const auto& binary_data =
+ assert_cast<const ColumnTimeStampNs&>(binary_result->get_nested_column()).get_data();
+ EXPECT_EQ(binary_data, source_data);
+}
+
+TEST(DataTypeTimeStampNsTest, SerDeExplicitlyRejectsUnsupportedOrcIo) {
+ const DataTypeTimeStampNs type;
+ const auto serde = type.get_serde();
+ auto column = type.create_column();
+
+ Arena arena;
+ DataTypeSerDe::FormatOptions options;
+ const auto orc_write_status =
+ serde->write_column_to_orc("UTC", *column, nullptr, nullptr, 0, 0, arena, options);
+ EXPECT_TRUE(orc_write_status.is<ErrorCode::NOT_IMPLEMENTED_ERROR>()) << orc_write_status;
+}
+
+TEST(DataTypeTimeStampNsTest, DataTypeLiteralField) {
+ const DataTypeTimeStampNs type;
+ const DataTypeDateTimeV2 legacy6(6);
+
+ EXPECT_TRUE(type.equals(DataTypeTimeStampNs {}));
+ EXPECT_FALSE(type.equals(legacy6));
+ EXPECT_FALSE(type.equals_ignore_precision(legacy6));
+ EXPECT_FALSE(legacy6.equals_ignore_precision(type));
+
+ TExprNode node;
+ node.date_literal.value = "2024-02-29 12:34:56.123456789";
+ const Field field = type.get_field(node);
+ EXPECT_EQ(field.get<TYPE_TIMESTAMP_NS>().to_string(), "2024-02-29 12:34:56.123456789");
+ EXPECT_EQ(field.to_debug_string(0), "2024-02-29 12:34:56.123456789");
+ EXPECT_EQ(field.to_debug_string(6), "2024-02-29 12:34:56.123456789");
+ node.date_literal.value = "not-a-datetime";
+ EXPECT_THROW(type.get_field(node), Exception);
+
+ auto column = type.create_column();
+ column->insert(field);
+ const auto field_with_type = type.get_field_with_data_type(*column, 0);
+ EXPECT_EQ(field_with_type.field, field);
+ EXPECT_EQ(field_with_type.base_scalar_type_id, TYPE_TIMESTAMP_NS);
+ EXPECT_EQ(field_with_type.precision, -1);
+ EXPECT_EQ(field_with_type.scale, 9);
+}
+
+TEST(DataTypeTimeStampNsTest, FormattingAndStdHash) {
+ TimeStampNsValue value;
+ ASSERT_TRUE(parse_timestamp_ns(StringRef("2024-02-29 12:34:56.123456789"), &value).ok());
+
+ EXPECT_TRUE(value.is_valid_date());
+
+ char text[40] = {};
+ const char* end = value.to_string(text);
+ EXPECT_STREQ(text, "2024-02-29 12:34:56.123456789");
+ EXPECT_EQ(end, text + std::strlen(text) + 1);
+
+ char buffer[40] = {};
+ const int32_t length = value.to_buffer(buffer);
+ EXPECT_EQ(length, 29);
+ EXPECT_EQ(std::string_view(buffer, length), "2024-02-29 12:34:56.123456789");
+
+ EXPECT_EQ(std::hash<TimeStampNsValue> {}(value), std::hash<int64_t> {}(value.epoch_nanos()));
+}
+
+TEST(DataTypeTimeStampNsTest, SerDeStrictBatchJsonJsonbMysqlAndBinaryField) {
+ const DataTypeTimeStampNs type;
+ const auto serde = type.get_serde();
+ DataTypeSerDe::FormatOptions options;
+ options.field_delim = ";";
+
+ auto strings = ColumnString::create();
+ strings->insert_data("1970-01-01 00:00:00.000000001", 29);
+ strings->insert_data("ignored-invalid-value", 21);
+ strings->insert_data("2024-02-29 12:34:56.123456789", 29);
+ NullMap null_map = {0, 1, 0};
+ auto strict_result = type.create_column();
+ ASSERT_TRUE(
+ serde->from_string_strict_mode_batch(*strings, *strict_result, options, null_map.data())
+ .ok());
+ const auto& strict_data = assert_cast<const ColumnTimeStampNs&>(*strict_result).get_data();
+ EXPECT_EQ(strict_data[0].epoch_nanos(), 1);
+ EXPECT_EQ(strict_data[2].to_string(), "2024-02-29 12:34:56.123456789");
+
+ auto invalid_strings = ColumnString::create();
+ invalid_strings->insert_data("invalid", 7);
+ auto invalid_result = type.create_column();
+ EXPECT_FALSE(serde->from_string_strict_mode_batch(*invalid_strings, *invalid_result, options,
+ nullptr)
+ .ok());
+
+ auto source = type.create_column();
+ for (const std::string input :
+ {"1970-01-01 00:00:00.000000001", "2024-02-29 12:34:56.123456789"}) {
+ StringRef ref(input);
+ ASSERT_TRUE(serde->from_string_strict_mode(ref, *source, options).ok());
+ }
+
+ auto serialized = ColumnString::create();
+ VectorBufferWriter writer(*serialized);
+ ASSERT_TRUE(serde->serialize_column_to_json(*source, 0, source->size(), writer, options).ok());
+ writer.commit();
+ EXPECT_EQ(serialized->get_data_at(0).to_string(),
+ "1970-01-01 00:00:00.000000001;2024-02-29 12:34:56.123456789");
+
+ std::vector<std::string> json_values = {"1970-01-01 00:00:00.000000001",
+ "2024-02-29 12:34:56.123456789"};
+ std::vector<Slice> slices;
+ for (auto& json_value : json_values) {
+ slices.emplace_back(json_value.data(), json_value.size());
+ }
+ auto json_result = type.create_column();
+ uint64_t num_deserialized = 0;
+ ASSERT_TRUE(serde->deserialize_column_from_json_vector(*json_result, slices, &num_deserialized,
+ options)
+ .ok());
+ EXPECT_EQ(num_deserialized, json_values.size());
+ EXPECT_EQ(assert_cast<const ColumnTimeStampNs&>(*json_result).get_data(),
+ assert_cast<const ColumnTimeStampNs&>(*source).get_data());
+
+ const auto nested_serde = type.get_serde(2);
+ auto nested_json = ColumnString::create();
+ VectorBufferWriter nested_writer(*nested_json);
+ ASSERT_TRUE(nested_serde->serialize_one_cell_to_json(*source, 1, nested_writer, options).ok());
+ nested_writer.commit();
+ EXPECT_EQ(nested_json->get_data_at(0).to_string(), "\"2024-02-29 12:34:56.123456789\"");
+ auto nested_result = type.create_column();
+ std::string quoted = nested_json->get_data_at(0).to_string();
+ Slice quoted_slice(quoted.data(), quoted.size());
+ ASSERT_TRUE(nested_serde->deserialize_one_cell_from_json(*nested_result, quoted_slice, options)
+ .ok());
+ EXPECT_EQ(assert_cast<const ColumnTimeStampNs&>(*nested_result).get_element(0),
+ assert_cast<const ColumnTimeStampNs&>(*source).get_element(1));
+
+ auto one_value = source->clone_resized(1);
+ auto const_column = ColumnConst::create(std::move(one_value), 2);
+ auto const_json = ColumnString::create();
+ VectorBufferWriter const_writer(*const_json);
+ ASSERT_TRUE(serde->serialize_one_cell_to_json(*const_column, 1, const_writer, options).ok());
+ const_writer.commit();
+ EXPECT_EQ(const_json->get_data_at(0).to_string(), "1970-01-01 00:00:00.000000001");
+
+ JsonbWriter jsonb_writer;
+ ASSERT_TRUE(serde->serialize_column_to_jsonb(*source, 1, jsonb_writer,
+ DataTypeSerDe::FormatOptions {})
+ .ok());
+ EXPECT_EQ(JsonbToJson::jsonb_to_json_string(jsonb_writer.getOutput()->getBuffer(),
+ jsonb_writer.getOutput()->getSize()),
+ "\"2024-02-29 12:34:56.123456789\"");
+ CastParameters cast_params {.status = Status::OK(), .is_strict = true};
+ auto jsonb_result = type.create_column();
+ ASSERT_TRUE(serde->deserialize_column_from_jsonb(*jsonb_result, jsonb_writer.getValue(),
+ cast_params)
+ .ok());
+ EXPECT_EQ(assert_cast<const ColumnTimeStampNs&>(*jsonb_result).get_element(0),
+ assert_cast<const ColumnTimeStampNs&>(*source).get_element(1));
+
+ auto jsonb_values = ColumnString::create();
+ ASSERT_TRUE(serde->serialize_column_to_jsonb_vector(*source, *jsonb_values,
+ DataTypeSerDe::FormatOptions {})
+ .ok());
+ auto jsonb_vector_result = ColumnNullable::create(type.create_column(), ColumnUInt8::create());
+ ASSERT_TRUE(serde->deserialize_column_from_jsonb_vector(*jsonb_vector_result, *jsonb_values,
+ cast_params)
+ .ok());
+ EXPECT_EQ(assert_cast<const ColumnTimeStampNs&>(jsonb_vector_result->get_nested_column())
+ .get_data(),
+ assert_cast<const ColumnTimeStampNs&>(*source).get_data());
... 52287 lines suppressed ...
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org