You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2022/06/26 08:08:02 UTC
[doris] branch master updated: [Feature-wip] add date v2 type (#9916)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new ca94867b4e [Feature-wip] add date v2 type (#9916)
ca94867b4e is described below
commit ca94867b4ef722682d47b9f30b9085e8e475e3e3
Author: Gabriel <ga...@gmail.com>
AuthorDate: Sun Jun 26 16:07:56 2022 +0800
[Feature-wip] add date v2 type (#9916)
---
be/src/exec/olap_common.h | 3 +-
be/src/exec/tablet_info.cpp | 25 +-
be/src/exec/tablet_sink.cpp | 1 +
be/src/exprs/anyval_util.cpp | 3 +
be/src/exprs/bloomfilter_predicate.h | 14 +
be/src/exprs/create_predicate_function.h | 2 +
be/src/exprs/expr.h | 9 +
be/src/exprs/expr_context.cpp | 4 +-
be/src/exprs/literal.cpp | 2 +
be/src/exprs/runtime_filter.cpp | 36 +
be/src/olap/aggregate_func.cpp | 5 +
be/src/olap/column_vector.cpp | 4 +
be/src/olap/comparison_predicate.cpp | 8 +
be/src/olap/delete_handler.cpp | 1 +
be/src/olap/key_coder.cpp | 1 +
be/src/olap/key_coder.h | 34 +
be/src/olap/olap_common.h | 4 +-
be/src/olap/reader.cpp | 18 +
be/src/olap/row_block2.cpp | 10 +
be/src/olap/rowset/column_reader.cpp | 6 +
be/src/olap/rowset/column_reader.h | 12 +
be/src/olap/rowset/column_writer.cpp | 5 +
be/src/olap/rowset/column_writer.h | 3 +
.../olap/rowset/segment_v2/bitmap_index_writer.cpp | 3 +
be/src/olap/rowset/segment_v2/column_reader.cpp | 7 +
be/src/olap/rowset/segment_v2/encoding_info.cpp | 18 +
be/src/olap/rowset/segment_v2/segment_iterator.cpp | 3 +
be/src/olap/schema.cpp | 3 +
be/src/olap/tablet_schema.cpp | 7 +
be/src/olap/types.cpp | 2 +
be/src/olap/types.h | 120 ++
be/src/olap/utils.h | 3 +-
be/src/runtime/datetime_value.h | 2 +
be/src/runtime/primitive_type.cpp | 55 +-
be/src/runtime/primitive_type.h | 26 +-
be/src/runtime/raw_value.cpp | 23 +
be/src/runtime/raw_value.h | 26 +
be/src/runtime/type_limit.h | 12 +
be/src/runtime/types.h | 2 +
be/src/udf/udf.h | 5 +-
be/src/util/binary_cast.hpp | 21 +-
be/src/util/date_func.cpp | 16 +
be/src/util/date_func.h | 1 +
be/src/vec/CMakeLists.txt | 1 +
.../aggregate_function_min_max.cpp | 4 +
.../aggregate_function_min_max_by.cpp | 9 +
.../aggregate_function_window.h | 4 +
.../aggregate_function_window_funnel.cpp | 2 +-
.../aggregate_function_window_funnel.h | 45 +-
be/src/vec/columns/column.h | 3 +
be/src/vec/columns/column_nullable.h | 2 +
be/src/vec/columns/columns_number.h | 1 +
be/src/vec/columns/predicate_column.h | 16 +-
be/src/vec/core/call_on_type_index.h | 3 +
be/src/vec/core/types.h | 10 +
be/src/vec/data_types/data_type.cpp | 2 +
be/src/vec/data_types/data_type.h | 11 +-
be/src/vec/data_types/data_type_date_time.h | 9 +
be/src/vec/data_types/data_type_factory.cpp | 10 +
be/src/vec/data_types/data_type_factory.hpp | 1 +
be/src/vec/data_types/data_type_time_v2.cpp | 73 ++
be/src/vec/data_types/data_type_time_v2.h | 47 +
be/src/vec/data_types/get_least_supertype.cpp | 13 +
be/src/vec/exec/join/vhash_join_node.cpp | 1 +
be/src/vec/exec/vaggregation_node.cpp | 1 +
be/src/vec/exec/varrow_scanner.cpp | 6 +-
be/src/vec/exec/volap_scan_node.cpp | 39 +-
be/src/vec/exec/volap_scan_node.h | 5 +-
be/src/vec/exec/vschema_scan_node.cpp | 6 +
be/src/vec/exec/vset_operation_node.cpp | 1 +
be/src/vec/exprs/vliteral.cpp | 6 +
.../vec/functions/array/function_array_element.h | 4 +
be/src/vec/functions/array/function_array_index.h | 10 +-
be/src/vec/functions/date_time_transforms.h | 151 ++-
be/src/vec/functions/function.h | 9 +-
be/src/vec/functions/function_cast.h | 66 +-
.../function_date_or_datetime_computation.cpp | 220 +++-
.../function_date_or_datetime_computation.h | 339 ++++--
.../function_date_or_datetime_to_something.h | 11 +-
.../function_date_or_datetime_to_string.cpp | 8 +-
.../function_date_or_datetime_to_string.h | 12 +-
.../function_datetime_string_to_string.cpp | 11 +-
.../functions/function_datetime_string_to_string.h | 40 +-
be/src/vec/functions/function_timestamp.cpp | 171 +--
be/src/vec/functions/function_totype.h | 13 +-
be/src/vec/functions/time_of_function.cpp | 37 +-
be/src/vec/functions/to_time_function.cpp | 71 +-
be/src/vec/io/io_helper.h | 17 +
be/src/vec/olap/olap_data_convertor.cpp | 121 ++-
be/src/vec/olap/olap_data_convertor.h | 95 ++
be/src/vec/runtime/vdatetime_value.cpp | 1146 +++++++++++++++++++-
be/src/vec/runtime/vdatetime_value.h | 665 ++++++++++--
be/src/vec/runtime/vfile_result_writer.cpp | 8 +
be/src/vec/sink/mysql_result_writer.cpp | 25 +-
be/src/vec/sink/vmysql_table_writer.cpp | 12 +
be/src/vec/utils/arrow_column_to_doris_column.cpp | 69 +-
be/src/vec/utils/arrow_column_to_doris_column.h | 4 +-
be/src/vec/utils/template_helpers.hpp | 3 +-
be/test/CMakeLists.txt | 1 +
be/test/olap/delta_writer_test.cpp | 105 +-
be/test/olap/in_list_predicate_test.cpp | 130 +++
be/test/olap/null_predicate_test.cpp | 107 ++
be/test/olap/row_cursor_test.cpp | 79 +-
be/test/olap/schema_change_test.cpp | 280 +++++
be/test/olap/storage_types_test.cpp | 6 +
be/test/vec/core/block_test.cpp | 36 +-
be/test/vec/exprs/vexpr_test.cpp | 29 +
be/test/vec/function/function_test_util.cpp | 23 +
be/test/vec/function/function_test_util.h | 3 +
be/test/vec/function/function_time_test.cpp | 552 +++++++++-
be/test/vec/runtime/vdatetime_value_test.cpp | 247 +++++
.../utils/arrow_column_to_doris_column_test.cpp | 19 +-
fe/fe-core/src/main/cup/sql_parser.cup | 10 +-
.../java/org/apache/doris/analysis/CastExpr.java | 7 +-
.../java/org/apache/doris/analysis/ColumnDef.java | 9 +-
.../apache/doris/analysis/CreateFunctionStmt.java | 3 +
.../org/apache/doris/analysis/DataDescription.java | 3 +-
.../org/apache/doris/analysis/DateLiteral.java | 228 +++-
.../org/apache/doris/analysis/FloatLiteral.java | 18 +-
.../apache/doris/analysis/FunctionCallExpr.java | 6 +-
.../org/apache/doris/analysis/LiteralExpr.java | 4 +
.../org/apache/doris/analysis/ShowAlterStmt.java | 3 +-
.../org/apache/doris/analysis/StringLiteral.java | 6 +-
.../java/org/apache/doris/analysis/TypeDef.java | 43 +-
.../main/java/org/apache/doris/catalog/Column.java | 8 -
.../java/org/apache/doris/catalog/ColumnType.java | 15 +-
.../java/org/apache/doris/catalog/Function.java | 6 +
.../org/apache/doris/catalog/PartitionKey.java | 2 +
.../org/apache/doris/catalog/PrimitiveType.java | 155 ++-
.../apache/doris/catalog/RangePartitionInfo.java | 6 +-
.../org/apache/doris/catalog/ScalarFunction.java | 6 +
.../java/org/apache/doris/catalog/ScalarType.java | 110 +-
.../main/java/org/apache/doris/catalog/Type.java | 209 +++-
.../main/java/org/apache/doris/common/Config.java | 3 +
.../doris/common/proc/PartitionsProcDir.java | 15 +-
.../apache/doris/common/proc/RollupProcDir.java | 15 +-
.../doris/common/proc/SchemaChangeProcDir.java | 15 +-
.../apache/doris/common/util/PropertyAnalyzer.java | 2 +-
.../java/org/apache/doris/common/util/Util.java | 2 +
.../external/iceberg/util/DorisTypeToType.java | 12 +-
.../external/iceberg/util/TypeToDorisType.java | 14 +-
.../doris/httpv2/util/StatementSubmitter.java | 3 +-
.../java/org/apache/doris/load/DeleteHandler.java | 9 +-
.../org/apache/doris/mysql/MysqlSerializer.java | 8 +-
.../org/apache/doris/qe/cache/PartitionRange.java | 3 +
.../java/org/apache/doris/rewrite/FEFunctions.java | 26 +-
.../doris/rewrite/RewriteDateLiteralRule.java | 1 +
.../org/apache/doris/statistics/ColumnStats.java | 2 +
.../apache/doris/task/HadoopLoadPendingTask.java | 2 +
fe/fe-core/src/main/jflex/sql_scanner.flex | 1 +
.../java/org/apache/doris/alter/AlterTest.java | 138 +++
.../org/apache/doris/analysis/AggregateTest.java | 105 +-
.../doris/analysis/ComparisonPredicateTest.java | 42 +
.../org/apache/doris/analysis/DateLiteralTest.java | 181 ++++
.../java/org/apache/doris/analysis/ExprTest.java | 13 +
.../org/apache/doris/analysis/InPredicateTest.java | 81 ++
.../doris/analysis/LiteralExprCompareTest.java | 120 ++
.../org/apache/doris/analysis/QueryStmtTest.java | 46 +-
.../apache/doris/analysis/RangeCompareTest.java | 16 +
.../org/apache/doris/catalog/ColumnTypeTest.java | 70 ++
.../doris/catalog/RangePartitionInfoTest.java | 67 ++
.../java/org/apache/doris/catalog/RecoverTest.java | 126 ++-
.../apache/doris/planner/ConstantExpressTest.java | 8 +
.../planner/MaterializedViewFunctionTest.java | 20 +-
.../org/apache/doris/planner/QueryPlanTest.java | 165 +--
.../org/apache/doris/rewrite/FEFunctionsTest.java | 406 ++++++-
.../doris/rewrite/RewriteDateLiteralRuleTest.java | 107 ++
.../doris/load/loadv2/dpp/SparkRDDAggregator.java | 4 +
gensrc/proto/internal_service.proto | 1 +
gensrc/proto/types.proto | 1 +
gensrc/thrift/Types.thrift | 3 +
.../correctness/test_last_value_window.groovy | 6 +-
172 files changed, 7724 insertions(+), 843 deletions(-)
diff --git a/be/src/exec/olap_common.h b/be/src/exec/olap_common.h
index 41d8f7aa5c..952e092cab 100644
--- a/be/src/exec/olap_common.h
+++ b/be/src/exec/olap_common.h
@@ -314,7 +314,8 @@ private:
typedef std::variant<ColumnValueRange<int8_t>, ColumnValueRange<int16_t>, ColumnValueRange<int32_t>,
ColumnValueRange<int64_t>, ColumnValueRange<__int128>,
ColumnValueRange<StringValue>, ColumnValueRange<DateTimeValue>,
- ColumnValueRange<DecimalV2Value>, ColumnValueRange<bool>>
+ ColumnValueRange<DecimalV2Value>, ColumnValueRange<bool>,
+ ColumnValueRange<doris::vectorized::DateV2Value>>
ColumnValueRangeType;
template <class T>
diff --git a/be/src/exec/tablet_info.cpp b/be/src/exec/tablet_info.cpp
index 542363b6e1..b346081b5d 100644
--- a/be/src/exec/tablet_info.cpp
+++ b/be/src/exec/tablet_info.cpp
@@ -578,14 +578,25 @@ Status VOlapTablePartitionParam::_create_partition_key(const TExprNode& t_expr,
auto column = std::move(*part_key->first->get_by_position(pos).column).mutate();
switch (t_expr.node_type) {
case TExprNodeType::DATE_LITERAL: {
- vectorized::VecDateTimeValue dt;
- if (!dt.from_date_str(t_expr.date_literal.value.c_str(),
- t_expr.date_literal.value.size())) {
- std::stringstream ss;
- ss << "invalid date literal in partition column, date=" << t_expr.date_literal;
- return Status::InternalError(ss.str());
+ if (TypeDescriptor::from_thrift(t_expr.type).is_date_v2_type()) {
+ vectorized::DateV2Value dt;
+ if (!dt.from_date_str(t_expr.date_literal.value.c_str(),
+ t_expr.date_literal.value.size())) {
+ std::stringstream ss;
+ ss << "invalid date literal in partition column, date=" << t_expr.date_literal;
+ return Status::InternalError(ss.str());
+ }
+ column->insert_data(reinterpret_cast<const char*>(&dt), 0);
+ } else {
+ vectorized::VecDateTimeValue dt;
+ if (!dt.from_date_str(t_expr.date_literal.value.c_str(),
+ t_expr.date_literal.value.size())) {
+ std::stringstream ss;
+ ss << "invalid date literal in partition column, date=" << t_expr.date_literal;
+ return Status::InternalError(ss.str());
+ }
+ column->insert_data(reinterpret_cast<const char*>(&dt), 0);
}
- column->insert_data(reinterpret_cast<const char*>(&dt), 0);
break;
}
case TExprNodeType::INT_LITERAL: {
diff --git a/be/src/exec/tablet_sink.cpp b/be/src/exec/tablet_sink.cpp
index a718fa0475..fc3e976d7d 100644
--- a/be/src/exec/tablet_sink.cpp
+++ b/be/src/exec/tablet_sink.cpp
@@ -777,6 +777,7 @@ Status OlapTableSink::prepare(RuntimeState* state) {
case TYPE_VARCHAR:
case TYPE_DATE:
case TYPE_DATETIME:
+ case TYPE_DATEV2:
case TYPE_HLL:
case TYPE_OBJECT:
case TYPE_STRING:
diff --git a/be/src/exprs/anyval_util.cpp b/be/src/exprs/anyval_util.cpp
index f22d857360..374c5a3130 100644
--- a/be/src/exprs/anyval_util.cpp
+++ b/be/src/exprs/anyval_util.cpp
@@ -142,6 +142,9 @@ FunctionContext::TypeDesc AnyValUtil::column_type_to_type_desc(const TypeDescrip
case TYPE_DATETIME:
out.type = FunctionContext::TYPE_DATETIME;
break;
+ case TYPE_DATEV2:
+ out.type = FunctionContext::TYPE_DATEV2;
+ break;
case TYPE_VARCHAR:
out.type = FunctionContext::TYPE_VARCHAR;
out.len = type.len;
diff --git a/be/src/exprs/bloomfilter_predicate.h b/be/src/exprs/bloomfilter_predicate.h
index d9e8c8b34e..cb0736dcca 100644
--- a/be/src/exprs/bloomfilter_predicate.h
+++ b/be/src/exprs/bloomfilter_predicate.h
@@ -248,6 +248,15 @@ struct DateFindOp : public CommonFindOp<DateTimeValue, BloomFilterAdaptor> {
}
};
+template <class BloomFilterAdaptor>
+struct DateV2FindOp : public CommonFindOp<doris::vectorized::DateV2Value, BloomFilterAdaptor> {
+ bool find_olap_engine(const BloomFilterAdaptor& bloom_filter, const void* data) const {
+ doris::vectorized::DateV2Value value;
+ value.from_date(*reinterpret_cast<const uint32_t*>(data));
+ return bloom_filter.test(Slice((char*)&value, sizeof(doris::vectorized::DateV2Value)));
+ }
+};
+
template <class BloomFilterAdaptor>
struct DecimalV2FindOp : public CommonFindOp<DecimalV2Value, BloomFilterAdaptor> {
bool find_olap_engine(const BloomFilterAdaptor& bloom_filter, const void* data) const {
@@ -275,6 +284,11 @@ struct BloomFilterTypeTraits<TYPE_DATE, BloomFilterAdaptor> {
using FindOp = DateFindOp<BloomFilterAdaptor>;
};
+template <class BloomFilterAdaptor>
+struct BloomFilterTypeTraits<TYPE_DATEV2, BloomFilterAdaptor> {
+ using FindOp = DateV2FindOp<BloomFilterAdaptor>;
+};
+
template <class BloomFilterAdaptor>
struct BloomFilterTypeTraits<TYPE_DATETIME, BloomFilterAdaptor> {
using FindOp = DateTimeFindOp<BloomFilterAdaptor>;
diff --git a/be/src/exprs/create_predicate_function.h b/be/src/exprs/create_predicate_function.h
index b8acae5964..364634057f 100644
--- a/be/src/exprs/create_predicate_function.h
+++ b/be/src/exprs/create_predicate_function.h
@@ -92,6 +92,8 @@ typename Traits::BasePtr create_predicate_function(PrimitiveType type) {
return Creator::template create<TYPE_DATE>();
case TYPE_DATETIME:
return Creator::template create<TYPE_DATETIME>();
+ case TYPE_DATEV2:
+ return Creator::template create<TYPE_DATEV2>();
case TYPE_CHAR:
return Creator::template create<TYPE_CHAR>();
diff --git a/be/src/exprs/expr.h b/be/src/exprs/expr.h
index e4cb02ea1b..56ff05e8a3 100644
--- a/be/src/exprs/expr.h
+++ b/be/src/exprs/expr.h
@@ -516,6 +516,15 @@ Status create_texpr_literal_node(const void* data, TExprNode* node) {
} else if (origin_value->type() == TimeType::TIME_TIME) {
(*node).__set_type(create_type_desc(PrimitiveType::TYPE_TIME));
}
+ } else if constexpr (std::is_same_v<doris::vectorized::DateV2Value, T>) {
+ auto origin_value = reinterpret_cast<const doris::vectorized::DateV2Value*>(data);
+ TDateLiteral date_literal;
+ char convert_buffer[30];
+ origin_value->to_string(convert_buffer);
+ date_literal.__set_value(convert_buffer);
+ (*node).__set_date_literal(date_literal);
+ (*node).__set_node_type(TExprNodeType::DATE_LITERAL);
+ (*node).__set_type(create_type_desc(PrimitiveType::TYPE_DATEV2));
} else if constexpr (std::is_same_v<DecimalV2Value, T>) {
auto origin_value = reinterpret_cast<const DecimalV2Value*>(data);
(*node).__set_node_type(TExprNodeType::DECIMAL_LITERAL);
diff --git a/be/src/exprs/expr_context.cpp b/be/src/exprs/expr_context.cpp
index 84a51ba619..199076cdcb 100644
--- a/be/src/exprs/expr_context.cpp
+++ b/be/src/exprs/expr_context.cpp
@@ -256,7 +256,9 @@ void* ExprContext::get_value(Expr* e, TupleRow* row) {
return &_result.string_val;
}
case TYPE_DATE:
- case TYPE_DATETIME: {
+ case TYPE_DATETIME:
+ case TYPE_DATEV2:
+ case TYPE_DATETIMEV2: {
doris_udf::DateTimeVal v = e->get_datetime_val(this, row);
if (v.is_null) {
return nullptr;
diff --git a/be/src/exprs/literal.cpp b/be/src/exprs/literal.cpp
index 94b9a97f0c..462ec259be 100644
--- a/be/src/exprs/literal.cpp
+++ b/be/src/exprs/literal.cpp
@@ -81,6 +81,8 @@ Literal::Literal(const TExprNode& node) : Expr(node) {
break;
case TYPE_DATE:
case TYPE_DATETIME:
+ case TYPE_DATEV2:
+ case TYPE_DATETIMEV2:
_value.datetime_val.from_date_str(node.date_literal.value.c_str(),
node.date_literal.value.size());
break;
diff --git a/be/src/exprs/runtime_filter.cpp b/be/src/exprs/runtime_filter.cpp
index 2533c8c2d1..bab230d73b 100644
--- a/be/src/exprs/runtime_filter.cpp
+++ b/be/src/exprs/runtime_filter.cpp
@@ -74,6 +74,9 @@ TExprNodeType::type get_expr_node_type(PrimitiveType type) {
case TYPE_DATETIME:
return TExprNodeType::DATE_LITERAL;
+ case TYPE_DATEV2:
+ return TExprNodeType::DATE_LITERAL;
+
case TYPE_CHAR:
case TYPE_VARCHAR:
case TYPE_HLL:
@@ -109,6 +112,8 @@ PColumnType to_proto(PrimitiveType type) {
return PColumnType::COLUMN_TYPE_DOUBLE;
case TYPE_DATE:
return PColumnType::COLUMN_TYPE_DATE;
+ case TYPE_DATEV2:
+ return PColumnType::COLUMN_TYPE_DATEV2;
case TYPE_DATETIME:
return PColumnType::COLUMN_TYPE_DATETIME;
case TYPE_DECIMALV2:
@@ -148,6 +153,8 @@ PrimitiveType to_primitive_type(PColumnType type) {
return TYPE_DOUBLE;
case PColumnType::COLUMN_TYPE_DATE:
return TYPE_DATE;
+ case PColumnType::COLUMN_TYPE_DATEV2:
+ return TYPE_DATEV2;
case PColumnType::COLUMN_TYPE_DATETIME:
return TYPE_DATETIME;
case PColumnType::COLUMN_TYPE_DECIMALV2:
@@ -233,6 +240,10 @@ Status create_literal(ObjectPool* pool, PrimitiveType type, const void* data, vo
create_texpr_literal_node<double_t>(data, &node);
break;
}
+ case TYPE_DATEV2: {
+ create_texpr_literal_node<doris::vectorized::DateV2Value>(data, &node);
+ break;
+ }
case TYPE_DATE:
case TYPE_DATETIME: {
create_texpr_literal_node<DateTimeValue>(data, &node);
@@ -669,6 +680,14 @@ public:
});
break;
}
+ case TYPE_DATEV2: {
+ batch_assign(in_filter, [](std::unique_ptr<HybridSetBase>& set, PColumnValue& column,
+ ObjectPool* pool) {
+ auto date_v2_val = column.intval();
+ set->insert(&date_v2_val);
+ });
+ break;
+ }
case TYPE_DATETIME:
case TYPE_DATE: {
batch_assign(in_filter, [](std::unique_ptr<HybridSetBase>& set, PColumnValue& column,
@@ -773,6 +792,11 @@ public:
double max_val = static_cast<double>(minmax_filter->max_val().doubleval());
return _minmax_func->assign(&min_val, &max_val);
}
+ case TYPE_DATEV2: {
+ int32_t min_val = minmax_filter->min_val().intval();
+ int32_t max_val = minmax_filter->max_val().intval();
+ return _minmax_func->assign(&min_val, &max_val);
+ }
case TYPE_DATETIME:
case TYPE_DATE: {
auto& min_val_ref = minmax_filter->min_val().stringval();
@@ -1246,6 +1270,13 @@ void IRuntimeFilter::to_protobuf(PInFilter* filter) {
});
return;
}
+ case TYPE_DATEV2: {
+ batch_copy<doris::vectorized::DateV2Value>(
+ filter, it, [](PColumnValue* column, const doris::vectorized::DateV2Value* value) {
+ column->set_intval(*reinterpret_cast<const int32_t*>(value));
+ });
+ return;
+ }
case TYPE_DATE:
case TYPE_DATETIME: {
batch_copy<DateTimeValue>(filter, it, [](PColumnValue* column, const DateTimeValue* value) {
@@ -1328,6 +1359,11 @@ void IRuntimeFilter::to_protobuf(PMinMaxFilter* filter) {
filter->mutable_max_val()->set_doubleval(*reinterpret_cast<const double*>(max_data));
return;
}
+ case TYPE_DATEV2: {
+ filter->mutable_min_val()->set_intval(*reinterpret_cast<const int32_t*>(min_data));
+ filter->mutable_max_val()->set_intval(*reinterpret_cast<const int32_t*>(max_data));
+ return;
+ }
case TYPE_DATE:
case TYPE_DATETIME: {
char convert_buffer[30];
diff --git a/be/src/olap/aggregate_func.cpp b/be/src/olap/aggregate_func.cpp
index 781ab39082..dda4d0f780 100644
--- a/be/src/olap/aggregate_func.cpp
+++ b/be/src/olap/aggregate_func.cpp
@@ -100,6 +100,7 @@ AggregateFuncResolver::AggregateFuncResolver() {
add_aggregate_mapping<OLAP_FIELD_AGGREGATION_NONE, OLAP_FIELD_TYPE_DECIMAL>();
add_aggregate_mapping<OLAP_FIELD_AGGREGATION_NONE, OLAP_FIELD_TYPE_DATE>();
add_aggregate_mapping<OLAP_FIELD_AGGREGATION_NONE, OLAP_FIELD_TYPE_DATETIME>();
+ add_aggregate_mapping<OLAP_FIELD_AGGREGATION_NONE, OLAP_FIELD_TYPE_DATEV2>();
add_aggregate_mapping<OLAP_FIELD_AGGREGATION_NONE, OLAP_FIELD_TYPE_CHAR>();
add_aggregate_mapping<OLAP_FIELD_AGGREGATION_NONE, OLAP_FIELD_TYPE_VARCHAR>();
add_aggregate_mapping<OLAP_FIELD_AGGREGATION_NONE, OLAP_FIELD_TYPE_STRING>();
@@ -147,6 +148,7 @@ AggregateFuncResolver::AggregateFuncResolver() {
add_aggregate_mapping<OLAP_FIELD_AGGREGATION_MIN, OLAP_FIELD_TYPE_DECIMAL>();
add_aggregate_mapping<OLAP_FIELD_AGGREGATION_MIN, OLAP_FIELD_TYPE_DATE>();
add_aggregate_mapping<OLAP_FIELD_AGGREGATION_MIN, OLAP_FIELD_TYPE_DATETIME>();
+ add_aggregate_mapping<OLAP_FIELD_AGGREGATION_MIN, OLAP_FIELD_TYPE_DATEV2>();
add_aggregate_mapping<OLAP_FIELD_AGGREGATION_MIN, OLAP_FIELD_TYPE_CHAR>();
add_aggregate_mapping<OLAP_FIELD_AGGREGATION_MIN, OLAP_FIELD_TYPE_VARCHAR>();
add_aggregate_mapping<OLAP_FIELD_AGGREGATION_MIN, OLAP_FIELD_TYPE_STRING>();
@@ -162,6 +164,7 @@ AggregateFuncResolver::AggregateFuncResolver() {
add_aggregate_mapping<OLAP_FIELD_AGGREGATION_MAX, OLAP_FIELD_TYPE_DECIMAL>();
add_aggregate_mapping<OLAP_FIELD_AGGREGATION_MAX, OLAP_FIELD_TYPE_DATE>();
add_aggregate_mapping<OLAP_FIELD_AGGREGATION_MAX, OLAP_FIELD_TYPE_DATETIME>();
+ add_aggregate_mapping<OLAP_FIELD_AGGREGATION_MAX, OLAP_FIELD_TYPE_DATEV2>();
add_aggregate_mapping<OLAP_FIELD_AGGREGATION_MAX, OLAP_FIELD_TYPE_CHAR>();
add_aggregate_mapping<OLAP_FIELD_AGGREGATION_MAX, OLAP_FIELD_TYPE_VARCHAR>();
add_aggregate_mapping<OLAP_FIELD_AGGREGATION_MAX, OLAP_FIELD_TYPE_STRING>();
@@ -188,6 +191,7 @@ AggregateFuncResolver::AggregateFuncResolver() {
add_aggregate_mapping<OLAP_FIELD_AGGREGATION_REPLACE, OLAP_FIELD_TYPE_DECIMAL>();
add_aggregate_mapping<OLAP_FIELD_AGGREGATION_REPLACE, OLAP_FIELD_TYPE_DATE>();
add_aggregate_mapping<OLAP_FIELD_AGGREGATION_REPLACE, OLAP_FIELD_TYPE_DATETIME>();
+ add_aggregate_mapping<OLAP_FIELD_AGGREGATION_REPLACE, OLAP_FIELD_TYPE_DATEV2>();
add_aggregate_mapping<OLAP_FIELD_AGGREGATION_REPLACE, OLAP_FIELD_TYPE_CHAR>();
add_aggregate_mapping<OLAP_FIELD_AGGREGATION_REPLACE, OLAP_FIELD_TYPE_VARCHAR>();
add_aggregate_mapping<OLAP_FIELD_AGGREGATION_REPLACE, OLAP_FIELD_TYPE_STRING>();
@@ -204,6 +208,7 @@ AggregateFuncResolver::AggregateFuncResolver() {
add_aggregate_mapping<OLAP_FIELD_AGGREGATION_REPLACE_IF_NOT_NULL, OLAP_FIELD_TYPE_DECIMAL>();
add_aggregate_mapping<OLAP_FIELD_AGGREGATION_REPLACE_IF_NOT_NULL, OLAP_FIELD_TYPE_DATE>();
add_aggregate_mapping<OLAP_FIELD_AGGREGATION_REPLACE_IF_NOT_NULL, OLAP_FIELD_TYPE_DATETIME>();
+ add_aggregate_mapping<OLAP_FIELD_AGGREGATION_REPLACE_IF_NOT_NULL, OLAP_FIELD_TYPE_DATEV2>();
add_aggregate_mapping<OLAP_FIELD_AGGREGATION_REPLACE_IF_NOT_NULL, OLAP_FIELD_TYPE_CHAR>();
add_aggregate_mapping<OLAP_FIELD_AGGREGATION_REPLACE_IF_NOT_NULL, OLAP_FIELD_TYPE_VARCHAR>();
add_aggregate_mapping<OLAP_FIELD_AGGREGATION_REPLACE_IF_NOT_NULL, OLAP_FIELD_TYPE_STRING>();
diff --git a/be/src/olap/column_vector.cpp b/be/src/olap/column_vector.cpp
index c4962926f2..14817c85cc 100644
--- a/be/src/olap/column_vector.cpp
+++ b/be/src/olap/column_vector.cpp
@@ -91,6 +91,10 @@ Status ColumnVectorBatch::create(size_t init_capacity, bool is_nullable, const T
local.reset(new ScalarColumnVectorBatch<CppTypeTraits<OLAP_FIELD_TYPE_DATE>::CppType>(
type_info, is_nullable));
break;
+ case OLAP_FIELD_TYPE_DATEV2:
+ local.reset(new ScalarColumnVectorBatch<CppTypeTraits<OLAP_FIELD_TYPE_DATEV2>::CppType>(
+ type_info, is_nullable));
+ break;
case OLAP_FIELD_TYPE_DATETIME:
local.reset(
new ScalarColumnVectorBatch<CppTypeTraits<OLAP_FIELD_TYPE_DATETIME>::CppType>(
diff --git a/be/src/olap/comparison_predicate.cpp b/be/src/olap/comparison_predicate.cpp
index 3424bbb6c9..870be42185 100644
--- a/be/src/olap/comparison_predicate.cpp
+++ b/be/src/olap/comparison_predicate.cpp
@@ -585,6 +585,7 @@ COMPARISON_PRED_CONSTRUCTOR_DECLARATION(GreaterEqualPredicate)
template void CLASS<decimal12_t>::evaluate(VectorizedRowBatch* batch) const; \
template void CLASS<StringValue>::evaluate(VectorizedRowBatch* batch) const; \
template void CLASS<uint24_t>::evaluate(VectorizedRowBatch* batch) const; \
+ template void CLASS<uint32_t>::evaluate(VectorizedRowBatch* batch) const; \
template void CLASS<uint64_t>::evaluate(VectorizedRowBatch* batch) const; \
template void CLASS<bool>::evaluate(VectorizedRowBatch* batch) const;
@@ -615,6 +616,8 @@ COMPARISON_PRED_EVALUATE_DECLARATION(GreaterEqualPredicate)
const; \
template void CLASS<uint24_t>::evaluate(ColumnBlock* block, uint16_t* sel, uint16_t* size) \
const; \
+ template void CLASS<uint32_t>::evaluate(ColumnBlock* block, uint16_t* sel, uint16_t* size) \
+ const; \
template void CLASS<uint64_t>::evaluate(ColumnBlock* block, uint16_t* sel, uint16_t* size) \
const; \
template void CLASS<bool>::evaluate(ColumnBlock* block, uint16_t* sel, uint16_t* size) const;
@@ -657,6 +660,9 @@ COMPARISON_PRED_COLUMN_BLOCK_EVALUATE_DECLARATION(GreaterEqualPredicate)
template Status CLASS<uint24_t>::evaluate(const Schema& schema, \
const std::vector<BitmapIndexIterator*>& iterators, \
uint32_t num_rows, roaring::Roaring* bitmap) const; \
+ template Status CLASS<uint32_t>::evaluate(const Schema& schema, \
+ const std::vector<BitmapIndexIterator*>& iterators, \
+ uint32_t num_rows, roaring::Roaring* bitmap) const; \
template Status CLASS<uint64_t>::evaluate(const Schema& schema, \
const std::vector<BitmapIndexIterator*>& iterators, \
uint32_t num_rows, roaring::Roaring* bitmap) const; \
@@ -692,6 +698,8 @@ COMPARISON_PRED_BITMAP_EVALUATE_DECLARATION(GreaterEqualPredicate)
uint16_t size) const; \
template uint16_t CLASS<uint24_t>::evaluate(vectorized::IColumn& column, uint16_t* sel, \
uint16_t size) const; \
+ template uint16_t CLASS<uint32_t>::evaluate(vectorized::IColumn& column, uint16_t* sel, \
+ uint16_t size) const; \
template uint16_t CLASS<uint64_t>::evaluate(vectorized::IColumn& column, uint16_t* sel, \
uint16_t size) const; \
template uint16_t CLASS<bool>::evaluate(vectorized::IColumn& column, uint16_t* sel, \
diff --git a/be/src/olap/delete_handler.cpp b/be/src/olap/delete_handler.cpp
index 4e39bc4c73..69e5f0b7af 100644
--- a/be/src/olap/delete_handler.cpp
+++ b/be/src/olap/delete_handler.cpp
@@ -146,6 +146,7 @@ bool DeleteConditionHandler::is_condition_value_valid(const TabletColumn& column
return value_str.size() <= config::string_type_length_soft_limit_bytes;
case OLAP_FIELD_TYPE_DATE:
case OLAP_FIELD_TYPE_DATETIME:
+ case OLAP_FIELD_TYPE_DATEV2:
return valid_datetime(value_str);
case OLAP_FIELD_TYPE_BOOL:
return valid_bool(value_str);
diff --git a/be/src/olap/key_coder.cpp b/be/src/olap/key_coder.cpp
index 3e2ab5ba55..49a06a13d8 100644
--- a/be/src/olap/key_coder.cpp
+++ b/be/src/olap/key_coder.cpp
@@ -73,6 +73,7 @@ private:
add_mapping<OLAP_FIELD_TYPE_VARCHAR>();
add_mapping<OLAP_FIELD_TYPE_STRING>();
add_mapping<OLAP_FIELD_TYPE_BOOL>();
+ add_mapping<OLAP_FIELD_TYPE_DATEV2>();
}
template <FieldType field_type>
diff --git a/be/src/olap/key_coder.h b/be/src/olap/key_coder.h
index b6afcf6c9e..7f452350d4 100644
--- a/be/src/olap/key_coder.h
+++ b/be/src/olap/key_coder.h
@@ -170,6 +170,40 @@ public:
}
};
+template <>
+class KeyCoderTraits<OLAP_FIELD_TYPE_DATEV2> {
+public:
+ using CppType = typename CppTypeTraits<OLAP_FIELD_TYPE_DATEV2>::CppType;
+ using UnsignedCppType = typename CppTypeTraits<OLAP_FIELD_TYPE_DATEV2>::UnsignedCppType;
+
+public:
+ static void full_encode_ascending(const void* value, std::string* buf) {
+ UnsignedCppType unsigned_val;
+ memcpy(&unsigned_val, value, sizeof(unsigned_val));
+ // make it bigendian
+ unsigned_val = BigEndian::FromHost32(unsigned_val);
+ buf->append((char*)&unsigned_val, sizeof(unsigned_val));
+ }
+
+ static void encode_ascending(const void* value, size_t index_size, std::string* buf) {
+ full_encode_ascending(value, buf);
+ }
+
+ static Status decode_ascending(Slice* encoded_key, size_t index_size, uint8_t* cell_ptr,
+ MemPool* pool) {
+ if (encoded_key->size < sizeof(UnsignedCppType)) {
+ return Status::InvalidArgument(Substitute("Key too short, need=$0 vs real=$1",
+ sizeof(UnsignedCppType), encoded_key->size));
+ }
+ UnsignedCppType unsigned_val;
+ memcpy(&unsigned_val, encoded_key->data, sizeof(UnsignedCppType));
+ unsigned_val = BigEndian::FromHost32(unsigned_val);
+ memcpy(cell_ptr, &unsigned_val, sizeof(UnsignedCppType));
+ encoded_key->remove_prefix(sizeof(UnsignedCppType));
+ return Status::OK();
+ }
+};
+
template <>
class KeyCoderTraits<OLAP_FIELD_TYPE_DECIMAL> {
public:
diff --git a/be/src/olap/olap_common.h b/be/src/olap/olap_common.h
index 7396a3cbf1..9bec6cfebe 100644
--- a/be/src/olap/olap_common.h
+++ b/be/src/olap/olap_common.h
@@ -143,7 +143,9 @@ enum FieldType {
OLAP_FIELD_TYPE_BOOL = 24,
OLAP_FIELD_TYPE_OBJECT = 25,
OLAP_FIELD_TYPE_STRING = 26,
- OLAP_FIELD_TYPE_QUANTILE_STATE = 27
+ OLAP_FIELD_TYPE_QUANTILE_STATE = 27,
+ OLAP_FIELD_TYPE_DATEV2 = 28,
+ OLAP_FIELD_TYPE_DATETIMEV2 = 29
};
// Define all aggregation methods supported by Field
diff --git a/be/src/olap/reader.cpp b/be/src/olap/reader.cpp
index 497c014e46..3d8a3f398f 100644
--- a/be/src/olap/reader.cpp
+++ b/be/src/olap/reader.cpp
@@ -533,6 +533,11 @@ void TabletReader::_init_conditions_param(const ReaderParams& read_params) {
predicate = new PREDICATE<uint24_t>(index, value, opposite); \
break; \
} \
+ case OLAP_FIELD_TYPE_DATEV2: { \
+ uint32_t value = timestamp_from_date_v2(cond); \
+ predicate = new PREDICATE<uint32_t>(index, value, opposite); \
+ break; \
+ } \
case OLAP_FIELD_TYPE_DATETIME: { \
uint64_t value = timestamp_from_datetime(cond); \
predicate = new PREDICATE<uint64_t>(index, value, opposite); \
@@ -745,6 +750,19 @@ ColumnPredicate* TabletReader::_parse_to_predicate(const TCondition& condition,
}
break;
}
+ case OLAP_FIELD_TYPE_DATEV2: {
+ phmap::flat_hash_set<uint32_t> values;
+ for (auto& cond_val : condition.condition_values) {
+ uint32_t value = timestamp_from_date_v2(cond_val);
+ values.insert(value);
+ }
+ if (condition.condition_op == "*=") {
+ predicate = new InListPredicate<uint32_t>(index, std::move(values), opposite);
+ } else {
+ predicate = new NotInListPredicate<uint32_t>(index, std::move(values), opposite);
+ }
+ break;
+ }
case OLAP_FIELD_TYPE_DATETIME: {
phmap::flat_hash_set<uint64_t> values;
for (auto& cond_val : condition.condition_values) {
diff --git a/be/src/olap/row_block2.cpp b/be/src/olap/row_block2.cpp
index 947bdd4055..393fe40adf 100644
--- a/be/src/olap/row_block2.cpp
+++ b/be/src/olap/row_block2.cpp
@@ -245,6 +245,11 @@ Status RowBlockV2::_copy_data_to_column(int cid,
}
break;
}
+ case OLAP_FIELD_TYPE_DATEV2: {
+ auto column_int = assert_cast<vectorized::ColumnVector<vectorized::UInt32>*>(column);
+ insert_data_directly(cid, column_int);
+ break;
+ }
case OLAP_FIELD_TYPE_DATETIME: {
auto column_int = assert_cast<vectorized::ColumnVector<vectorized::Int64>*>(column);
@@ -507,6 +512,11 @@ Status RowBlockV2::_append_data_to_column(const ColumnVectorBatch* batch, size_t
}
break;
}
+ case OLAP_FIELD_TYPE_DATEV2: {
+ auto column_int = assert_cast<vectorized::ColumnVector<vectorized::UInt32>*>(column);
+ insert_data_directly(batch, column_int, start, len);
+ break;
+ }
case OLAP_FIELD_TYPE_DATETIME: {
auto column_int = assert_cast<vectorized::ColumnVector<vectorized::Int64>*>(column);
diff --git a/be/src/olap/rowset/column_reader.cpp b/be/src/olap/rowset/column_reader.cpp
index ee6c7fb1ad..8c6cc1676c 100644
--- a/be/src/olap/rowset/column_reader.cpp
+++ b/be/src/olap/rowset/column_reader.cpp
@@ -610,6 +610,12 @@ ColumnReader* ColumnReader::create(uint32_t column_id, const std::vector<TabletC
break;
}
+ case OLAP_FIELD_TYPE_DATEV2: {
+ reader = new (std::nothrow) DateV2ColumnReader(column_id, column_unique_id);
+
+ break;
+ }
+
case OLAP_FIELD_TYPE_DECIMAL: {
reader = new (std::nothrow) DecimalColumnReader(column_id, column_unique_id);
break;
diff --git a/be/src/olap/rowset/column_reader.h b/be/src/olap/rowset/column_reader.h
index b1533f373f..2880cef335 100644
--- a/be/src/olap/rowset/column_reader.h
+++ b/be/src/olap/rowset/column_reader.h
@@ -347,6 +347,16 @@ public:
}
break;
}
+ case OLAP_FIELD_TYPE_DATEV2: {
+ _values = reinterpret_cast<void*>(mem_pool->allocate(size * sizeof(uint32_t)));
+ uint32_t value = 0;
+ std::stringstream ss(_default_value);
+ ss >> value;
+ for (int i = 0; i < size; ++i) {
+ ((uint32_t*)_values)[i] = value;
+ }
+ break;
+ }
case OLAP_FIELD_TYPE_DATETIME: {
_values = reinterpret_cast<void*>(mem_pool->allocate(size * sizeof(uint64_t)));
uint64_t value = timestamp_from_datetime(_default_value);
@@ -824,6 +834,8 @@ typedef IntegerColumnReaderWrapper<uint24_t, false> DateColumnReader;
// Internal use LONG implementation
typedef IntegerColumnReaderWrapper<uint64_t, false> DateTimeColumnReader;
+typedef IntegerColumnReaderWrapper<uint32_t, false> DateV2ColumnReader;
+
} // namespace doris
#endif // DORIS_BE_SRC_OLAP_ROWSET_COLUMN_READER_H
diff --git a/be/src/olap/rowset/column_writer.cpp b/be/src/olap/rowset/column_writer.cpp
index 8053d92621..af212a6927 100644
--- a/be/src/olap/rowset/column_writer.cpp
+++ b/be/src/olap/rowset/column_writer.cpp
@@ -96,6 +96,11 @@ ColumnWriter* ColumnWriter::create(uint32_t column_id, const TabletSchema& schem
DateColumnWriter(column_id, stream_factory, column, num_rows_per_row_block, bf_fpp);
break;
}
+ case OLAP_FIELD_TYPE_DATEV2: {
+ column_writer = new (std::nothrow) DateV2ColumnWriter(column_id, stream_factory, column,
+ num_rows_per_row_block, bf_fpp);
+ break;
+ }
case OLAP_FIELD_TYPE_DECIMAL: {
column_writer = new (std::nothrow) DecimalColumnWriter(column_id, stream_factory, column,
num_rows_per_row_block, bf_fpp);
diff --git a/be/src/olap/rowset/column_writer.h b/be/src/olap/rowset/column_writer.h
index e4904de363..7771189e74 100644
--- a/be/src/olap/rowset/column_writer.h
+++ b/be/src/olap/rowset/column_writer.h
@@ -489,6 +489,9 @@ typedef IntegerColumnWriterWrapper<uint24_t, false> DateColumnWriter;
// DateTime is implemented with int64
typedef IntegerColumnWriterWrapper<uint64_t, false> DateTimeColumnWriter;
+// DateTime is implemented with int64
+typedef IntegerColumnWriterWrapper<uint32_t, false> DateV2ColumnWriter;
+
class DecimalColumnWriter : public ColumnWriter {
public:
DecimalColumnWriter(uint32_t column_id, OutStreamFactory* stream_factory,
diff --git a/be/src/olap/rowset/segment_v2/bitmap_index_writer.cpp b/be/src/olap/rowset/segment_v2/bitmap_index_writer.cpp
index 93cac8b5d2..9308c432b4 100644
--- a/be/src/olap/rowset/segment_v2/bitmap_index_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/bitmap_index_writer.cpp
@@ -220,6 +220,9 @@ Status BitmapIndexWriter::create(const TypeInfo* type_info,
case OLAP_FIELD_TYPE_DATETIME:
res->reset(new BitmapIndexWriterImpl<OLAP_FIELD_TYPE_DATETIME>(type_info));
break;
+ case OLAP_FIELD_TYPE_DATEV2:
+ res->reset(new BitmapIndexWriterImpl<OLAP_FIELD_TYPE_DATEV2>(type_info));
+ break;
case OLAP_FIELD_TYPE_LARGEINT:
res->reset(new BitmapIndexWriterImpl<OLAP_FIELD_TYPE_LARGEINT>(type_info));
break;
diff --git a/be/src/olap/rowset/segment_v2/column_reader.cpp b/be/src/olap/rowset/segment_v2/column_reader.cpp
index 8b0c8bc649..d790c8aee3 100644
--- a/be/src/olap/rowset/segment_v2/column_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/column_reader.cpp
@@ -888,6 +888,13 @@ void DefaultValueColumnIterator::insert_default_data(vectorized::MutableColumnPt
insert_column_data();
break;
}
+ case OLAP_FIELD_TYPE_DATEV2: {
+ assert(_type_size == sizeof(FieldTypeTraits<OLAP_FIELD_TYPE_DATEV2>::CppType)); //uint32_t
+
+ int128 = *((FieldTypeTraits<OLAP_FIELD_TYPE_DATEV2>::CppType*)_mem_value);
+ insert_column_data();
+ break;
+ }
case OLAP_FIELD_TYPE_DECIMAL: {
assert(_type_size ==
sizeof(FieldTypeTraits<OLAP_FIELD_TYPE_DECIMAL>::CppType)); //decimal12_t
diff --git a/be/src/olap/rowset/segment_v2/encoding_info.cpp b/be/src/olap/rowset/segment_v2/encoding_info.cpp
index 2c9a83e8a3..ac3a1a624f 100644
--- a/be/src/olap/rowset/segment_v2/encoding_info.cpp
+++ b/be/src/olap/rowset/segment_v2/encoding_info.cpp
@@ -135,6 +135,20 @@ struct TypeEncodingTraits<OLAP_FIELD_TYPE_DATE, FOR_ENCODING,
}
};
+template <>
+struct TypeEncodingTraits<OLAP_FIELD_TYPE_DATEV2, FOR_ENCODING,
+ typename CppTypeTraits<OLAP_FIELD_TYPE_DATEV2>::CppType> {
+ static Status create_page_builder(const PageBuilderOptions& opts, PageBuilder** builder) {
+ *builder = new FrameOfReferencePageBuilder<OLAP_FIELD_TYPE_DATEV2>(opts);
+ return Status::OK();
+ }
+ static Status create_page_decoder(const Slice& data, const PageDecoderOptions& opts,
+ PageDecoder** decoder) {
+ *decoder = new FrameOfReferencePageDecoder<OLAP_FIELD_TYPE_DATEV2>(data, opts);
+ return Status::OK();
+ }
+};
+
template <FieldType type, typename CppType>
struct TypeEncodingTraits<type, FOR_ENCODING, CppType,
typename std::enable_if<std::is_integral<CppType>::value>::type> {
@@ -268,6 +282,10 @@ EncodingInfoResolver::EncodingInfoResolver() {
_add_map<OLAP_FIELD_TYPE_DATE, PLAIN_ENCODING>();
_add_map<OLAP_FIELD_TYPE_DATE, FOR_ENCODING, true>();
+ _add_map<OLAP_FIELD_TYPE_DATEV2, BIT_SHUFFLE>();
+ _add_map<OLAP_FIELD_TYPE_DATEV2, PLAIN_ENCODING>();
+ _add_map<OLAP_FIELD_TYPE_DATEV2, FOR_ENCODING, true>();
+
_add_map<OLAP_FIELD_TYPE_DATETIME, BIT_SHUFFLE>();
_add_map<OLAP_FIELD_TYPE_DATETIME, PLAIN_ENCODING>();
_add_map<OLAP_FIELD_TYPE_DATETIME, FOR_ENCODING, true>();
diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
index 0769db56eb..61adede9f2 100644
--- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
@@ -831,7 +831,10 @@ void SegmentIterator::_init_current_block(
if (column_desc->type() == OLAP_FIELD_TYPE_DATE) {
current_columns[cid]->set_date_type();
} else if (column_desc->type() == OLAP_FIELD_TYPE_DATETIME) {
+ // TODO(Gabriel): support datetime v2
current_columns[cid]->set_datetime_type();
+ } else if (column_desc->type() == OLAP_FIELD_TYPE_DATEV2) {
+ current_columns[cid]->set_date_v2_type();
}
current_columns[cid]->reserve(_opts.block_row_max);
}
diff --git a/be/src/olap/schema.cpp b/be/src/olap/schema.cpp
index a5e7896147..dce43a1bdb 100644
--- a/be/src/olap/schema.cpp
+++ b/be/src/olap/schema.cpp
@@ -153,6 +153,9 @@ vectorized::IColumn::MutablePtr Schema::get_predicate_column_ptr(FieldType type)
case OLAP_FIELD_TYPE_DATE:
return doris::vectorized::PredicateColumnType<uint32_t>::create();
+ case OLAP_FIELD_TYPE_DATEV2:
+ return doris::vectorized::PredicateColumnType<uint32_t>::create();
+
case OLAP_FIELD_TYPE_DATETIME:
return doris::vectorized::PredicateColumnType<uint64_t>::create();
diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp
index 683dff9d62..cda1370b7f 100644
--- a/be/src/olap/tablet_schema.cpp
+++ b/be/src/olap/tablet_schema.cpp
@@ -58,6 +58,8 @@ FieldType TabletColumn::get_field_type_by_string(const std::string& type_str) {
type = OLAP_FIELD_TYPE_CHAR;
} else if (0 == upper_type_str.compare("DATE")) {
type = OLAP_FIELD_TYPE_DATE;
+ } else if (0 == upper_type_str.compare("DATEV2")) {
+ type = OLAP_FIELD_TYPE_DATEV2;
} else if (0 == upper_type_str.compare("DATETIME")) {
type = OLAP_FIELD_TYPE_DATETIME;
} else if (0 == upper_type_str.compare(0, 7, "DECIMAL")) {
@@ -165,6 +167,9 @@ std::string TabletColumn::get_string_by_field_type(FieldType type) {
case OLAP_FIELD_TYPE_DATE:
return "DATE";
+ case OLAP_FIELD_TYPE_DATEV2:
+ return "DATEV2";
+
case OLAP_FIELD_TYPE_DATETIME:
return "DATETIME";
@@ -251,6 +256,8 @@ uint32_t TabletColumn::get_field_length_by_type(TPrimitiveType::type type, uint3
return 16;
case TPrimitiveType::DATE:
return 3;
+ case TPrimitiveType::DATEV2:
+ return 4;
case TPrimitiveType::DATETIME:
return 8;
case TPrimitiveType::FLOAT:
diff --git a/be/src/olap/types.cpp b/be/src/olap/types.cpp
index bced5024c1..d1b2f345bf 100644
--- a/be/src/olap/types.cpp
+++ b/be/src/olap/types.cpp
@@ -83,6 +83,7 @@ const TypeInfo* get_scalar_type_info(FieldType field_type) {
get_scalar_type_info<OLAP_FIELD_TYPE_OBJECT>(),
get_scalar_type_info<OLAP_FIELD_TYPE_STRING>(),
get_scalar_type_info<OLAP_FIELD_TYPE_QUANTILE_STATE>(),
+ get_scalar_type_info<OLAP_FIELD_TYPE_DATEV2>(),
};
return field_type_array[field_type];
}
@@ -147,6 +148,7 @@ const TypeInfo* get_array_type_info(FieldType leaf_type, int32_t iterations) {
INIT_ARRAY_TYPE_INFO_LIST(OLAP_FIELD_TYPE_OBJECT),
INIT_ARRAY_TYPE_INFO_LIST(OLAP_FIELD_TYPE_STRING),
INIT_ARRAY_TYPE_INFO_LIST(OLAP_FIELD_TYPE_QUANTILE_STATE),
+ INIT_ARRAY_TYPE_INFO_LIST(OLAP_FIELD_TYPE_DATEV2),
};
return array_type_Info_arr[leaf_type][iterations];
}
diff --git a/be/src/olap/types.h b/be/src/olap/types.h
index 915239b0ee..6f5bb48ef3 100644
--- a/be/src/olap/types.h
+++ b/be/src/olap/types.h
@@ -500,6 +500,11 @@ struct CppTypeTraits<OLAP_FIELD_TYPE_DATE> {
using UnsignedCppType = uint24_t;
};
template <>
+struct CppTypeTraits<OLAP_FIELD_TYPE_DATEV2> {
+ using CppType = uint32_t;
+ using UnsignedCppType = uint32_t;
+};
+template <>
struct CppTypeTraits<OLAP_FIELD_TYPE_DATETIME> {
using CppType = int64_t;
using UnsignedCppType = uint64_t;
@@ -926,6 +931,15 @@ struct FieldTypeTraits<OLAP_FIELD_TYPE_DATE> : public BaseFieldtypeTraits<OLAP_F
CppType mday = static_cast<CppType>(part1 % 100);
*reinterpret_cast<CppType*>(dest) = (year << 9) + (mon << 5) + mday;
return Status::OK();
+ } else if (src_type->type() == FieldType::OLAP_FIELD_TYPE_DATEV2) {
+ using SrcType = typename CppTypeTraits<OLAP_FIELD_TYPE_DATEV2>::CppType;
+ SrcType src_value = *reinterpret_cast<const SrcType*>(src);
+ //only need part one
+ CppType year = static_cast<CppType>((src_value & 0xFFFF0000) >> 16);
+ CppType mon = static_cast<CppType>((src_value & 0xFF00) >> 8);
+ CppType mday = static_cast<CppType>(src_value & 0xFF);
+ *reinterpret_cast<CppType*>(dest) = (year << 9) + (mon << 5) + mday;
+ return Status::OK();
}
if (src_type->type() == FieldType::OLAP_FIELD_TYPE_INT) {
@@ -974,6 +988,104 @@ struct FieldTypeTraits<OLAP_FIELD_TYPE_DATE> : public BaseFieldtypeTraits<OLAP_F
}
};
+template <>
+struct FieldTypeTraits<OLAP_FIELD_TYPE_DATEV2>
+ : public BaseFieldtypeTraits<OLAP_FIELD_TYPE_DATEV2> {
+ static Status from_string(void* buf, const std::string& scan_key) {
+ tm time_tm;
+ char* res = strptime(scan_key.c_str(), "%Y-%m-%d", &time_tm);
+
+ if (nullptr != res) {
+ uint32_t value = ((time_tm.tm_year + 1900) << 16) | ((time_tm.tm_mon + 1) << 8) |
+ time_tm.tm_mday;
+ *reinterpret_cast<CppType*>(buf) = value;
+ } else {
+ *reinterpret_cast<CppType*>(buf) = doris::vectorized::MIN_DATE_V2;
+ }
+
+ return Status::OK();
+ }
+ static std::string to_string(const void* src) {
+ CppType tmp = *reinterpret_cast<const CppType*>(src);
+ doris::vectorized::DateV2Value value =
+ binary_cast<CppType, doris::vectorized::DateV2Value>(tmp);
+ string format = "%Y-%m-%d";
+ string res;
+ res.resize(12);
+ res.reserve(12);
+ value.to_format_string(format.c_str(), format.size(), res.data());
+ return res;
+ }
+ static Status convert_from(void* dest, const void* src, const TypeInfo* src_type,
+ MemPool* mem_pool, size_t variable_len = 0) {
+ if (src_type->type() == FieldType::OLAP_FIELD_TYPE_DATETIME) {
+ using SrcType = typename CppTypeTraits<OLAP_FIELD_TYPE_DATETIME>::CppType;
+ SrcType src_value = *reinterpret_cast<const SrcType*>(src);
+ //only need part one
+ SrcType part1 = (src_value / 1000000L);
+ CppType year = static_cast<CppType>((part1 / 10000L) % 10000);
+ CppType mon = static_cast<CppType>((part1 / 100) % 100);
+ CppType mday = static_cast<CppType>(part1 % 100);
+ *reinterpret_cast<CppType*>(dest) = (year << 16) | (mon << 8) | mday;
+ return Status::OK();
+ }
+
+ if (src_type->type() == FieldType::OLAP_FIELD_TYPE_DATE) {
+ using SrcType = typename CppTypeTraits<OLAP_FIELD_TYPE_DATE>::CppType;
+ SrcType value = *reinterpret_cast<const SrcType*>(src);
+ int day = static_cast<int>(value & 31);
+ int mon = static_cast<int>(value >> 5 & 15);
+ int year = static_cast<int>(value >> 9);
+ *reinterpret_cast<CppType*>(dest) = (year << 16) | (mon << 8) | day;
+ return Status::OK();
+ }
+
+ if (src_type->type() == FieldType::OLAP_FIELD_TYPE_INT) {
+ using SrcType = typename CppTypeTraits<OLAP_FIELD_TYPE_INT>::CppType;
+ SrcType src_value = *reinterpret_cast<const SrcType*>(src);
+ doris::vectorized::DateV2Value dt;
+ if (!dt.from_date_int64(src_value)) {
+ return Status::OLAPInternalError(OLAP_ERR_INVALID_SCHEMA);
+ }
+ CppType year = static_cast<CppType>(src_value / 10000);
+ CppType month = static_cast<CppType>((src_value % 10000) / 100);
+ CppType day = static_cast<CppType>(src_value % 100);
+ *reinterpret_cast<CppType*>(dest) = (year << 16) | (month << 8) | day;
+ return Status::OK();
+ }
+
+ if (src_type->type() == OLAP_FIELD_TYPE_VARCHAR ||
+ src_type->type() == OLAP_FIELD_TYPE_CHAR ||
+ src_type->type() == OLAP_FIELD_TYPE_STRING) {
+ if (src_type->type() == OLAP_FIELD_TYPE_CHAR) {
+ prepare_char_before_convert(src);
+ }
+ using SrcType = typename CppTypeTraits<OLAP_FIELD_TYPE_VARCHAR>::CppType;
+ auto src_value = *reinterpret_cast<const SrcType*>(src);
+ doris::vectorized::DateV2Value dt;
+ for (const auto& format : DATE_FORMATS) {
+ if (dt.from_date_format_str(format.c_str(), format.length(), src_value.get_data(),
+ src_value.get_size())) {
+ *reinterpret_cast<CppType*>(dest) =
+ (dt.year() << 16) | (dt.month() << 8) | dt.day();
+ return Status::OK();
+ }
+ }
+ return Status::OLAPInternalError(OLAP_ERR_INVALID_SCHEMA);
+ }
+
+ return Status::OLAPInternalError(OLAP_ERR_INVALID_SCHEMA);
+ }
+ static void set_to_max(void* buf) {
+ // max is 9999 * 16 * 32 + 12 * 32 + 31;
+ *reinterpret_cast<CppType*>(buf) = doris::vectorized::MAX_DATE_V2;
+ }
+ static void set_to_min(void* buf) {
+ // min is 0 * 16 * 32 + 1 * 32 + 1;
+ *reinterpret_cast<CppType*>(buf) = doris::vectorized::MIN_DATE_V2;
+ }
+};
+
template <>
struct FieldTypeTraits<OLAP_FIELD_TYPE_DATETIME>
: public BaseFieldtypeTraits<OLAP_FIELD_TYPE_DATETIME> {
@@ -1023,6 +1135,14 @@ struct FieldTypeTraits<OLAP_FIELD_TYPE_DATETIME>
int year = static_cast<int>(value >> 9);
*reinterpret_cast<CppType*>(dest) = (year * 10000L + mon * 100L + day) * 1000000;
return Status::OK();
+ } else if (src_type->type() == FieldType::OLAP_FIELD_TYPE_DATEV2) {
+ using SrcType = typename CppTypeTraits<OLAP_FIELD_TYPE_DATEV2>::CppType;
+ auto value = *reinterpret_cast<const SrcType*>(src);
+ int day = static_cast<int>(value & 0xFF);
+ int mon = static_cast<int>((value & 0xFF00) >> 8);
+ int year = static_cast<int>((value & 0xFFFF0000) >> 16);
+ *reinterpret_cast<CppType*>(dest) = (year * 10000L + mon * 100L + day) * 1000000;
+ return Status::OK();
}
return Status::OLAPInternalError(OLAP_ERR_INVALID_SCHEMA);
}
diff --git a/be/src/olap/utils.h b/be/src/olap/utils.h
index b84fe024ce..8cd6f2b2c4 100644
--- a/be/src/olap/utils.h
+++ b/be/src/olap/utils.h
@@ -266,7 +266,8 @@ bool valid_unsigned_number(const std::string& value_str) {
bool valid_decimal(const std::string& value_str, const uint32_t precision, const uint32_t frac);
-// 粗略检查date或者datetime类型是否正确
+// Validate for date/datetime roughly. The format is 'yyyy-MM-dd HH:mm:ss'
+// TODO: support 'yyyy-MM-dd HH:mm:ss.SSS'
bool valid_datetime(const std::string& value_str);
bool valid_bool(const std::string& value_str);
diff --git a/be/src/runtime/datetime_value.h b/be/src/runtime/datetime_value.h
index 21ed59b2d1..20caa2a428 100644
--- a/be/src/runtime/datetime_value.h
+++ b/be/src/runtime/datetime_value.h
@@ -570,6 +570,8 @@ private:
friend class UnusedClass;
friend void doris::vectorized::VecDateTimeValue::convert_vec_dt_to_dt(DateTimeValue* dt);
friend void doris::vectorized::VecDateTimeValue::convert_dt_to_vec_dt(DateTimeValue* dt);
+ friend void doris::vectorized::DateV2Value::convert_date_v2_to_dt(DateTimeValue* dt);
+ friend void doris::vectorized::DateV2Value::convert_dt_to_date_v2(DateTimeValue* dt);
void from_packed_time(int64_t packed_time) {
_microsecond = packed_time % (1LL << 24);
diff --git a/be/src/runtime/primitive_type.cpp b/be/src/runtime/primitive_type.cpp
index 2521559ab6..1a69e3368b 100644
--- a/be/src/runtime/primitive_type.cpp
+++ b/be/src/runtime/primitive_type.cpp
@@ -65,6 +65,12 @@ PrimitiveType convert_type_to_primitive(FunctionContext::Type type) {
return PrimitiveType::TYPE_LARGEINT;
case FunctionContext::Type::TYPE_DATE:
return PrimitiveType::TYPE_DATE;
+ case FunctionContext::Type::TYPE_DATEV2:
+ return PrimitiveType::TYPE_DATEV2;
+ case FunctionContext::Type::TYPE_DATETIMEV2:
+ return PrimitiveType::TYPE_DATETIMEV2;
+ case FunctionContext::Type::TYPE_TIMEV2:
+ return PrimitiveType::TYPE_TIMEV2;
default:
DCHECK(false);
}
@@ -81,6 +87,8 @@ bool is_enumeration_type(PrimitiveType type) {
case TYPE_VARCHAR:
case TYPE_STRING:
case TYPE_DATETIME:
+ case TYPE_DATETIMEV2:
+ case TYPE_TIMEV2:
case TYPE_DECIMALV2:
case TYPE_BOOLEAN:
case TYPE_ARRAY:
@@ -92,6 +100,7 @@ bool is_enumeration_type(PrimitiveType type) {
case TYPE_BIGINT:
case TYPE_LARGEINT:
case TYPE_DATE:
+ case TYPE_DATEV2:
return true;
case INVALID_TYPE:
@@ -103,7 +112,8 @@ bool is_enumeration_type(PrimitiveType type) {
}
bool is_date_type(PrimitiveType type) {
- return type == TYPE_DATETIME || type == TYPE_DATE;
+ return type == TYPE_DATETIME || type == TYPE_DATE || type == TYPE_DATETIMEV2 ||
+ type == TYPE_DATEV2;
}
bool is_string_type(PrimitiveType type) {
@@ -151,6 +161,10 @@ int get_byte_size(PrimitiveType type) {
return 16;
case INVALID_TYPE:
+ // datev2/datetimev2/timev2 is not supported on row-based engine
+ case TYPE_DATEV2:
+ case TYPE_DATETIMEV2:
+ case TYPE_TIMEV2:
default:
DCHECK(false);
}
@@ -233,6 +247,15 @@ PrimitiveType thrift_to_type(TPrimitiveType::type ttype) {
case TPrimitiveType::DATETIME:
return TYPE_DATETIME;
+ case TPrimitiveType::DATEV2:
+ return TYPE_DATEV2;
+
+ case TPrimitiveType::DATETIMEV2:
+ return TYPE_DATETIMEV2;
+
+ case TPrimitiveType::TIMEV2:
+ return TYPE_TIMEV2;
+
case TPrimitiveType::TIME:
return TYPE_TIME;
@@ -309,6 +332,15 @@ TPrimitiveType::type to_thrift(PrimitiveType ptype) {
case TYPE_TIME:
return TPrimitiveType::TIME;
+ case TYPE_DATEV2:
+ return TPrimitiveType::DATEV2;
+
+ case TYPE_DATETIMEV2:
+ return TPrimitiveType::DATETIMEV2;
+
+ case TYPE_TIMEV2:
+ return TPrimitiveType::TIMEV2;
+
case TYPE_VARCHAR:
return TPrimitiveType::VARCHAR;
@@ -382,6 +414,15 @@ std::string type_to_string(PrimitiveType t) {
case TYPE_TIME:
return "TIME";
+ case TYPE_DATEV2:
+ return "DATEV2";
+
+ case TYPE_DATETIMEV2:
+ return "DATETIMEV2";
+
+ case TYPE_TIMEV2:
+ return "TIMEV2";
+
case TYPE_VARCHAR:
return "VARCHAR";
@@ -456,6 +497,15 @@ std::string type_to_odbc_string(PrimitiveType t) {
case TYPE_DATETIME:
return "datetime";
+ case TYPE_DATEV2:
+ return "datev2";
+
+ case TYPE_DATETIMEV2:
+ return "datetimev2";
+
+ case TYPE_TIMEV2:
+ return "timev2";
+
case TYPE_VARCHAR:
return "string";
@@ -535,6 +585,7 @@ int get_slot_size(PrimitiveType type) {
return 2;
case TYPE_INT:
+ case TYPE_DATEV2:
case TYPE_FLOAT:
return 4;
@@ -548,6 +599,8 @@ int get_slot_size(PrimitiveType type) {
case TYPE_DATE:
case TYPE_DATETIME:
+ case TYPE_DATETIMEV2:
+ case TYPE_TIMEV2:
// This is the size of the slot, the actual size of the data is 12.
return sizeof(DateTimeValue);
diff --git a/be/src/runtime/primitive_type.h b/be/src/runtime/primitive_type.h
index 7ba53588b4..3c0254013a 100644
--- a/be/src/runtime/primitive_type.h
+++ b/be/src/runtime/primitive_type.h
@@ -58,10 +58,13 @@ enum PrimitiveType {
TYPE_HLL, /* 19 */
TYPE_DECIMALV2, /* 20 */
- TYPE_TIME, /* 21 */
- TYPE_OBJECT, /* 22 */
- TYPE_STRING, /* 23 */
- TYPE_QUANTILE_STATE /* 24 */
+ TYPE_TIME, /* 21 */
+ TYPE_OBJECT, /* 22 */
+ TYPE_STRING, /* 23 */
+ TYPE_QUANTILE_STATE, /* 24 */
+ TYPE_DATEV2, /* 25 */
+ TYPE_DATETIMEV2, /* 26 */
+ TYPE_TIMEV2, /* 27 */
};
PrimitiveType convert_type_to_primitive(FunctionContext::Type type);
@@ -141,6 +144,11 @@ struct PrimitiveTypeTraits<TYPE_DATETIME> {
using ColumnType = vectorized::ColumnVector<vectorized::DateTime>;
};
template <>
+struct PrimitiveTypeTraits<TYPE_DATEV2> {
+ using CppType = doris::vectorized::DateV2Value;
+ using ColumnType = vectorized::ColumnVector<vectorized::DateV2>;
+};
+template <>
struct PrimitiveTypeTraits<TYPE_DECIMALV2> {
using CppType = DecimalV2Value;
using ColumnType = vectorized::ColumnDecimal<vectorized::Decimal128>;
@@ -188,4 +196,14 @@ struct PredicatePrimitiveTypeTraits<TYPE_DATETIME> {
using PredicateFieldType = uint64_t;
};
+template <>
+struct PredicatePrimitiveTypeTraits<TYPE_DATEV2> {
+ using PredicateFieldType = uint32_t;
+};
+
+template <>
+struct PredicatePrimitiveTypeTraits<TYPE_DATETIMEV2> {
+ using PredicateFieldType = uint64_t;
+};
+
} // namespace doris
diff --git a/be/src/runtime/raw_value.cpp b/be/src/runtime/raw_value.cpp
index ac0a95c0c4..740ca3e47a 100644
--- a/be/src/runtime/raw_value.cpp
+++ b/be/src/runtime/raw_value.cpp
@@ -84,6 +84,10 @@ void RawValue::print_value_as_bytes(const void* value, const TypeDescriptor& typ
stream->write(chars, sizeof(DateTimeValue));
break;
+ case TYPE_DATEV2:
+ stream->write(chars, sizeof(doris::vectorized::DateV2Value));
+ break;
+
case TYPE_DECIMALV2:
stream->write(chars, sizeof(DecimalV2Value));
break;
@@ -162,6 +166,10 @@ void RawValue::print_value(const void* value, const TypeDescriptor& type, int sc
*stream << *reinterpret_cast<const DateTimeValue*>(value);
break;
+ case TYPE_DATEV2:
+ *stream << *reinterpret_cast<const doris::vectorized::DateV2Value*>(value);
+ break;
+
case TYPE_DECIMALV2:
*stream << DecimalV2Value(reinterpret_cast<const PackedInt128*>(value)->value).to_string();
break;
@@ -293,6 +301,11 @@ void RawValue::write(const void* value, void* dst, const TypeDescriptor& type, M
*reinterpret_cast<DateTimeValue*>(dst) = *reinterpret_cast<const DateTimeValue*>(value);
break;
+ case TYPE_DATEV2:
+ *reinterpret_cast<doris::vectorized::DateV2Value*>(dst) =
+ *reinterpret_cast<const doris::vectorized::DateV2Value*>(value);
+ break;
+
case TYPE_DECIMALV2:
*reinterpret_cast<PackedInt128*>(dst) = *reinterpret_cast<const PackedInt128*>(value);
break;
@@ -378,6 +391,10 @@ void RawValue::write(const void* value, const TypeDescriptor& type, void* dst, u
case TYPE_DATETIME:
*reinterpret_cast<DateTimeValue*>(dst) = *reinterpret_cast<const DateTimeValue*>(value);
break;
+ case TYPE_DATEV2:
+ *reinterpret_cast<doris::vectorized::DateV2Value*>(dst) =
+ *reinterpret_cast<const doris::vectorized::DateV2Value*>(value);
+ break;
case TYPE_VARCHAR:
case TYPE_CHAR:
case TYPE_STRING: {
@@ -481,6 +498,12 @@ int RawValue::compare(const void* v1, const void* v2, const TypeDescriptor& type
ts_value2 = reinterpret_cast<const DateTimeValue*>(v2);
return *ts_value1 > *ts_value2 ? 1 : (*ts_value1 < *ts_value2 ? -1 : 0);
+ case TYPE_DATEV2: {
+ auto date_v2_value1 = reinterpret_cast<const doris::vectorized::DateV2Value*>(v1);
+ auto date_v2_value2 = reinterpret_cast<const doris::vectorized::DateV2Value*>(v2);
+ return *date_v2_value1 > *date_v2_value2 ? 1 : (*date_v2_value1 < *date_v2_value2 ? -1 : 0);
+ }
+
case TYPE_DECIMALV2: {
DecimalV2Value decimal_value1(reinterpret_cast<const PackedInt128*>(v1)->value);
DecimalV2Value decimal_value2(reinterpret_cast<const PackedInt128*>(v2)->value);
diff --git a/be/src/runtime/raw_value.h b/be/src/runtime/raw_value.h
index 6804eecaad..424ede11e8 100644
--- a/be/src/runtime/raw_value.h
+++ b/be/src/runtime/raw_value.h
@@ -160,6 +160,10 @@ inline bool RawValue::lt(const void* v1, const void* v2, const TypeDescriptor& t
return *reinterpret_cast<const DateTimeValue*>(v1) <
*reinterpret_cast<const DateTimeValue*>(v2);
+ case TYPE_DATEV2:
+ return *reinterpret_cast<const vectorized::DateV2Value*>(v1) <
+ *reinterpret_cast<const vectorized::DateV2Value*>(v2);
+
case TYPE_DECIMALV2:
return reinterpret_cast<const PackedInt128*>(v1)->value <
reinterpret_cast<const PackedInt128*>(v2)->value;
@@ -212,6 +216,10 @@ inline bool RawValue::eq(const void* v1, const void* v2, const TypeDescriptor& t
return *reinterpret_cast<const DateTimeValue*>(v1) ==
*reinterpret_cast<const DateTimeValue*>(v2);
+ case TYPE_DATEV2:
+ return *reinterpret_cast<const vectorized::DateV2Value*>(v1) ==
+ *reinterpret_cast<const vectorized::DateV2Value*>(v2);
+
case TYPE_DECIMALV2:
return reinterpret_cast<const PackedInt128*>(v1)->value ==
reinterpret_cast<const PackedInt128*>(v2)->value;
@@ -273,6 +281,9 @@ inline uint32_t RawValue::get_hash_value(const void* v, const PrimitiveType& typ
case TYPE_DATETIME:
return HashUtil::hash(v, 16, seed);
+ case TYPE_DATEV2:
+ return HashUtil::hash(v, 4, seed);
+
case TYPE_DECIMALV2:
return HashUtil::hash(v, 16, seed);
@@ -330,6 +341,9 @@ inline uint32_t RawValue::get_hash_value_fvn(const void* v, const PrimitiveType&
case TYPE_DATETIME:
return HashUtil::fnv_hash(v, 16, seed);
+ case TYPE_DATEV2:
+ return HashUtil::fnv_hash(v, 4, seed);
+
case TYPE_DECIMALV2:
return HashUtil::fnv_hash(v, 16, seed);
@@ -392,6 +406,12 @@ inline uint32_t RawValue::zlib_crc32(const void* v, const TypeDescriptor& type,
int len = date_val->to_buffer(buf);
return HashUtil::zlib_crc_hash(buf, len, seed);
}
+ case TYPE_DATEV2: {
+ const vectorized::DateV2Value* date_v2_val = (const vectorized::DateV2Value*)v;
+ char buf[64];
+ int len = date_v2_val->to_buffer(buf);
+ return HashUtil::zlib_crc_hash(buf, len, seed);
+ }
case TYPE_DECIMALV2: {
const DecimalV2Value* dec_val = (const DecimalV2Value*)v;
@@ -446,6 +466,12 @@ inline uint32_t RawValue::zlib_crc32(const void* v, size_t len, const TypeDescri
int len = date_val->to_buffer(buf);
return HashUtil::zlib_crc_hash(buf, len, seed);
}
+ case TYPE_DATEV2: {
+ auto* date_v2_val = (const vectorized::DateV2Value*)v;
+ char buf[64];
+ int date_v2_len = date_v2_val->to_buffer(buf);
+ return HashUtil::zlib_crc_hash(buf, date_v2_len, seed);
+ }
case TYPE_DECIMALV2: {
const DecimalV2Value* dec_val = (const DecimalV2Value*)v;
diff --git a/be/src/runtime/type_limit.h b/be/src/runtime/type_limit.h
index 7cc567549d..b868e25447 100644
--- a/be/src/runtime/type_limit.h
+++ b/be/src/runtime/type_limit.h
@@ -47,4 +47,16 @@ struct type_limit<DateTimeValue> {
static DateTimeValue max() { return DateTimeValue::datetime_max_value(); }
};
+template <>
+struct type_limit<doris::vectorized::DateV2Value> {
+ static doris::vectorized::DateV2Value min() {
+ uint32_t min = doris::vectorized::MIN_DATE_V2;
+ return binary_cast<uint32_t, doris::vectorized::DateV2Value>(min);
+ }
+ static doris::vectorized::DateV2Value max() {
+ uint32_t max = doris::vectorized::MAX_DATE_V2;
+ return binary_cast<uint32_t, doris::vectorized::DateV2Value>(max);
+ }
+};
+
} // namespace doris
diff --git a/be/src/runtime/types.h b/be/src/runtime/types.h
index 5852c2fb91..20cdc4663b 100644
--- a/be/src/runtime/types.h
+++ b/be/src/runtime/types.h
@@ -167,6 +167,8 @@ struct TypeDescriptor {
bool is_date_type() const { return type == TYPE_DATE || type == TYPE_DATETIME; }
+ bool is_date_v2_type() const { return type == TYPE_DATEV2 || type == TYPE_DATETIMEV2; }
+
bool is_decimal_type() const { return (type == TYPE_DECIMALV2); }
bool is_datetime_type() const { return type == TYPE_DATETIME; }
diff --git a/be/src/udf/udf.h b/be/src/udf/udf.h
index f46b7c6129..2a5296f477 100644
--- a/be/src/udf/udf.h
+++ b/be/src/udf/udf.h
@@ -87,7 +87,10 @@ public:
TYPE_DECIMALV2,
TYPE_OBJECT,
TYPE_ARRAY,
- TYPE_QUANTILE_STATE
+ TYPE_QUANTILE_STATE,
+ TYPE_DATEV2,
+ TYPE_DATETIMEV2,
+ TYPE_TIMEV2
};
struct TypeDesc {
diff --git a/be/src/util/binary_cast.hpp b/be/src/util/binary_cast.hpp
index 077885482b..013c9e598e 100644
--- a/be/src/util/binary_cast.hpp
+++ b/be/src/util/binary_cast.hpp
@@ -62,6 +62,13 @@ union VecDateTimeInt64Union {
__int64_t i64;
~VecDateTimeInt64Union() {}
};
+
+union DateV2UInt32Union {
+ doris::vectorized::DateV2Value dt;
+ uint32_t ui32;
+ ~DateV2UInt32Union() {}
+};
+
// similar to reinterpret_cast but won't break strict-aliasing rules
template <typename From, typename To>
To binary_cast(From from) {
@@ -79,10 +86,16 @@ To binary_cast(From from) {
constexpr bool from_i128_to_decv2 = match_v<From, __int128_t, To, DecimalV2Value>;
constexpr bool from_decv2_to_i128 = match_v<From, DecimalV2Value, To, __int128_t>;
+ constexpr bool from_ui32_to_date_v2 =
+ match_v<From, uint32_t, To, doris::vectorized::DateV2Value>;
+
+ constexpr bool from_date_v2_to_ui32 =
+ match_v<From, doris::vectorized::DateV2Value, To, uint32_t>;
+
static_assert(from_u64_to_db || from_i64_to_db || from_db_to_i64 || from_db_to_u64 ||
from_decv2_to_packed128 || from_i128_to_dt || from_dt_to_i128 ||
from_i64_to_vec_dt || from_vec_dt_to_i64 || from_i128_to_decv2 ||
- from_decv2_to_i128);
+ from_decv2_to_i128 || from_ui32_to_date_v2 || from_date_v2_to_ui32);
if constexpr (from_u64_to_db) {
TypeConverter conv;
@@ -113,6 +126,12 @@ To binary_cast(From from) {
} else if constexpr (from_i64_to_vec_dt) {
VecDateTimeInt64Union conv = {.i64 = from};
return conv.dt;
+ } else if constexpr (from_ui32_to_date_v2) {
+ DateV2UInt32Union conv = {.ui32 = from};
+ return conv.dt;
+ } else if constexpr (from_date_v2_to_ui32) {
+ DateV2UInt32Union conv = {.dt = from};
+ return conv.ui32;
} else if constexpr (from_vec_dt_to_i64) {
VecDateTimeInt64Union conv = {.dt = from};
return conv.i64;
diff --git a/be/src/util/date_func.cpp b/be/src/util/date_func.cpp
index 93559b684f..4f7add3ad9 100644
--- a/be/src/util/date_func.cpp
+++ b/be/src/util/date_func.cpp
@@ -19,6 +19,8 @@
#include <iomanip>
+#include "vec/runtime/vdatetime_value.h"
+
namespace doris {
uint64_t timestamp_from_datetime(const std::string& datetime_str) {
@@ -53,6 +55,20 @@ uint24_t timestamp_from_date(const std::string& date_str) {
return uint24_t(value);
}
+
+uint32_t timestamp_from_date_v2(const std::string& date_str) {
+ tm time_tm;
+ char* res = strptime(date_str.c_str(), "%Y-%m-%d", &time_tm);
+
+ uint32_t value = 0;
+ if (nullptr != res) {
+ value = ((time_tm.tm_year + 1900) << 16) | ((time_tm.tm_mon + 1) << 8) | time_tm.tm_mday;
+ } else {
+ value = doris::vectorized::MIN_DATE_V2;
+ }
+
+ return value;
+}
// refer to https://dev.mysql.com/doc/refman/5.7/en/time.html
// the time value between '-838:59:59' and '838:59:59'
int32_t time_to_buffer_from_double(double time, char* buffer) {
diff --git a/be/src/util/date_func.h b/be/src/util/date_func.h
index f3cb926a62..507ee9652e 100644
--- a/be/src/util/date_func.h
+++ b/be/src/util/date_func.h
@@ -29,5 +29,6 @@ namespace doris {
uint64_t timestamp_from_datetime(const std::string& datetime_str);
uint24_t timestamp_from_date(const std::string& date_str);
int32_t time_to_buffer_from_double(double time, char* buffer);
+uint32_t timestamp_from_date_v2(const std::string& date_str);
} // namespace doris
diff --git a/be/src/vec/CMakeLists.txt b/be/src/vec/CMakeLists.txt
index 69aaf816ba..d6549701b0 100644
--- a/be/src/vec/CMakeLists.txt
+++ b/be/src/vec/CMakeLists.txt
@@ -78,6 +78,7 @@ set(VEC_FILES
data_types/nested_utils.cpp
data_types/data_type_date.cpp
data_types/data_type_date_time.cpp
+ data_types/data_type_time_v2.cpp
exec/vaggregation_node.cpp
exec/varrow_scanner.cpp
exec/ves_http_scan_node.cpp
diff --git a/be/src/vec/aggregate_functions/aggregate_function_min_max.cpp b/be/src/vec/aggregate_functions/aggregate_function_min_max.cpp
index 813674a312..be4ce7ef0d 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_min_max.cpp
+++ b/be/src/vec/aggregate_functions/aggregate_function_min_max.cpp
@@ -50,6 +50,10 @@ static IAggregateFunction* create_aggregate_function_single_value(const String&
return new AggregateFunctionTemplate<Data<SingleValueDataFixed<Int64>>, false>(
argument_type);
}
+ if (which.idx == TypeIndex::DateV2) {
+ return new AggregateFunctionTemplate<Data<SingleValueDataFixed<UInt32>>, false>(
+ argument_type);
+ }
if (which.idx == TypeIndex::Decimal128) {
return new AggregateFunctionTemplate<Data<SingleValueDataFixed<DecimalV2Value>>, false>(
argument_type);
diff --git a/be/src/vec/aggregate_functions/aggregate_function_min_max_by.cpp b/be/src/vec/aggregate_functions/aggregate_function_min_max_by.cpp
index 0b88f5106e..8b834f4202 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_min_max_by.cpp
+++ b/be/src/vec/aggregate_functions/aggregate_function_min_max_by.cpp
@@ -47,6 +47,10 @@ static IAggregateFunction* create_aggregate_function_min_max_by_impl(
return new AggregateFunctionTemplate<Data<VT, SingleValueDataFixed<Int64>>, false>(
value_arg_type, key_arg_type);
}
+ if (which.idx == TypeIndex::DateV2) {
+ return new AggregateFunctionTemplate<Data<VT, SingleValueDataFixed<UInt32>>, false>(
+ value_arg_type, key_arg_type);
+ }
if (which.idx == TypeIndex::Decimal128) {
return new AggregateFunctionTemplate<Data<VT, SingleValueDataFixed<DecimalV2Value>>, false>(
value_arg_type, key_arg_type);
@@ -82,6 +86,11 @@ static IAggregateFunction* create_aggregate_function_min_max_by(const String& na
SingleValueDataFixed<Int64>>(
argument_types);
}
+ if (which.idx == TypeIndex::DateV2) {
+ return create_aggregate_function_min_max_by_impl<AggregateFunctionTemplate, Data,
+ SingleValueDataFixed<UInt32>>(
+ argument_types);
+ }
if (which.idx == TypeIndex::Decimal128) {
return create_aggregate_function_min_max_by_impl<AggregateFunctionTemplate, Data,
SingleValueDataFixed<DecimalV2Value>>(
diff --git a/be/src/vec/aggregate_functions/aggregate_function_window.h b/be/src/vec/aggregate_functions/aggregate_function_window.h
index 37b73bd7c1..ab519156ab 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_window.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_window.h
@@ -579,6 +579,10 @@ static IAggregateFunction* create_function_single_value(const String& name,
return new AggregateFunctionTemplate<
Data<LeadAndLagData<Int64, result_is_nullable, false, StoreType>>>(argument_types);
}
+ if (which.is_date_v2()) {
+ return new AggregateFunctionTemplate<
+ Data<LeadAndLagData<UInt32, result_is_nullable, false, StoreType>>>(argument_types);
+ }
if (which.is_string_or_fixed_string()) {
return new AggregateFunctionTemplate<
Data<LeadAndLagData<StringRef, result_is_nullable, true, StoreType>>>(
diff --git a/be/src/vec/aggregate_functions/aggregate_function_window_funnel.cpp b/be/src/vec/aggregate_functions/aggregate_function_window_funnel.cpp
index a889299f33..dda947bb10 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_window_funnel.cpp
+++ b/be/src/vec/aggregate_functions/aggregate_function_window_funnel.cpp
@@ -28,7 +28,7 @@ AggregateFunctionPtr create_aggregate_function_window_funnel(const std::string&
const DataTypes& argument_types,
const Array& parameters,
const bool result_is_nullable) {
- return std::make_shared<AggregateFunctionWindowFunnel>(argument_types);
+ return std::make_shared<AggregateFunctionWindowFunnel<VecDateTimeValue, Int64>>(argument_types);
}
void register_aggregate_function_window_funnel(AggregateFunctionSimpleFactory& factory) {
diff --git a/be/src/vec/aggregate_functions/aggregate_function_window_funnel.h b/be/src/vec/aggregate_functions/aggregate_function_window_funnel.h
index 7b62ea2ad6..92eb8d99f0 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_window_funnel.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_window_funnel.h
@@ -29,8 +29,9 @@
namespace doris::vectorized {
+template <typename DateValueType, typename NativeType>
struct WindowFunnelState {
- std::vector<std::pair<VecDateTimeValue, int>> events;
+ std::vector<std::pair<DateValueType, int>> events;
int max_event_level;
bool sorted;
int64_t window;
@@ -48,7 +49,7 @@ struct WindowFunnelState {
events.shrink_to_fit();
}
- void add(const VecDateTimeValue& timestamp, int event_idx, int event_num, int64_t win) {
+ void add(const DateValueType& timestamp, int event_idx, int event_num, int64_t win) {
window = win;
max_event_level = event_num;
if (sorted && events.size() > 0) {
@@ -69,17 +70,17 @@ struct WindowFunnelState {
}
int get() const {
- std::vector<std::optional<VecDateTimeValue>> events_timestamp(max_event_level);
+ std::vector<std::optional<DateValueType>> events_timestamp(max_event_level);
for (int64_t i = 0; i < events.size(); i++) {
const int& event_idx = events[i].second;
- const VecDateTimeValue& timestamp = events[i].first;
+ const DateValueType& timestamp = events[i].first;
if (event_idx == 0) {
events_timestamp[0] = timestamp;
continue;
}
if (events_timestamp[event_idx - 1].has_value()) {
- const VecDateTimeValue& first_timestamp = events_timestamp[event_idx - 1].value();
- VecDateTimeValue last_timestamp = first_timestamp;
+ const DateValueType& first_timestamp = events_timestamp[event_idx - 1].value();
+ DateValueType last_timestamp = first_timestamp;
TimeInterval interval(SECOND, window, false);
last_timestamp.date_add_interval(interval, SECOND);
@@ -132,8 +133,7 @@ struct WindowFunnelState {
write_var_int(events.size(), out);
for (int64_t i = 0; i < events.size(); i++) {
- int64_t timestamp =
- binary_cast<vectorized::VecDateTimeValue, vectorized::Int64>(events[i].first);
+ int64_t timestamp = binary_cast<DateValueType, NativeType>(events[i].first);
int event_idx = events[i].second;
write_var_int(timestamp, out);
write_var_int(event_idx, out);
@@ -153,19 +153,23 @@ struct WindowFunnelState {
read_var_int(timestamp, in);
read_var_int(event_idx, in);
- VecDateTimeValue time_value =
- binary_cast<vectorized::Int64, vectorized::VecDateTimeValue>(timestamp);
+ DateValueType time_value = binary_cast<NativeType, DateValueType>(timestamp);
add(time_value, (int)event_idx, max_event_level, window);
}
}
};
+template <typename DateValueType, typename NativeType>
class AggregateFunctionWindowFunnel
- : public IAggregateFunctionDataHelper<WindowFunnelState, AggregateFunctionWindowFunnel> {
+ : public IAggregateFunctionDataHelper<
+ WindowFunnelState<DateValueType, NativeType>,
+ AggregateFunctionWindowFunnel<DateValueType, NativeType>> {
public:
AggregateFunctionWindowFunnel(const DataTypes& argument_types_)
- : IAggregateFunctionDataHelper<WindowFunnelState, AggregateFunctionWindowFunnel>(
- argument_types_, {}) {}
+ : IAggregateFunctionDataHelper<
+ WindowFunnelState<DateValueType, NativeType>,
+ AggregateFunctionWindowFunnel<DateValueType, NativeType>>(argument_types_,
+ {}) {}
String get_name() const override { return "window_funnel"; }
@@ -180,14 +184,15 @@ public:
// TODO: handle mode in the future.
// be/src/olap/row_block2.cpp copy_data_to_column
const auto& timestamp =
- static_cast<const ColumnVector<VecDateTimeValue>&>(*columns[2]).get_data()[row_num];
+ static_cast<const ColumnVector<DateValueType>&>(*columns[2]).get_data()[row_num];
const int NON_EVENT_NUM = 3;
- for (int i = NON_EVENT_NUM; i < get_argument_types().size(); i++) {
+ for (int i = NON_EVENT_NUM; i < IAggregateFunction::get_argument_types().size(); i++) {
const auto& is_set =
static_cast<const ColumnVector<UInt8>&>(*columns[i]).get_data()[row_num];
if (is_set) {
- this->data(place).add(timestamp, i - NON_EVENT_NUM,
- get_argument_types().size() - NON_EVENT_NUM, window);
+ this->data(place).add(
+ timestamp, i - NON_EVENT_NUM,
+ IAggregateFunction::get_argument_types().size() - NON_EVENT_NUM, window);
}
}
}
@@ -208,7 +213,11 @@ public:
void insert_result_into(ConstAggregateDataPtr __restrict place, IColumn& to) const override {
this->data(const_cast<AggregateDataPtr>(place)).sort();
- assert_cast<ColumnInt32&>(to).get_data().push_back(data(place).get());
+ assert_cast<ColumnInt32&>(to).get_data().push_back(
+ IAggregateFunctionDataHelper<
+ WindowFunnelState<DateValueType, NativeType>,
+ AggregateFunctionWindowFunnel<DateValueType, NativeType>>::data(place)
+ .get());
}
};
diff --git a/be/src/vec/columns/column.h b/be/src/vec/columns/column.h
index af40c96e07..35035d419f 100644
--- a/be/src/vec/columns/column.h
+++ b/be/src/vec/columns/column.h
@@ -475,14 +475,17 @@ public:
virtual void replace_column_data_default(size_t self_row = 0) = 0;
virtual bool is_date_type() const { return is_date; }
+ virtual bool is_date_v2_type() const { return is_date_v2; }
virtual bool is_datetime_type() const { return is_date_time; }
virtual void set_date_type() { is_date = true; }
+ virtual void set_date_v2_type() { is_date_v2 = true; }
virtual void set_datetime_type() { is_date_time = true; }
// todo(wb): a temporary implemention, need re-abstract here
bool is_date = false;
bool is_date_time = false;
+ bool is_date_v2 = false;
protected:
/// Template is to devirtualize calls to insert_from method.
diff --git a/be/src/vec/columns/column_nullable.h b/be/src/vec/columns/column_nullable.h
index e6f4495c30..31e1f2b80e 100644
--- a/be/src/vec/columns/column_nullable.h
+++ b/be/src/vec/columns/column_nullable.h
@@ -168,8 +168,10 @@ public:
}
bool is_date_type() const override { return get_nested_column().is_date_type(); }
+ bool is_date_v2_type() const override { return get_nested_column().is_date_v2_type(); }
bool is_datetime_type() const override { return get_nested_column().is_datetime_type(); }
void set_date_type() override { get_nested_column().set_date_type(); }
+ void set_date_v2_type() override { get_nested_column().set_date_v2_type(); }
void set_datetime_type() override { get_nested_column().set_datetime_type(); }
bool is_nullable() const override { return true; }
diff --git a/be/src/vec/columns/columns_number.h b/be/src/vec/columns/columns_number.h
index 500bab40aa..af878136d3 100644
--- a/be/src/vec/columns/columns_number.h
+++ b/be/src/vec/columns/columns_number.h
@@ -40,6 +40,7 @@ using ColumnInt64 = ColumnVector<Int64>;
using ColumnInt128 = ColumnVector<Int128>;
using ColumnDate = ColumnVector<Int64>;
+using ColumnDateV2 = ColumnVector<UInt32>;
using ColumnDateTime = ColumnVector<Int64>;
using ColumnFloat32 = ColumnVector<Float32>;
diff --git a/be/src/vec/columns/predicate_column.h b/be/src/vec/columns/predicate_column.h
index 27b1b7e5e0..0565ad0ff9 100644
--- a/be/src/vec/columns/predicate_column.h
+++ b/be/src/vec/columns/predicate_column.h
@@ -438,9 +438,19 @@ public:
} else if constexpr (std::is_same_v<T, uint24_t>) {
insert_date_to_res_column(sel, sel_size,
reinterpret_cast<vectorized::ColumnVector<Int64>*>(col_ptr));
- } else if constexpr (std::is_same_v<T, uint32_t>) { // a trick type judge, need refactor it.
- insert_date32_to_res_column(
- sel, sel_size, reinterpret_cast<vectorized::ColumnVector<Int64>*>(col_ptr));
+ } else if constexpr (std::is_same_v<T, uint32_t>) {
+ if (const vectorized::ColumnVector<Int64>* date_col =
+ check_and_get_column<vectorized::ColumnVector<Int64>>(
+ const_cast<const IColumn*>(col_ptr))) {
+ // a trick type judge, need refactor it.
+ insert_date32_to_res_column(sel, sel_size,
+ const_cast<vectorized::ColumnVector<Int64>*>(date_col));
+ } else {
+ insert_default_value_res_column(
+ sel, sel_size,
+ reinterpret_cast<vectorized::ColumnVector<doris::vectorized::UInt32>*>(
+ col_ptr));
+ }
} else if constexpr (std::is_same_v<T, doris::vectorized::Int128>) {
insert_default_value_res_column(
sel, sel_size,
diff --git a/be/src/vec/core/call_on_type_index.h b/be/src/vec/core/call_on_type_index.h
index 9b552ac381..d62288611d 100644
--- a/be/src/vec/core/call_on_type_index.h
+++ b/be/src/vec/core/call_on_type_index.h
@@ -159,6 +159,7 @@ inline bool call_on_basic_types(TypeIndex type_num1, TypeIndex type_num2, F&& f)
}
class DataTypeDate;
+class DataTypeDateV2;
class DataTypeDateTime;
class DataTypeString;
template <typename T>
@@ -205,6 +206,8 @@ bool call_on_index_and_data_type(TypeIndex number, F&& f) {
case TypeIndex::Date:
return f(TypePair<DataTypeDate, T>());
+ case TypeIndex::DateV2:
+ return f(TypePair<DataTypeDateV2, T>());
case TypeIndex::DateTime:
return f(TypePair<DataTypeDateTime, T>());
diff --git a/be/src/vec/core/types.h b/be/src/vec/core/types.h
index 650b64a627..a9c61bb31e 100644
--- a/be/src/vec/core/types.h
+++ b/be/src/vec/core/types.h
@@ -74,6 +74,9 @@ enum class TypeIndex {
LowCardinality,
BitMap,
HLL,
+ DateV2,
+ DateTimeV2,
+ TimeV2,
};
struct Consted {
@@ -255,6 +258,7 @@ struct TypeId<Int128> {
using Date = Int64;
using DateTime = Int64;
+using DateV2 = UInt32;
/// Own FieldType for Decimal.
/// It is only a "storage" for decimal. To perform operations, you also have to provide a scale (number of digits after point).
@@ -401,6 +405,12 @@ inline const char* getTypeName(TypeIndex idx) {
return "Date";
case TypeIndex::DateTime:
return "DateTime";
+ case TypeIndex::DateV2:
+ return "DateV2";
+ case TypeIndex::DateTimeV2:
+ return "DateTimeV2";
+ case TypeIndex::TimeV2:
+ return "TimeV2";
case TypeIndex::String:
return TypeName<String>::get();
case TypeIndex::FixedString:
diff --git a/be/src/vec/data_types/data_type.cpp b/be/src/vec/data_types/data_type.cpp
index 414959a517..4be9a6112a 100644
--- a/be/src/vec/data_types/data_type.cpp
+++ b/be/src/vec/data_types/data_type.cpp
@@ -131,6 +131,8 @@ PGenericType_TypeId IDataType::get_pdata_type(const IDataType* data_type) {
return PGenericType::STRING;
case TypeIndex::Date:
return PGenericType::DATE;
+ case TypeIndex::DateV2:
+ return PGenericType::DATEV2;
case TypeIndex::DateTime:
return PGenericType::DATETIME;
case TypeIndex::BitMap:
diff --git a/be/src/vec/data_types/data_type.h b/be/src/vec/data_types/data_type.h
index 6a4dc16899..047c9fa0b6 100644
--- a/be/src/vec/data_types/data_type.h
+++ b/be/src/vec/data_types/data_type.h
@@ -295,7 +295,10 @@ struct WhichDataType {
bool is_date() const { return idx == TypeIndex::Date; }
bool is_date_time() const { return idx == TypeIndex::DateTime; }
+ bool is_date_v2() const { return idx == TypeIndex::DateV2; }
+ bool is_date_time_v2() const { return idx == TypeIndex::DateTimeV2; }
bool is_date_or_datetime() const { return is_date() || is_date_time(); }
+ bool is_date_v2_or_datetime_v2() const { return is_date_v2() || is_date_time_v2(); }
bool is_string() const { return idx == TypeIndex::String; }
bool is_fixed_string() const { return idx == TypeIndex::FixedString; }
@@ -318,9 +321,15 @@ struct WhichDataType {
inline bool is_date(const DataTypePtr& data_type) {
return WhichDataType(data_type).is_date();
}
+inline bool is_date_v2(const DataTypePtr& data_type) {
+ return WhichDataType(data_type).is_date_v2();
+}
inline bool is_date_or_datetime(const DataTypePtr& data_type) {
return WhichDataType(data_type).is_date_or_datetime();
}
+inline bool is_date_v2_or_datetime_v2(const DataTypePtr& data_type) {
+ return WhichDataType(data_type).is_date_v2_or_datetime_v2();
+}
inline bool is_enum(const DataTypePtr& data_type) {
return WhichDataType(data_type).is_enum();
}
@@ -376,7 +385,7 @@ template <typename T>
inline bool is_columned_as_number(const T& data_type) {
WhichDataType which(data_type);
return which.is_int() || which.is_uint() || which.is_float() || which.is_date_or_datetime() ||
- which.is_uuid();
+ which.is_uuid() || which.is_date_v2();
}
template <typename T>
diff --git a/be/src/vec/data_types/data_type_date_time.h b/be/src/vec/data_types/data_type_date_time.h
index 8085407253..ebee94ac70 100644
--- a/be/src/vec/data_types/data_type_date_time.h
+++ b/be/src/vec/data_types/data_type_date_time.h
@@ -22,6 +22,7 @@
#include "vec/data_types/data_type_date.h"
#include "vec/data_types/data_type_number_base.h"
+#include "vec/data_types/data_type_time_v2.h"
class DateLUTImpl;
@@ -81,6 +82,14 @@ constexpr bool IsDateType = false;
template <>
inline constexpr bool IsDateType<DataTypeDate> = true;
+template <typename DataType>
+constexpr bool IsDateV2Type = false;
+template <>
+inline constexpr bool IsDateV2Type<DataTypeDateV2> = true;
+
+template <typename DataType>
+constexpr bool IsDateTimeV2Type = false;
+
template <typename DataType>
constexpr bool IsTimeType = IsDateTimeType<DataType> || IsDateType<DataType>;
diff --git a/be/src/vec/data_types/data_type_factory.cpp b/be/src/vec/data_types/data_type_factory.cpp
index 43897e1f44..8ab9d6a933 100644
--- a/be/src/vec/data_types/data_type_factory.cpp
+++ b/be/src/vec/data_types/data_type_factory.cpp
@@ -81,6 +81,10 @@ DataTypePtr DataTypeFactory::create_data_type(const TypeDescriptor& col_desc, bo
case TYPE_DATE:
nested = std::make_shared<vectorized::DataTypeDate>();
break;
+ case TYPE_DATEV2:
+ nested = std::make_shared<vectorized::DataTypeDateV2>();
+ break;
+ case TYPE_DATETIMEV2:
case TYPE_DATETIME:
nested = std::make_shared<vectorized::DataTypeDateTime>();
break;
@@ -150,6 +154,9 @@ DataTypePtr DataTypeFactory::_create_primitive_data_type(const FieldType& type)
case OLAP_FIELD_TYPE_DATE:
result = std::make_shared<vectorized::DataTypeDate>();
break;
+ case OLAP_FIELD_TYPE_DATEV2:
+ result = std::make_shared<vectorized::DataTypeDateV2>();
+ break;
case OLAP_FIELD_TYPE_DATETIME:
result = std::make_shared<vectorized::DataTypeDateTime>();
break;
@@ -223,6 +230,9 @@ DataTypePtr DataTypeFactory::create_data_type(const PColumnMeta& pcolumn) {
case PGenericType::DATE:
nested = std::make_shared<DataTypeDate>();
break;
+ case PGenericType::DATEV2:
+ nested = std::make_shared<DataTypeDateV2>();
+ break;
case PGenericType::DATETIME:
nested = std::make_shared<DataTypeDateTime>();
break;
diff --git a/be/src/vec/data_types/data_type_factory.hpp b/be/src/vec/data_types/data_type_factory.hpp
index e64bdcf3ac..08dc6a9f31 100644
--- a/be/src/vec/data_types/data_type_factory.hpp
+++ b/be/src/vec/data_types/data_type_factory.hpp
@@ -62,6 +62,7 @@ public:
{"Float32", std::make_shared<DataTypeFloat32>()},
{"Float64", std::make_shared<DataTypeFloat64>()},
{"Date", std::make_shared<DataTypeDate>()},
+ {"DateV2", std::make_shared<DataTypeDateV2>()},
{"DateTime", std::make_shared<DataTypeDateTime>()},
{"String", std::make_shared<DataTypeString>()},
{"Decimal", std::make_shared<DataTypeDecimal<Decimal128>>(27, 9)},
diff --git a/be/src/vec/data_types/data_type_time_v2.cpp b/be/src/vec/data_types/data_type_time_v2.cpp
new file mode 100644
index 0000000000..ce8a7cd3eb
--- /dev/null
+++ b/be/src/vec/data_types/data_type_time_v2.cpp
@@ -0,0 +1,73 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "vec/data_types/data_type_time_v2.h"
+
+#include "util/binary_cast.hpp"
+#include "vec/columns/columns_number.h"
+#include "vec/runtime/vdatetime_value.h"
+
+namespace doris::vectorized {
+bool DataTypeDateV2::equals(const IDataType& rhs) const {
+ return typeid(rhs) == typeid(*this);
+}
+
+std::string DataTypeDateV2::to_string(const IColumn& column, size_t row_num) const {
+ UInt32 int_val =
+ assert_cast<const ColumnUInt32&>(*column.convert_to_full_column_if_const().get())
+ .get_data()[row_num];
+ std::stringstream ss;
+ // Year
+ ss << (int_val & 0xFFFF0000) << '-';
+ // Month
+ ss << (int_val & 0x0000FF00) << '-';
+ // Day
+ ss << int_val % 0xFF;
+ return ss.str();
+}
+
+void DataTypeDateV2::to_string(const IColumn& column, size_t row_num, BufferWritable& ostr) const {
+ UInt32 int_val =
+ assert_cast<const ColumnUInt32&>(*column.convert_to_full_column_if_const().get())
+ .get_data()[row_num];
+ DateV2Value value = binary_cast<UInt32, DateV2Value>(int_val);
+
+ char buf[64];
+ char* pos = value.to_string(buf);
+ // DateTime to_string the end is /0
+ ostr.write(buf, pos - buf - 1);
+}
+
+MutableColumnPtr DataTypeDateV2::create_column() const {
+ auto col = DataTypeNumberBase<UInt32>::create_column();
+ col->set_date_v2_type();
+ return col;
+}
+
+void DataTypeDateV2::cast_to_date_time(const UInt32& from, Int64& to) {
+ auto& to_value = (doris::vectorized::VecDateTimeValue&)to;
+ auto& from_value = (doris::vectorized::DateV2Value&)from;
+ to_value.create_from_date_v2(from_value, TimeType::TIME_DATETIME);
+}
+
+void DataTypeDateV2::cast_to_date(const UInt32& from, Int64& to) {
+ auto& to_value = (doris::vectorized::VecDateTimeValue&)(to);
+ auto& from_value = (doris::vectorized::DateV2Value&)from;
+ to_value.create_from_date_v2(from_value, TimeType::TIME_DATE);
+}
+
+} // namespace doris::vectorized
diff --git a/be/src/vec/data_types/data_type_time_v2.h b/be/src/vec/data_types/data_type_time_v2.h
new file mode 100644
index 0000000000..c126b0966b
--- /dev/null
+++ b/be/src/vec/data_types/data_type_time_v2.h
@@ -0,0 +1,47 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include "vec/data_types/data_type_number_base.h"
+
+namespace doris::vectorized {
+
+/**
+ * Use UInt32 as underlying type to represent DateV2 type.
+ * Specifically, a dateV2 type is represented as (YYYY (16 bits), MM (8 bits), DD (8 bits)).
+ */
+class DataTypeDateV2 final : public DataTypeNumberBase<UInt32> {
+public:
+ TypeIndex get_type_id() const override { return TypeIndex::DateV2; }
+ const char* get_family_name() const override { return "DateV2"; }
+ std::string do_get_name() const override { return "DateV2"; }
+
+ bool can_be_used_as_version() const override { return true; }
+ bool can_be_inside_nullable() const override { return true; }
+
+ bool equals(const IDataType& rhs) const override;
+ std::string to_string(const IColumn& column, size_t row_num) const override;
+ void to_string(const IColumn& column, size_t row_num, BufferWritable& ostr) const override;
+
+ MutableColumnPtr create_column() const override;
+
+ static void cast_to_date(const UInt32& from, Int64& to);
+ static void cast_to_date_time(const UInt32& from, Int64& to);
+};
+
+} // namespace doris::vectorized
diff --git a/be/src/vec/data_types/get_least_supertype.cpp b/be/src/vec/data_types/get_least_supertype.cpp
index a661035af9..1aa9da5c55 100644
--- a/be/src/vec/data_types/get_least_supertype.cpp
+++ b/be/src/vec/data_types/get_least_supertype.cpp
@@ -146,6 +146,19 @@ DataTypePtr get_least_supertype(const DataTypes& types) {
}
}
+ {
+ UInt32 have_date_v2 = type_ids.count(TypeIndex::DateV2);
+
+ if (have_date_v2) {
+ if (type_ids.size() != have_date_v2) {
+ LOG(FATAL) << get_exception_message_prefix(types)
+ << " because some of them are DateV2 and some of them are not";
+ }
+
+ return std::make_shared<DataTypeDateV2>();
+ }
+ }
+
/// Decimals
{
UInt32 have_decimal32 = type_ids.count(TypeIndex::Decimal32);
diff --git a/be/src/vec/exec/join/vhash_join_node.cpp b/be/src/vec/exec/join/vhash_join_node.cpp
index 4471034ea4..fa1400a626 100644
--- a/be/src/vec/exec/join/vhash_join_node.cpp
+++ b/be/src/vec/exec/join/vhash_join_node.cpp
@@ -1204,6 +1204,7 @@ void HashJoinNode::_hash_table_init() {
break;
case TYPE_INT:
case TYPE_FLOAT:
+ case TYPE_DATEV2:
_hash_table_variants.emplace<I32HashTableContext>();
break;
case TYPE_BIGINT:
diff --git a/be/src/vec/exec/vaggregation_node.cpp b/be/src/vec/exec/vaggregation_node.cpp
index 8910674467..bc699b855c 100644
--- a/be/src/vec/exec/vaggregation_node.cpp
+++ b/be/src/vec/exec/vaggregation_node.cpp
@@ -135,6 +135,7 @@ void AggregationNode::_init_hash_method(std::vector<VExprContext*>& probe_exprs)
return;
case TYPE_INT:
case TYPE_FLOAT:
+ case TYPE_DATEV2:
_agg_data.init(AggregatedDataVariants::Type::int32_key, is_nullable);
return;
case TYPE_BIGINT:
diff --git a/be/src/vec/exec/varrow_scanner.cpp b/be/src/vec/exec/varrow_scanner.cpp
index 14ae373988..237c65fb62 100644
--- a/be/src/vec/exec/varrow_scanner.cpp
+++ b/be/src/vec/exec/varrow_scanner.cpp
@@ -254,9 +254,9 @@ Status VArrowScanner::_append_batch_to_src_block(Block* block) {
}
auto* array = _batch->column(column_pos++).get();
auto& column_with_type_and_name = block->get_by_name(slot_desc->col_name());
- RETURN_IF_ERROR(arrow_column_to_doris_column(array, _arrow_batch_cur_idx,
- column_with_type_and_name.column, num_elements,
- _state->timezone()));
+ RETURN_IF_ERROR(arrow_column_to_doris_column(
+ array, _arrow_batch_cur_idx, column_with_type_and_name.column,
+ column_with_type_and_name.type, num_elements, _state->timezone()));
}
_arrow_batch_cur_idx += num_elements;
diff --git a/be/src/vec/exec/volap_scan_node.cpp b/be/src/vec/exec/volap_scan_node.cpp
index aef88e4e0f..cd7e39492b 100644
--- a/be/src/vec/exec/volap_scan_node.cpp
+++ b/be/src/vec/exec/volap_scan_node.cpp
@@ -676,6 +676,13 @@ Status VOlapScanNode::normalize_conjuncts() {
break;
}
+ case TYPE_DATEV2: {
+ ColumnValueRange<doris::vectorized::DateV2Value> range(slots[slot_idx]->col_name(),
+ slots[slot_idx]->type().type);
+ normalize_predicate(range, slots[slot_idx]);
+ break;
+ }
+
case TYPE_DECIMALV2: {
ColumnValueRange<DecimalV2Value> range(slots[slot_idx]->col_name(),
slots[slot_idx]->type().type);
@@ -829,7 +836,8 @@ Status VOlapScanNode::normalize_predicate(ColumnValueRange<T>& range, SlotDescri
}
static bool ignore_cast(SlotDescriptor* slot, Expr* expr) {
- if (slot->type().is_date_type() && expr->type().is_date_type()) {
+ if ((slot->type().is_date_type() || slot->type().is_date_v2_type()) &&
+ (expr->type().is_date_type() || expr->type().is_date_v2_type())) {
return true;
}
if (slot->type().is_string_type() && expr->type().is_string_type()) {
@@ -952,6 +960,19 @@ Status VOlapScanNode::change_fixed_value_range(ColumnValueRange<T>& temp_range,
func(temp_range, reinterpret_cast<T*>(&v));
break;
}
+ case TYPE_DATEV2: {
+ DateTimeValue date_value = *reinterpret_cast<DateTimeValue*>(value);
+ if (!date_value.check_loss_accuracy_cast_to_date()) {
+ doris::vectorized::DateV2Value date_v2;
+ date_v2.convert_dt_to_date_v2(&date_value);
+ if constexpr (std::is_same_v<T, doris::vectorized::DateV2Value>) {
+ func(temp_range, &date_v2);
+ } else {
+ __builtin_unreachable();
+ }
+ }
+ break;
+ }
default: {
LOG(WARNING) << "Normalize filter fail, Unsupported Primitive type. [type=" << type << "]";
return Status::InternalError("Normalize filter fail, Unsupported Primitive type");
@@ -1296,6 +1317,22 @@ Status VOlapScanNode::normalize_noneq_binary_predicate(SlotDescriptor* slot,
*reinterpret_cast<T*>(&date_value));
break;
}
+ case TYPE_DATEV2: {
+ DateTimeValue date_value = *reinterpret_cast<DateTimeValue*>(value);
+ if (date_value.check_loss_accuracy_cast_to_date()) {
+ if (pred->op() == TExprOpcode::LT || pred->op() == TExprOpcode::GE) {
+ ++date_value;
+ }
+ }
+ doris::vectorized::DateV2Value date_v2;
+ date_v2.convert_dt_to_date_v2(&date_value);
+ if constexpr (std::is_same_v<T, doris::vectorized::DateV2Value>) {
+ range->add_range(to_olap_filter_type(pred->op(), child_idx), date_v2);
+ break;
+ } else {
+ __builtin_unreachable();
+ }
+ }
case TYPE_TINYINT:
case TYPE_DECIMALV2:
case TYPE_CHAR:
diff --git a/be/src/vec/exec/volap_scan_node.h b/be/src/vec/exec/volap_scan_node.h
index 23ce669cf1..2222390652 100644
--- a/be/src/vec/exec/volap_scan_node.h
+++ b/be/src/vec/exec/volap_scan_node.h
@@ -17,10 +17,13 @@
#pragma once
-#include "exec/olap_scan_node.h"
+#include "exec/olap_common.h"
+#include "exec/scan_node.h"
+#include "exprs/bloomfilter_predicate.h"
#include "exprs/in_predicate.h"
#include "exprs/runtime_filter.h"
#include "gen_cpp/PlanNodes_types.h"
+#include "olap/tablet.h"
#include "util/progress_updater.h"
namespace doris {
diff --git a/be/src/vec/exec/vschema_scan_node.cpp b/be/src/vec/exec/vschema_scan_node.cpp
index bacf6b7d56..07e8d638f5 100644
--- a/be/src/vec/exec/vschema_scan_node.cpp
+++ b/be/src/vec/exec/vschema_scan_node.cpp
@@ -399,6 +399,12 @@ Status VSchemaScanNode::write_slot_to_vectorized_column(void* slot, SlotDescript
break;
}
+ case TYPE_DATEV2: {
+ uint32_t num = *reinterpret_cast<uint32_t*>(slot);
+ reinterpret_cast<vectorized::ColumnVector<vectorized::UInt32>*>(col_ptr)->insert_value(num);
+ break;
+ }
+
case TYPE_DATETIME: {
VecDateTimeValue value;
DateTimeValue* ts_slot = reinterpret_cast<DateTimeValue*>(slot);
diff --git a/be/src/vec/exec/vset_operation_node.cpp b/be/src/vec/exec/vset_operation_node.cpp
index 8d27120ed1..e3a2659b73 100644
--- a/be/src/vec/exec/vset_operation_node.cpp
+++ b/be/src/vec/exec/vset_operation_node.cpp
@@ -160,6 +160,7 @@ void VSetOperationNode::hash_table_init() {
break;
case TYPE_INT:
case TYPE_FLOAT:
+ case TYPE_DATEV2:
_hash_table_variants.emplace<I32HashTableContext>();
break;
case TYPE_BIGINT:
diff --git a/be/src/vec/exprs/vliteral.cpp b/be/src/vec/exprs/vliteral.cpp
index 73f7f231fd..673c8237b1 100644
--- a/be/src/vec/exprs/vliteral.cpp
+++ b/be/src/vec/exprs/vliteral.cpp
@@ -94,6 +94,12 @@ void VLiteral::init(const TExprNode& node) {
field = Int64(*reinterpret_cast<__int64_t*>(&value));
break;
}
+ case TYPE_DATEV2: {
+ DateV2Value value;
+ value.from_date_str(node.date_literal.value.c_str(), node.date_literal.value.size());
+ field = value.to_date_uint32();
+ break;
+ }
case TYPE_DATETIME: {
VecDateTimeValue value;
value.from_date_str(node.date_literal.value.c_str(), node.date_literal.value.size());
diff --git a/be/src/vec/functions/array/function_array_element.h b/be/src/vec/functions/array/function_array_element.h
index 315ce5cb2a..4f6eb12a07 100644
--- a/be/src/vec/functions/array/function_array_element.h
+++ b/be/src/vec/functions/array/function_array_element.h
@@ -200,6 +200,10 @@ private:
if (nested_column->is_date_type()) {
res = _execute_number<ColumnDate>(offsets, *nested_column, src_null_map,
*arguments[1].column, nested_null_map, dst_null_map);
+ } else if (nested_column->is_date_v2_type()) {
+ res = _execute_number<ColumnDateV2>(offsets, *nested_column, src_null_map,
+ *arguments[1].column, nested_null_map,
+ dst_null_map);
} else if (nested_column->is_datetime_type()) {
res = _execute_number<ColumnDateTime>(offsets, *nested_column, src_null_map,
*arguments[1].column, nested_null_map,
diff --git a/be/src/vec/functions/array/function_array_index.h b/be/src/vec/functions/array/function_array_index.h
index 046846424f..241bbda6b2 100644
--- a/be/src/vec/functions/array/function_array_index.h
+++ b/be/src/vec/functions/array/function_array_index.h
@@ -174,6 +174,9 @@ private:
} else if (right_column.is_date_type()) {
return _execute_number<NestedColumnType, ColumnDate>(offsets, nested_null_map,
nested_column, right_column);
+ } else if (right_column.is_date_v2_type()) {
+ return _execute_number<NestedColumnType, ColumnDateV2>(offsets, nested_null_map,
+ nested_column, right_column);
} else if (right_column.is_datetime_type()) {
return _execute_number<NestedColumnType, ColumnDateTime>(offsets, nested_null_map,
nested_column, right_column);
@@ -245,10 +248,15 @@ private:
return_column = _execute_number_expanded<ColumnDecimal128>(
offsets, nested_null_map, *nested_column, *right_column);
}
- } else if (is_date_or_datetime(right_type) && is_date_or_datetime(left_element_type)) {
+ } else if ((is_date_or_datetime(right_type) || is_date_v2_or_datetime_v2(right_type)) &&
+ (is_date_or_datetime(left_element_type) ||
+ is_date_v2_or_datetime_v2(left_element_type))) {
if (nested_column->is_date_type()) {
return_column = _execute_number_expanded<ColumnDate>(offsets, nested_null_map,
*nested_column, *right_column);
+ } else if (nested_column->is_date_v2_type()) {
+ return_column = _execute_number_expanded<ColumnDateV2>(
+ offsets, nested_null_map, *nested_column, *right_column);
} else if (nested_column->is_datetime_type()) {
return_column = _execute_number_expanded<ColumnDateTime>(
offsets, nested_null_map, *nested_column, *right_column);
diff --git a/be/src/vec/functions/date_time_transforms.h b/be/src/vec/functions/date_time_transforms.h
index 7381a96885..d42cd2d336 100644
--- a/be/src/vec/functions/date_time_transforms.h
+++ b/be/src/vec/functions/date_time_transforms.h
@@ -33,14 +33,25 @@
namespace doris::vectorized {
-#define TIME_FUNCTION_IMPL(CLASS, UNIT, FUNCTION) \
- struct CLASS { \
- static constexpr auto name = #UNIT; \
- static inline auto execute(const Int64& t, bool& is_null) { \
- const auto& date_time_value = (doris::vectorized::VecDateTimeValue&)(t); \
- is_null = !date_time_value.is_valid_date(); \
- return date_time_value.FUNCTION; \
- } \
+#define TIME_FUNCTION_IMPL(CLASS, UNIT, FUNCTION) \
+ template <typename DateValueType, typename ArgType> \
+ struct CLASS { \
+ using ARG_TYPE = ArgType; \
+ static constexpr auto name = #UNIT; \
+ \
+ static inline auto execute(const ARG_TYPE& t, bool& is_null) { \
+ const auto& date_time_value = (DateValueType&)(t); \
+ is_null = !date_time_value.is_valid_date(); \
+ return date_time_value.FUNCTION; \
+ } \
+ \
+ static DataTypes get_variadic_argument_types() { \
+ if constexpr (std::is_same_v<DateValueType, VecDateTimeValue>) { \
+ return {std::make_shared<DataTypeDateTime>()}; \
+ } else { \
+ return {std::make_shared<DataTypeDateV2>()}; \
+ } \
+ } \
}
#define TO_TIME_FUNCTION(CLASS, UNIT) TIME_FUNCTION_IMPL(CLASS, UNIT, UNIT())
@@ -61,49 +72,76 @@ TIME_FUNCTION_IMPL(WeekDayImpl, weekday, weekday());
// TODO: the method should be always not nullable
TIME_FUNCTION_IMPL(ToDaysImpl, to_days, daynr());
-#define TIME_FUNCTION_ONE_ARG_IMPL(CLASS, UNIT, FUNCTION) \
- struct CLASS { \
- static constexpr auto name = #UNIT; \
- static inline auto execute(const Int64& t, bool& is_null) { \
- const auto& date_time_value = (doris::vectorized::VecDateTimeValue&)(t); \
- is_null = !date_time_value.is_valid_date(); \
- return date_time_value.FUNCTION; \
- } \
- \
- static DataTypes get_variadic_argument_types() { \
- return {std::make_shared<DataTypeDateTime>()}; \
- } \
+#define TIME_FUNCTION_ONE_ARG_IMPL(CLASS, UNIT, FUNCTION) \
+ template <typename DateValueType, typename ArgType> \
+ struct CLASS { \
+ using ARG_TYPE = ArgType; \
+ static constexpr auto name = #UNIT; \
+ \
+ static inline auto execute(const ARG_TYPE& t, bool& is_null) { \
+ const auto& date_time_value = (DateValueType&)(t); \
+ is_null = !date_time_value.is_valid_date(); \
+ return date_time_value.FUNCTION; \
+ } \
+ \
+ static DataTypes get_variadic_argument_types() { \
+ if constexpr (std::is_same_v<DateValueType, VecDateTimeValue>) { \
+ return {std::make_shared<DataTypeDateTime>()}; \
+ } else { \
+ return {std::make_shared<DataTypeDateV2>()}; \
+ } \
+ } \
}
TIME_FUNCTION_ONE_ARG_IMPL(ToWeekOneArgImpl, week, week(mysql_week_mode(0)));
TIME_FUNCTION_ONE_ARG_IMPL(ToYearWeekOneArgImpl, yearweek, year_week(mysql_week_mode(0)));
+template <typename DateValueType, typename ArgType>
struct ToDateImpl {
+ using ARG_TYPE = ArgType;
static constexpr auto name = "to_date";
- static inline auto execute(const Int64& t, bool& is_null) {
- auto dt = binary_cast<Int64, doris::vectorized::VecDateTimeValue>(t);
+ static inline auto execute(const ArgType& t, bool& is_null) {
+ auto dt = binary_cast<ArgType, DateValueType>(t);
is_null = !dt.is_valid_date();
- dt.cast_to_date();
- return binary_cast<doris::vectorized::VecDateTimeValue, Int64>(dt);
+ if constexpr (!std::is_same_v<DateValueType, doris::vectorized::DateV2Value>) {
+ dt.cast_to_date();
+ }
+ return binary_cast<DateValueType, ArgType>(dt);
+ }
+
+ static DataTypes get_variadic_argument_types() {
+ if constexpr (std::is_same_v<DateValueType, VecDateTimeValue>) {
+ return {std::make_shared<DataTypeDateTime>()};
+ } else {
+ return {std::make_shared<DataTypeDateV2>()};
+ }
}
};
-struct DateImpl : public ToDateImpl {
+
+template <typename DateValue, typename ArgType>
+struct DateImpl : public ToDateImpl<DateValue, ArgType> {
static constexpr auto name = "date";
};
// TODO: This function look like no need do indeed copy here, we should optimize
// this function
+template <typename ArgType>
struct TimeStampImpl {
+ using ARG_TYPE = ArgType;
static constexpr auto name = "timestamp";
- static inline auto execute(const Int64& t, bool& is_null) { return t; }
+
+ static inline auto execute(const ARG_TYPE& t, bool& is_null) { return t; }
};
+template <typename DateValueType, typename ArgType>
struct DayNameImpl {
+ using ARG_TYPE = ArgType;
+ using DATE_TYPE = DateValueType;
static constexpr auto name = "dayname";
static constexpr auto max_size = MAX_DAY_NAME_LEN;
- static inline auto execute(const VecDateTimeValue& dt, ColumnString::Chars& res_data,
+ static inline auto execute(const DateValueType& dt, ColumnString::Chars& res_data,
size_t& offset, bool& is_null) {
const auto* day_name = dt.day_name();
is_null = !dt.is_valid_date();
@@ -118,14 +156,25 @@ struct DayNameImpl {
}
return offset;
}
+
+ static DataTypes get_variadic_argument_types() {
+ if constexpr (std::is_same_v<DATE_TYPE, VecDateTimeValue>) {
+ return {std::make_shared<DataTypeDateTime>()};
+ } else {
+ return {std::make_shared<DataTypeDateV2>()};
+ }
+ }
};
+template <typename DateValueType, typename ArgType>
struct MonthNameImpl {
+ using ARG_TYPE = ArgType;
+ using DATE_TYPE = DateValueType;
static constexpr auto name = "monthname";
static constexpr auto max_size = MAX_MONTH_NAME_LEN;
- static inline auto execute(const VecDateTimeValue& dt, ColumnString::Chars& res_data,
- size_t& offset, bool& is_null) {
+ static inline auto execute(const DATE_TYPE& dt, ColumnString::Chars& res_data, size_t& offset,
+ bool& is_null) {
const auto* month_name = dt.month_name();
is_null = !dt.is_valid_date();
if (month_name == nullptr || is_null) {
@@ -139,16 +188,25 @@ struct MonthNameImpl {
}
return offset;
}
+
+ static DataTypes get_variadic_argument_types() {
+ if constexpr (std::is_same_v<DATE_TYPE, VecDateTimeValue>) {
+ return {std::make_shared<DataTypeDateTime>()};
+ } else {
+ return {std::make_shared<DataTypeDateV2>()};
+ }
+ }
};
+template <typename DateType, typename ArgType>
struct DateFormatImpl {
- using FromType = Int64;
+ using FromType = ArgType;
static constexpr auto name = "date_format";
- static inline auto execute(const Int64& t, StringRef format, ColumnString::Chars& res_data,
+ static inline auto execute(const FromType& t, StringRef format, ColumnString::Chars& res_data,
size_t& offset) {
- const auto& dt = (VecDateTimeValue&)t;
+ const auto& dt = (DateType&)t;
if (format.size > 128) {
offset += 1;
res_data.emplace_back(0);
@@ -166,9 +224,20 @@ struct DateFormatImpl {
offset += len;
return std::pair {offset, false};
}
+
+ static DataTypes get_variadic_argument_types() {
+ if constexpr (std::is_same_v<DateType, VecDateTimeValue>) {
+ return std::vector<DataTypePtr> {std::make_shared<vectorized::DataTypeDateTime>(),
+ std::make_shared<vectorized::DataTypeString>()};
+ } else {
+ return std::vector<DataTypePtr> {std::make_shared<vectorized::DataTypeDateV2>(),
+ std::make_shared<vectorized::DataTypeString>()};
+ }
+ }
};
// TODO: This function should be depend on argments not always nullable
+template <typename DateType>
struct FromUnixTimeImpl {
using FromType = Int32;
@@ -179,7 +248,7 @@ struct FromUnixTimeImpl {
// TODO: use default time zone, slowly and incorrect, just for test use
static cctz::time_zone time_zone = cctz::fixed_time_zone(cctz::seconds(8 * 60 * 60));
- VecDateTimeValue dt;
+ DateType dt;
if (format.size > 128 || val < 0 || val > INT_MAX || !dt.from_unixtime(val, time_zone)) {
offset += 1;
res_data.emplace_back(0);
@@ -217,6 +286,22 @@ struct TransformerToStringOneArgument {
reinterpret_cast<bool&>(null_map[i]));
}
}
+
+ static void vector(const PaddedPODArray<UInt32>& ts, ColumnString::Chars& res_data,
+ ColumnString::Offsets& res_offsets, NullMap& null_map) {
+ const auto len = ts.size();
+ res_data.resize(len * Transform::max_size);
+ res_offsets.resize(len);
+ null_map.resize_fill(len, false);
+
+ size_t offset = 0;
+ for (int i = 0; i < len; ++i) {
+ const auto& t = ts[i];
+ const auto& date_time_value = reinterpret_cast<const DateV2Value&>(t);
+ res_offsets[i] = Transform::execute(date_time_value, res_data, offset,
+ reinterpret_cast<bool&>(null_map[i]));
+ }
+ }
};
template <typename Transform>
diff --git a/be/src/vec/functions/function.h b/be/src/vec/functions/function.h
index c4464e4956..2272430a8e 100644
--- a/be/src/vec/functions/function.h
+++ b/be/src/vec/functions/function.h
@@ -292,7 +292,14 @@ public:
is_date_or_datetime(get_return_type(arguments)->is_nullable()
? ((DataTypeNullable*)get_return_type(arguments).get())
->get_nested_type()
- : get_return_type(arguments))))
+ : get_return_type(arguments))) ||
+ (is_date_v2(return_type->is_nullable()
+ ? ((DataTypeNullable*)return_type.get())->get_nested_type()
+ : return_type) &&
+ is_date_v2(get_return_type(arguments)->is_nullable()
+ ? ((DataTypeNullable*)get_return_type(arguments).get())
+ ->get_nested_type()
+ : get_return_type(arguments))))
<< " with " << return_type->get_name() << " and " << func_return_type->get_name();
return build_impl(arguments, return_type);
diff --git a/be/src/vec/functions/function_cast.h b/be/src/vec/functions/function_cast.h
index 5799e60d93..490d414418 100644
--- a/be/src/vec/functions/function_cast.h
+++ b/be/src/vec/functions/function_cast.h
@@ -83,7 +83,8 @@ struct ConvertImpl {
ColumnDecimal<ToFieldType>, ColumnVector<ToFieldType>>;
if constexpr (IsDataTypeDecimal<FromDataType> || IsDataTypeDecimal<ToDataType>) {
- if constexpr (!(IsDataTypeDecimalOrNumber<FromDataType> || IsTimeType<FromDataType>) ||
+ if constexpr (!(IsDataTypeDecimalOrNumber<FromDataType> || IsTimeType<FromDataType> ||
+ IsDateV2Type<FromDataType>) ||
!IsDataTypeDecimalOrNumber<ToDataType>)
return Status::RuntimeError(
fmt::format("Illegal column {} of first argument of function {}",
@@ -121,21 +122,41 @@ struct ConvertImpl {
vec_to[i] = convert_to_decimal<DataTypeInt64, ToDataType>(
reinterpret_cast<const VecDateTimeValue&>(vec_from[i]).to_int64(),
vec_to.get_scale());
+ } else if constexpr (IsDateV2Type<FromDataType> &&
+ IsDataTypeDecimal<ToDataType>) {
+ vec_to[i] = convert_to_decimal<DataTypeUInt32, ToDataType>(
+ reinterpret_cast<const DateV2Value&>(vec_from[i]).to_date_uint32(),
+ vec_to.get_scale());
}
} else if constexpr (IsTimeType<FromDataType>) {
if constexpr (IsTimeType<ToDataType>) {
vec_to[i] = static_cast<ToFieldType>(vec_from[i]);
- if constexpr (IsDateType<FromDataType> && IsDateTimeType<ToDataType>) {
+ if constexpr (IsDateTimeType<ToDataType>) {
DataTypeDateTime::cast_to_date_time(vec_to[i]);
} else {
DataTypeDate::cast_to_date(vec_to[i]);
}
+ } else if constexpr (IsDateV2Type<ToDataType>) {
+ auto date_v2 = binary_cast<UInt32, DateV2Value>(vec_to[i]);
+ date_v2.from_date_int64(vec_from[i]);
} else {
vec_to[i] =
reinterpret_cast<const VecDateTimeValue&>(vec_from[i]).to_int64();
}
- } else
+ } else if constexpr (IsDateV2Type<FromDataType>) {
+ if constexpr (IsTimeType<ToDataType> || IsDateTimeV2Type<ToDataType>) {
+ if constexpr (IsDateTimeType<ToDataType> || IsDateTimeV2Type<ToDataType>) {
+ DataTypeDateV2::cast_to_date_time(vec_from[i], vec_to[i]);
+ } else {
+ DataTypeDateV2::cast_to_date(vec_from[i], vec_to[i]);
+ }
+ } else {
+ vec_to[i] =
+ reinterpret_cast<const DateV2Value&>(vec_from[i]).to_date_uint32();
+ }
+ } else {
vec_to[i] = static_cast<ToFieldType>(vec_from[i]);
+ }
}
// TODO: support boolean cast more reasonable
@@ -180,7 +201,10 @@ struct ConvertImplToTimeType {
using ColVecFrom =
std::conditional_t<IsDecimalNumber<FromFieldType>, ColumnDecimal<FromFieldType>,
ColumnVector<FromFieldType>>;
- using ColVecTo = ColumnVector<Int64>;
+
+ using DateValueType =
+ std::conditional_t<IsDateV2Type<ToDataType>, DateV2Value, VecDateTimeValue>;
+ using ColVecTo = ColumnVector<ToFieldType>;
if (const ColVecFrom* col_from =
check_and_get_column<ColVecFrom>(named_from.column.get())) {
@@ -197,11 +221,13 @@ struct ConvertImplToTimeType {
auto& vec_null_map_to = col_null_map_to->get_data();
for (size_t i = 0; i < size; ++i) {
- auto& date_value = reinterpret_cast<VecDateTimeValue&>(vec_to[i]);
+ auto& date_value = reinterpret_cast<DateValueType&>(vec_to[i]);
if constexpr (IsDecimalNumber<FromFieldType>) {
vec_null_map_to[i] = !date_value.from_date_int64(
convert_from_decimal<FromDataType, DataTypeInt64>(
vec_from[i], vec_from.get_scale()));
+ } else if constexpr (IsDateV2Type<FromFieldType>) {
+ DataTypeDateV2::cast_to_date_time(vec_from[i], vec_to[i]);
} else {
vec_null_map_to[i] = !date_value.from_date_int64(vec_from[i]);
}
@@ -352,6 +378,10 @@ bool try_parse_impl(typename DataType::FieldType& x, ReadBuffer& rb, const DateL
return try_read_date_text(x, rb);
}
+ if constexpr (IsDateV2Type<DataType>) {
+ return try_read_date_v2_text(x, rb);
+ }
+
if constexpr (std::is_floating_point_v<typename DataType::FieldType>) {
return try_read_float_text(x, rb);
}
@@ -640,6 +670,7 @@ using FunctionToDecimal64 =
using FunctionToDecimal128 =
FunctionConvert<DataTypeDecimal<Decimal128>, NameToDecimal128, UnknownMonotonicity>;
using FunctionToDate = FunctionConvert<DataTypeDate, NameToDate, UnknownMonotonicity>;
+using FunctionToDateV2 = FunctionConvert<DataTypeDateV2, NameToDate, UnknownMonotonicity>;
using FunctionToDateTime = FunctionConvert<DataTypeDateTime, NameToDateTime, UnknownMonotonicity>;
template <typename DataType>
@@ -705,6 +736,10 @@ struct FunctionTo<DataTypeDate> {
using Type = FunctionToDate;
};
template <>
+struct FunctionTo<DataTypeDateV2> {
+ using Type = FunctionToDateV2;
+};
+template <>
struct FunctionTo<DataTypeDateTime> {
using Type = FunctionToDateTime;
};
@@ -968,9 +1003,11 @@ private:
/// In case when converting to Nullable type, we apply different parsing rule,
/// that will not throw an exception but return NULL in case of malformed input.
function = FunctionConvertFromString<DataType, NameCast>::create();
- } else if (requested_result_is_nullable && IsTimeType<DataType> &&
- !(check_and_get_data_type<DataTypeDateTime>(from_type.get()) ||
- check_and_get_data_type<DataTypeDate>(from_type.get()))) {
+ } else if (requested_result_is_nullable &&
+ (IsTimeType<DataType> || IsDateV2Type<DataType>)&&!(
+ check_and_get_data_type<DataTypeDateTime>(from_type.get()) ||
+ check_and_get_data_type<DataTypeDate>(from_type.get()) ||
+ check_and_get_data_type<DataTypeDateV2>(from_type.get()))) {
function = FunctionConvertToTimeType<DataType, NameCast>::create();
} else {
function = FunctionTo<DataType>::Type::create();
@@ -1008,7 +1045,7 @@ private:
WhichDataType which(type_index);
bool ok = which.is_int() || which.is_native_uint() || which.is_decimal() ||
- which.is_float() || which.is_date_or_datetime() ||
+ which.is_float() || which.is_date_or_datetime() || which.is_date_v2() ||
which.is_string_or_fixed_string();
if (!ok) {
LOG(FATAL) << fmt::format(
@@ -1236,8 +1273,9 @@ private:
block.get_by_position(result).column = tmp_block.get_by_position(result).column;
return Status::OK();
};
- } else
+ } else {
return wrapper;
+ }
}
/// 'from_type' and 'to_type' are nested types in case of Nullable.
@@ -1267,7 +1305,8 @@ private:
std::is_same_v<ToDataType, DataTypeFloat32> ||
std::is_same_v<ToDataType, DataTypeFloat64> ||
std::is_same_v<ToDataType, DataTypeDate> ||
- std::is_same_v<ToDataType, DataTypeDateTime>) {
+ std::is_same_v<ToDataType, DataTypeDateTime> ||
+ std::is_same_v<ToDataType, DataTypeDateV2>) {
ret = create_wrapper(from_type, check_and_get_data_type<ToDataType>(to_type.get()),
requested_result_is_nullable);
return true;
@@ -1350,6 +1389,11 @@ protected:
arguments[0].type->get_type_id() != TypeIndex::DateTime) &&
(type->get_type_id() == TypeIndex::Date ||
type->get_type_id() == TypeIndex::DateTime);
+ // 4. from_type is not DateTimeV2/DateV2, to_type is DateTimeV2/DateV2
+ need_to_be_nullable |= (arguments[0].type->get_type_id() != TypeIndex::DateV2 &&
+ arguments[0].type->get_type_id() != TypeIndex::DateTimeV2) &&
+ (type->get_type_id() == TypeIndex::DateV2 ||
+ type->get_type_id() == TypeIndex::DateTimeV2);
if (need_to_be_nullable) {
return make_nullable(type);
}
diff --git a/be/src/vec/functions/function_date_or_datetime_computation.cpp b/be/src/vec/functions/function_date_or_datetime_computation.cpp
index 7469dba76e..6a923d7d5e 100644
--- a/be/src/vec/functions/function_date_or_datetime_computation.cpp
+++ b/be/src/vec/functions/function_date_or_datetime_computation.cpp
@@ -21,36 +21,117 @@
namespace doris::vectorized {
-using FunctionAddSeconds = FunctionDateOrDateTimeComputation<AddSecondsImpl>;
-using FunctionAddMinutes = FunctionDateOrDateTimeComputation<AddMinutesImpl>;
-using FunctionAddHours = FunctionDateOrDateTimeComputation<AddHoursImpl>;
-using FunctionAddDays = FunctionDateOrDateTimeComputation<AddDaysImpl>;
-using FunctionAddWeeks = FunctionDateOrDateTimeComputation<AddWeeksImpl>;
-using FunctionAddMonths = FunctionDateOrDateTimeComputation<AddMonthsImpl>;
-using FunctionAddQuarters = FunctionDateOrDateTimeComputation<AddQuartersImpl>;
-using FunctionAddYears = FunctionDateOrDateTimeComputation<AddYearsImpl>;
-
-using FunctionSubSeconds = FunctionDateOrDateTimeComputation<SubtractSecondsImpl>;
-using FunctionSubMinutes = FunctionDateOrDateTimeComputation<SubtractMinutesImpl>;
-using FunctionSubHours = FunctionDateOrDateTimeComputation<SubtractHoursImpl>;
-using FunctionSubDays = FunctionDateOrDateTimeComputation<SubtractDaysImpl>;
-using FunctionSubWeeks = FunctionDateOrDateTimeComputation<SubtractWeeksImpl>;
-using FunctionSubMonths = FunctionDateOrDateTimeComputation<SubtractMonthsImpl>;
-using FunctionSubQuarters = FunctionDateOrDateTimeComputation<SubtractQuartersImpl>;
-using FunctionSubYears = FunctionDateOrDateTimeComputation<SubtractYearsImpl>;
-
-using FunctionDateDiff = FunctionDateOrDateTimeComputation<DateDiffImpl>;
-using FunctionTimeDiff = FunctionDateOrDateTimeComputation<TimeDiffImpl>;
-using FunctionYearsDiff = FunctionDateOrDateTimeComputation<YearsDiffImpl>;
-using FunctionMonthsDiff = FunctionDateOrDateTimeComputation<MonthsDiffImpl>;
-using FunctionDaysDiff = FunctionDateOrDateTimeComputation<DaysDiffImpl>;
-using FunctionWeeksDiff = FunctionDateOrDateTimeComputation<WeeksDiffImpl>;
-using FunctionHoursDiff = FunctionDateOrDateTimeComputation<HoursDiffImpl>;
-using FunctionMinutesDiff = FunctionDateOrDateTimeComputation<MintueSDiffImpl>;
-using FunctionSecondsDiff = FunctionDateOrDateTimeComputation<SecondsDiffImpl>;
-
-using FunctionToYearWeekTwoArgs = FunctionDateOrDateTimeComputation<ToYearWeekTwoArgsImpl>;
-using FunctionToWeekTwoArgs = FunctionDateOrDateTimeComputation<ToWeekTwoArgsImpl>;
+using FunctionAddSeconds = FunctionDateOrDateTimeComputation<
+ AddSecondsImpl<DataTypeDateTime, Int64, DataTypeDateTime>>;
+using FunctionAddSecondsV2 =
+ FunctionDateOrDateTimeComputation<AddSecondsImpl<DataTypeDateV2, UInt32, DataTypeDateV2>>;
+using FunctionAddMinutes = FunctionDateOrDateTimeComputation<
+ AddMinutesImpl<DataTypeDateTime, Int64, DataTypeDateTime>>;
+using FunctionAddHours =
+ FunctionDateOrDateTimeComputation<AddHoursImpl<DataTypeDateTime, Int64, DataTypeDateTime>>;
+using FunctionAddDays =
+ FunctionDateOrDateTimeComputation<AddDaysImpl<DataTypeDateTime, Int64, DataTypeDateTime>>;
+using FunctionAddWeeks =
+ FunctionDateOrDateTimeComputation<AddWeeksImpl<DataTypeDateTime, Int64, DataTypeDateTime>>;
+using FunctionAddMonths =
+ FunctionDateOrDateTimeComputation<AddMonthsImpl<DataTypeDateTime, Int64, DataTypeDateTime>>;
+using FunctionAddMinutesV2 =
+ FunctionDateOrDateTimeComputation<AddMinutesImpl<DataTypeDateV2, UInt32, DataTypeDateV2>>;
+using FunctionAddHoursV2 =
+ FunctionDateOrDateTimeComputation<AddHoursImpl<DataTypeDateV2, UInt32, DataTypeDateV2>>;
+using FunctionAddDaysV2 =
+ FunctionDateOrDateTimeComputation<AddDaysImpl<DataTypeDateV2, UInt32, DataTypeDateV2>>;
+using FunctionAddWeeksV2 =
+ FunctionDateOrDateTimeComputation<AddWeeksImpl<DataTypeDateV2, UInt32, DataTypeDateV2>>;
+using FunctionAddMonthsV2 =
+ FunctionDateOrDateTimeComputation<AddMonthsImpl<DataTypeDateV2, UInt32, DataTypeDateV2>>;
+using FunctionAddQuarters = FunctionDateOrDateTimeComputation<
+ AddQuartersImpl<DataTypeDateTime, Int64, DataTypeDateTime>>;
+using FunctionAddQuartersV2 =
+ FunctionDateOrDateTimeComputation<AddQuartersImpl<DataTypeDateV2, UInt32, DataTypeDateV2>>;
+using FunctionAddYears =
+ FunctionDateOrDateTimeComputation<AddYearsImpl<DataTypeDateTime, Int64, DataTypeDateTime>>;
+using FunctionAddYearsV2 =
+ FunctionDateOrDateTimeComputation<AddYearsImpl<DataTypeDateV2, UInt32, DataTypeDateV2>>;
+
+using FunctionSubSeconds = FunctionDateOrDateTimeComputation<
+ SubtractSecondsImpl<DataTypeDateTime, Int64, DataTypeDateTime>>;
+using FunctionSubMinutes = FunctionDateOrDateTimeComputation<
+ SubtractMinutesImpl<DataTypeDateTime, Int64, DataTypeDateTime>>;
+using FunctionSubHours = FunctionDateOrDateTimeComputation<
+ SubtractHoursImpl<DataTypeDateTime, Int64, DataTypeDateTime>>;
+using FunctionSubDays = FunctionDateOrDateTimeComputation<
+ SubtractDaysImpl<DataTypeDateTime, Int64, DataTypeDateTime>>;
+using FunctionSubWeeks = FunctionDateOrDateTimeComputation<
+ SubtractWeeksImpl<DataTypeDateTime, Int64, DataTypeDateTime>>;
+using FunctionSubMonths = FunctionDateOrDateTimeComputation<
+ SubtractMonthsImpl<DataTypeDateTime, Int64, DataTypeDateTime>>;
+using FunctionSubQuarters = FunctionDateOrDateTimeComputation<
+ SubtractQuartersImpl<DataTypeDateTime, Int64, DataTypeDateTime>>;
+using FunctionSubYears = FunctionDateOrDateTimeComputation<
+ SubtractYearsImpl<DataTypeDateTime, Int64, DataTypeDateTime>>;
+using FunctionSubSecondsV2 = FunctionDateOrDateTimeComputation<
+ SubtractSecondsImpl<DataTypeDateV2, UInt32, DataTypeDateV2>>;
+using FunctionSubMinutesV2 = FunctionDateOrDateTimeComputation<
+ SubtractMinutesImpl<DataTypeDateV2, UInt32, DataTypeDateV2>>;
+using FunctionSubHoursV2 = FunctionDateOrDateTimeComputation<
+ SubtractHoursImpl<DataTypeDateV2, UInt32, DataTypeDateV2>>;
+using FunctionSubDaysV2 =
+ FunctionDateOrDateTimeComputation<SubtractDaysImpl<DataTypeDateV2, UInt32, DataTypeDateV2>>;
+using FunctionSubWeeksV2 = FunctionDateOrDateTimeComputation<
+ SubtractWeeksImpl<DataTypeDateV2, UInt32, DataTypeDateV2>>;
+using FunctionSubMonthsV2 = FunctionDateOrDateTimeComputation<
+ SubtractMonthsImpl<DataTypeDateV2, UInt32, DataTypeDateV2>>;
+using FunctionSubQuartersV2 = FunctionDateOrDateTimeComputation<
+ SubtractQuartersImpl<DataTypeDateV2, UInt32, DataTypeDateV2>>;
+using FunctionSubYearsV2 = FunctionDateOrDateTimeComputation<
+ SubtractYearsImpl<DataTypeDateV2, UInt32, DataTypeDateV2>>;
+
+using FunctionDateDiff = FunctionDateOrDateTimeComputation<DateDiffImpl<
+ VecDateTimeValue, VecDateTimeValue, DataTypeDateTime, DataTypeDateTime, Int64, Int64>>;
+using FunctionTimeDiff = FunctionDateOrDateTimeComputation<TimeDiffImpl<
+ VecDateTimeValue, VecDateTimeValue, DataTypeDateTime, DataTypeDateTime, Int64, Int64>>;
+using FunctionYearsDiff = FunctionDateOrDateTimeComputation<YearsDiffImpl<
+ VecDateTimeValue, VecDateTimeValue, DataTypeDateTime, DataTypeDateTime, Int64, Int64>>;
+using FunctionMonthsDiff = FunctionDateOrDateTimeComputation<MonthsDiffImpl<
+ VecDateTimeValue, VecDateTimeValue, DataTypeDateTime, DataTypeDateTime, Int64, Int64>>;
+using FunctionDaysDiff = FunctionDateOrDateTimeComputation<DaysDiffImpl<
+ VecDateTimeValue, VecDateTimeValue, DataTypeDateTime, DataTypeDateTime, Int64, Int64>>;
+using FunctionWeeksDiff = FunctionDateOrDateTimeComputation<WeeksDiffImpl<
+ VecDateTimeValue, VecDateTimeValue, DataTypeDateTime, DataTypeDateTime, Int64, Int64>>;
+using FunctionHoursDiff = FunctionDateOrDateTimeComputation<HoursDiffImpl<
+ VecDateTimeValue, VecDateTimeValue, DataTypeDateTime, DataTypeDateTime, Int64, Int64>>;
+using FunctionMinutesDiff = FunctionDateOrDateTimeComputation<MintueSDiffImpl<
+ VecDateTimeValue, VecDateTimeValue, DataTypeDateTime, DataTypeDateTime, Int64, Int64>>;
+using FunctionSecondsDiff = FunctionDateOrDateTimeComputation<SecondsDiffImpl<
+ VecDateTimeValue, VecDateTimeValue, DataTypeDateTime, DataTypeDateTime, Int64, Int64>>;
+using FunctionDateDiffV2 = FunctionDateOrDateTimeComputation<
+ DateDiffImpl<DateV2Value, DateV2Value, DataTypeDateV2, DataTypeDateV2, UInt32, UInt32>>;
+using FunctionTimeDiffV2 = FunctionDateOrDateTimeComputation<
+ TimeDiffImpl<DateV2Value, DateV2Value, DataTypeDateV2, DataTypeDateV2, UInt32, UInt32>>;
+using FunctionYearsDiffV2 = FunctionDateOrDateTimeComputation<
+ YearsDiffImpl<DateV2Value, DateV2Value, DataTypeDateV2, DataTypeDateV2, UInt32, UInt32>>;
+using FunctionMonthsDiffV2 = FunctionDateOrDateTimeComputation<
+ MonthsDiffImpl<DateV2Value, DateV2Value, DataTypeDateV2, DataTypeDateV2, UInt32, UInt32>>;
+using FunctionDaysDiffV2 = FunctionDateOrDateTimeComputation<
+ DaysDiffImpl<DateV2Value, DateV2Value, DataTypeDateV2, DataTypeDateV2, UInt32, UInt32>>;
+using FunctionWeeksDiffV2 = FunctionDateOrDateTimeComputation<
+ WeeksDiffImpl<DateV2Value, DateV2Value, DataTypeDateV2, DataTypeDateV2, UInt32, UInt32>>;
+using FunctionHoursDiffV2 = FunctionDateOrDateTimeComputation<
+ HoursDiffImpl<DateV2Value, DateV2Value, DataTypeDateV2, DataTypeDateV2, UInt32, UInt32>>;
+using FunctionMinutesDiffV2 = FunctionDateOrDateTimeComputation<
+ MintueSDiffImpl<DateV2Value, DateV2Value, DataTypeDateV2, DataTypeDateV2, UInt32, UInt32>>;
+using FunctionSecondsDiffV2 = FunctionDateOrDateTimeComputation<
+ SecondsDiffImpl<DateV2Value, DateV2Value, DataTypeDateV2, DataTypeDateV2, UInt32, UInt32>>;
+
+using FunctionToYearWeekTwoArgs = FunctionDateOrDateTimeComputation<
+ ToYearWeekTwoArgsImpl<VecDateTimeValue, DataTypeDateTime, Int64>>;
+using FunctionToYearWeekTwoArgsV2 = FunctionDateOrDateTimeComputation<
+ ToYearWeekTwoArgsImpl<DateV2Value, DataTypeDateV2, UInt32>>;
+using FunctionToWeekTwoArgs = FunctionDateOrDateTimeComputation<
+ ToWeekTwoArgsImpl<VecDateTimeValue, DataTypeDateTime, Int64>>;
+using FunctionToWeekTwoArgsV2 =
+ FunctionDateOrDateTimeComputation<ToWeekTwoArgsImpl<DateV2Value, DataTypeDateV2, UInt32>>;
struct NowFunctionName {
static constexpr auto name = "now";
@@ -83,8 +164,12 @@ struct CurrentDateFunctionName {
static constexpr auto name = "current_date";
};
-using FunctionCurDate = FunctionCurrentDateOrDateTime<CurrentDateImpl<CurDateFunctionName>>;
-using FunctionCurrentDate = FunctionCurrentDateOrDateTime<CurrentDateImpl<CurrentDateFunctionName>>;
+FunctionBuilderPtr createCurrentDateFunctionBuilderFunction() {
+ return std::make_shared<CurrentDateFunctionBuilder<CurrentDateFunctionName>>();
+}
+FunctionBuilderPtr createCurDateFunctionBuilderFunction() {
+ return std::make_shared<CurrentDateFunctionBuilder<CurDateFunctionName>>();
+}
struct CurTimeFunctionName {
static constexpr auto name = "curtime";
@@ -102,23 +187,39 @@ void register_function_date_time_computation(SimpleFunctionFactory& factory) {
factory.register_function<FunctionAddMinutes>();
factory.register_function<FunctionAddHours>();
factory.register_function<FunctionAddDays>();
+ factory.register_function<FunctionAddSecondsV2>();
+ factory.register_function<FunctionAddMinutesV2>();
+ factory.register_function<FunctionAddHoursV2>();
+ factory.register_function<FunctionAddDaysV2>();
factory.register_alias("days_add", "date_add");
factory.register_alias("days_add", "adddate");
factory.register_function<FunctionAddWeeks>();
factory.register_function<FunctionAddMonths>();
factory.register_function<FunctionAddYears>();
factory.register_function<FunctionAddQuarters>();
+ factory.register_function<FunctionAddWeeksV2>();
+ factory.register_function<FunctionAddMonthsV2>();
+ factory.register_function<FunctionAddYearsV2>();
+ factory.register_function<FunctionAddQuartersV2>();
factory.register_function<FunctionSubSeconds>();
factory.register_function<FunctionSubMinutes>();
factory.register_function<FunctionSubHours>();
factory.register_function<FunctionSubDays>();
+ factory.register_function<FunctionSubSecondsV2>();
+ factory.register_function<FunctionSubMinutesV2>();
+ factory.register_function<FunctionSubHoursV2>();
+ factory.register_function<FunctionSubDaysV2>();
factory.register_alias("days_sub", "date_sub");
factory.register_alias("days_sub", "subdate");
factory.register_function<FunctionSubMonths>();
factory.register_function<FunctionSubYears>();
factory.register_function<FunctionSubQuarters>();
factory.register_function<FunctionSubWeeks>();
+ factory.register_function<FunctionSubMonthsV2>();
+ factory.register_function<FunctionSubYearsV2>();
+ factory.register_function<FunctionSubQuartersV2>();
+ factory.register_function<FunctionSubWeeksV2>();
factory.register_function<FunctionDateDiff>();
factory.register_function<FunctionTimeDiff>();
@@ -129,16 +230,67 @@ void register_function_date_time_computation(SimpleFunctionFactory& factory) {
factory.register_function<FunctionHoursDiff>();
factory.register_function<FunctionMinutesDiff>();
factory.register_function<FunctionSecondsDiff>();
+ factory.register_function<FunctionDateDiffV2>();
+ factory.register_function<FunctionTimeDiffV2>();
+ factory.register_function<FunctionYearsDiffV2>();
+ factory.register_function<FunctionMonthsDiffV2>();
+ factory.register_function<FunctionWeeksDiffV2>();
+ factory.register_function<FunctionDaysDiffV2>();
+ factory.register_function<FunctionHoursDiffV2>();
+ factory.register_function<FunctionMinutesDiffV2>();
+ factory.register_function<FunctionSecondsDiffV2>();
+
+ // Functions below make dateV2 be compatible with V1
+ factory.register_function<FunctionDateOrDateTimeComputation<DateDiffImpl<
+ VecDateTimeValue, DateV2Value, DataTypeDateTime, DataTypeDateV2, Int64, UInt32>>>();
+ factory.register_function<FunctionDateOrDateTimeComputation<TimeDiffImpl<
+ VecDateTimeValue, DateV2Value, DataTypeDateTime, DataTypeDateV2, Int64, UInt32>>>();
+ factory.register_function<FunctionDateOrDateTimeComputation<YearsDiffImpl<
+ VecDateTimeValue, DateV2Value, DataTypeDateTime, DataTypeDateV2, Int64, UInt32>>>();
+ factory.register_function<FunctionDateOrDateTimeComputation<MonthsDiffImpl<
+ VecDateTimeValue, DateV2Value, DataTypeDateTime, DataTypeDateV2, Int64, UInt32>>>();
+ factory.register_function<FunctionDateOrDateTimeComputation<WeeksDiffImpl<
+ VecDateTimeValue, DateV2Value, DataTypeDateTime, DataTypeDateV2, Int64, UInt32>>>();
+ factory.register_function<FunctionDateOrDateTimeComputation<DaysDiffImpl<
+ VecDateTimeValue, DateV2Value, DataTypeDateTime, DataTypeDateV2, Int64, UInt32>>>();
+ factory.register_function<FunctionDateOrDateTimeComputation<HoursDiffImpl<
+ VecDateTimeValue, DateV2Value, DataTypeDateTime, DataTypeDateV2, Int64, UInt32>>>();
+ factory.register_function<FunctionDateOrDateTimeComputation<MintueSDiffImpl<
+ VecDateTimeValue, DateV2Value, DataTypeDateTime, DataTypeDateV2, Int64, UInt32>>>();
+ factory.register_function<FunctionDateOrDateTimeComputation<SecondsDiffImpl<
+ VecDateTimeValue, DateV2Value, DataTypeDateTime, DataTypeDateV2, Int64, UInt32>>>();
+
+ factory.register_function<FunctionDateOrDateTimeComputation<DateDiffImpl<
+ DateV2Value, VecDateTimeValue, DataTypeDateV2, DataTypeDateTime, UInt32, Int64>>>();
+ factory.register_function<FunctionDateOrDateTimeComputation<TimeDiffImpl<
+ DateV2Value, VecDateTimeValue, DataTypeDateV2, DataTypeDateTime, UInt32, Int64>>>();
+ factory.register_function<FunctionDateOrDateTimeComputation<YearsDiffImpl<
+ DateV2Value, VecDateTimeValue, DataTypeDateV2, DataTypeDateTime, UInt32, Int64>>>();
+ factory.register_function<FunctionDateOrDateTimeComputation<MonthsDiffImpl<
+ DateV2Value, VecDateTimeValue, DataTypeDateV2, DataTypeDateTime, UInt32, Int64>>>();
+ factory.register_function<FunctionDateOrDateTimeComputation<WeeksDiffImpl<
+ DateV2Value, VecDateTimeValue, DataTypeDateV2, DataTypeDateTime, UInt32, Int64>>>();
+ factory.register_function<FunctionDateOrDateTimeComputation<DaysDiffImpl<
+ DateV2Value, VecDateTimeValue, DataTypeDateV2, DataTypeDateTime, UInt32, Int64>>>();
+ factory.register_function<FunctionDateOrDateTimeComputation<HoursDiffImpl<
+ DateV2Value, VecDateTimeValue, DataTypeDateV2, DataTypeDateTime, UInt32, Int64>>>();
+ factory.register_function<FunctionDateOrDateTimeComputation<MintueSDiffImpl<
+ DateV2Value, VecDateTimeValue, DataTypeDateV2, DataTypeDateTime, UInt32, Int64>>>();
+ factory.register_function<FunctionDateOrDateTimeComputation<SecondsDiffImpl<
+ DateV2Value, VecDateTimeValue, DataTypeDateV2, DataTypeDateTime, UInt32, Int64>>>();
factory.register_function<FunctionToYearWeekTwoArgs>();
factory.register_function<FunctionToWeekTwoArgs>();
+ factory.register_function<FunctionToYearWeekTwoArgsV2>();
+ factory.register_function<FunctionToWeekTwoArgsV2>();
factory.register_function<FunctionNow>();
factory.register_function<FunctionCurrentTimestamp>();
factory.register_function<FunctionLocalTime>();
factory.register_function<FunctionLocalTimestamp>();
- factory.register_function<FunctionCurDate>();
- factory.register_function<FunctionCurrentDate>();
+ factory.register_function(CurrentDateFunctionName::name,
+ &createCurrentDateFunctionBuilderFunction);
+ factory.register_function(CurDateFunctionName::name, &createCurDateFunctionBuilderFunction);
factory.register_function<FunctionCurTime>();
factory.register_function<FunctionCurrentTime>();
factory.register_function<FunctionUtcTimeStamp>();
diff --git a/be/src/vec/functions/function_date_or_datetime_computation.h b/be/src/vec/functions/function_date_or_datetime_computation.h
index efca50e65e..f748d5ad6d 100644
--- a/be/src/vec/functions/function_date_or_datetime_computation.h
+++ b/be/src/vec/functions/function_date_or_datetime_computation.h
@@ -41,14 +41,28 @@ inline Int64 date_time_add(const Int64& t, Int64 delta, bool& is_null) {
return binary_cast<doris::vectorized::VecDateTimeValue, Int64>(ts_value);
}
-#define ADD_TIME_FUNCTION_IMPL(CLASS, NAME, UNIT) \
- struct CLASS { \
- using ReturnType = DataTypeDateTime; \
- static constexpr auto name = #NAME; \
- static constexpr auto is_nullable = true; \
- static inline Int64 execute(const Int64& t, Int64 delta, bool& is_null) { \
- return date_time_add<TimeUnit::UNIT>(t, delta, is_null); \
- } \
+template <TimeUnit unit>
+inline Int64 date_time_add(const UInt32& t, Int64 delta, bool& is_null) {
+ auto ts_value = binary_cast<UInt32, doris::vectorized::DateV2Value>(t);
+ TimeInterval interval(unit, delta, false);
+ is_null = !ts_value.date_add_interval(interval, unit);
+
+ return binary_cast<doris::vectorized::DateV2Value, UInt32>(ts_value);
+}
+
+#define ADD_TIME_FUNCTION_IMPL(CLASS, NAME, UNIT) \
+ template <typename DateType, typename ArgType, typename ResultType> \
+ struct CLASS { \
+ using ReturnType = ResultType; \
+ static constexpr auto name = #NAME; \
+ static constexpr auto is_nullable = true; \
+ static inline ArgType execute(const ArgType& t, Int64 delta, bool& is_null) { \
+ return date_time_add<TimeUnit::UNIT>(t, delta, is_null); \
+ } \
+ \
+ static DataTypes get_variadic_argument_types() { \
+ return {std::make_shared<DateType>(), std::make_shared<DataTypeInt32>()}; \
+ } \
}
ADD_TIME_FUNCTION_IMPL(AddSecondsImpl, seconds_add, SECOND);
@@ -59,84 +73,132 @@ ADD_TIME_FUNCTION_IMPL(AddWeeksImpl, weeks_add, WEEK);
ADD_TIME_FUNCTION_IMPL(AddMonthsImpl, months_add, MONTH);
ADD_TIME_FUNCTION_IMPL(AddYearsImpl, years_add, YEAR);
+template <typename DateType, typename ArgType, typename ResultType>
struct AddQuartersImpl {
- using ReturnType = DataTypeDateTime;
+ using ReturnType = ResultType;
static constexpr auto name = "quarters_add";
static constexpr auto is_nullable = true;
- static inline Int64 execute(const Int64& t, Int64 delta, bool& is_null) {
+ static inline Int64 execute(const ArgType& t, Int64 delta, bool& is_null) {
return date_time_add<TimeUnit::MONTH>(t, delta * 3, is_null);
}
+
+ static DataTypes get_variadic_argument_types() { return {std::make_shared<DateType>()}; }
};
-template <typename Transform>
+template <typename Transform, typename DateType, typename ArgType, typename ResultType>
struct SubtractIntervalImpl {
- using ReturnType = DataTypeDateTime;
+ using ReturnType = ResultType;
static constexpr auto is_nullable = true;
- static inline Int64 execute(const Int64& t, Int64 delta, bool& is_null) {
+ static inline Int64 execute(const ArgType& t, Int64 delta, bool& is_null) {
return Transform::execute(t, -delta, is_null);
}
+
+ static DataTypes get_variadic_argument_types() {
+ return {std::make_shared<DateType>(), std::make_shared<DataTypeInt32>()};
+ }
};
-struct SubtractSecondsImpl : SubtractIntervalImpl<AddSecondsImpl> {
+template <typename DateType, typename ArgType, typename ResultType>
+struct SubtractSecondsImpl : SubtractIntervalImpl<AddSecondsImpl<DateType, ArgType, ResultType>,
+ DateType, ArgType, ResultType> {
static constexpr auto name = "seconds_sub";
};
-struct SubtractMinutesImpl : SubtractIntervalImpl<AddMinutesImpl> {
+
+template <typename DateType, typename ArgType, typename ResultType>
+struct SubtractMinutesImpl : SubtractIntervalImpl<AddMinutesImpl<DateType, ArgType, ResultType>,
+ DateType, ArgType, ResultType> {
static constexpr auto name = "minutes_sub";
};
-struct SubtractHoursImpl : SubtractIntervalImpl<AddHoursImpl> {
+
+template <typename DateType, typename ArgType, typename ResultType>
+struct SubtractHoursImpl : SubtractIntervalImpl<AddHoursImpl<DateType, ArgType, ResultType>,
+ DateType, ArgType, ResultType> {
static constexpr auto name = "hours_sub";
};
-struct SubtractDaysImpl : SubtractIntervalImpl<AddDaysImpl> {
+
+template <typename DateType, typename ArgType, typename ResultType>
+struct SubtractDaysImpl : SubtractIntervalImpl<AddDaysImpl<DateType, ArgType, ResultType>, DateType,
+ ArgType, ResultType> {
static constexpr auto name = "days_sub";
};
-struct SubtractWeeksImpl : SubtractIntervalImpl<AddWeeksImpl> {
+
+template <typename DateType, typename ArgType, typename ResultType>
+struct SubtractWeeksImpl : SubtractIntervalImpl<AddWeeksImpl<DateType, ArgType, ResultType>,
+ DateType, ArgType, ResultType> {
static constexpr auto name = "weeks_sub";
};
-struct SubtractMonthsImpl : SubtractIntervalImpl<AddMonthsImpl> {
+
+template <typename DateType, typename ArgType, typename ResultType>
+struct SubtractMonthsImpl : SubtractIntervalImpl<AddMonthsImpl<DateType, ArgType, ResultType>,
+ DateType, ArgType, ResultType> {
static constexpr auto name = "months_sub";
};
-struct SubtractQuartersImpl : SubtractIntervalImpl<AddQuartersImpl> {
+
+template <typename DateType, typename ArgType, typename ResultType>
+struct SubtractQuartersImpl : SubtractIntervalImpl<AddQuartersImpl<DateType, ArgType, ResultType>,
+ DateType, ArgType, ResultType> {
static constexpr auto name = "quarters_sub";
};
-struct SubtractYearsImpl : SubtractIntervalImpl<AddYearsImpl> {
+
+template <typename DateType, typename ArgType, typename ResultType>
+struct SubtractYearsImpl : SubtractIntervalImpl<AddYearsImpl<DateType, ArgType, ResultType>,
+ DateType, ArgType, ResultType> {
static constexpr auto name = "years_sub";
};
+template <typename DateValueType1, typename DateValueType2, typename DateType1, typename DateType2,
+ typename ArgType1, typename ArgType2>
struct DateDiffImpl {
using ReturnType = DataTypeInt32;
static constexpr auto name = "datediff";
static constexpr auto is_nullable = false;
- static inline Int32 execute(const Int64& t0, const Int64& t1, bool& is_null) {
- const auto& ts0 = reinterpret_cast<const doris::vectorized::VecDateTimeValue&>(t0);
- const auto& ts1 = reinterpret_cast<const doris::vectorized::VecDateTimeValue&>(t1);
+ static inline Int32 execute(const ArgType1& t0, const ArgType2& t1, bool& is_null) {
+ const auto& ts0 = reinterpret_cast<const DateValueType1&>(t0);
+ const auto& ts1 = reinterpret_cast<const DateValueType2&>(t1);
is_null = !ts0.is_valid_date() || !ts1.is_valid_date();
return ts0.daynr() - ts1.daynr();
}
+
+ static DataTypes get_variadic_argument_types() {
+ return {std::make_shared<DateType1>(), std::make_shared<DateType2>()};
+ }
};
+template <typename DateValueType1, typename DateValueType2, typename DateType1, typename DateType2,
+ typename ArgType1, typename ArgType2>
struct TimeDiffImpl {
using ReturnType = DataTypeFloat64;
static constexpr auto name = "timediff";
static constexpr auto is_nullable = false;
- static inline double execute(const Int64& t0, const Int64& t1, bool& is_null) {
- const auto& ts0 = reinterpret_cast<const doris::vectorized::VecDateTimeValue&>(t0);
- const auto& ts1 = reinterpret_cast<const doris::vectorized::VecDateTimeValue&>(t1);
+ static inline double execute(const ArgType1& t0, const ArgType2& t1, bool& is_null) {
+ const auto& ts0 = reinterpret_cast<const DateValueType1&>(t0);
+ const auto& ts1 = reinterpret_cast<const DateValueType2&>(t1);
is_null = !ts0.is_valid_date() || !ts1.is_valid_date();
return ts0.second_diff(ts1);
}
+
+ static DataTypes get_variadic_argument_types() {
+ return {std::make_shared<DateType1>(), std::make_shared<DateType2>()};
+ }
};
-#define TIME_DIFF_FUNCTION_IMPL(CLASS, NAME, UNIT) \
- struct CLASS { \
- using ReturnType = DataTypeInt64; \
- static constexpr auto name = #NAME; \
- static constexpr auto is_nullable = false; \
- static inline int64_t execute(const Int64& t0, const Int64& t1, bool& is_null) { \
- const auto& ts0 = reinterpret_cast<const doris::vectorized::VecDateTimeValue&>(t0); \
- const auto& ts1 = reinterpret_cast<const doris::vectorized::VecDateTimeValue&>(t1); \
- is_null = !ts0.is_valid_date() || !ts1.is_valid_date(); \
- return VecDateTimeValue::datetime_diff<TimeUnit::UNIT>(ts1, ts0); \
- } \
+#define TIME_DIFF_FUNCTION_IMPL(CLASS, NAME, UNIT) \
+ template <typename DateValueType1, typename DateValueType2, typename DateType1, \
+ typename DateType2, typename ArgType1, typename ArgType2> \
+ struct CLASS { \
+ using ReturnType = DataTypeInt64; \
+ static constexpr auto name = #NAME; \
+ static constexpr auto is_nullable = false; \
+ static inline Int64 execute(const ArgType1& t0, const ArgType2& t1, bool& is_null) { \
+ const auto& ts0 = reinterpret_cast<const DateValueType1&>(t0); \
+ const auto& ts1 = reinterpret_cast<const DateValueType2&>(t1); \
+ is_null = !ts0.is_valid_date() || !ts1.is_valid_date(); \
+ return datetime_diff<TimeUnit::UNIT>(ts1, ts0); \
+ } \
+ \
+ static DataTypes get_variadic_argument_types() { \
+ return {std::make_shared<DateType1>(), std::make_shared<DateType2>()}; \
+ } \
}
TIME_DIFF_FUNCTION_IMPL(YearsDiffImpl, years_diff, YEAR);
@@ -147,29 +209,30 @@ TIME_DIFF_FUNCTION_IMPL(HoursDiffImpl, hours_diff, HOUR);
TIME_DIFF_FUNCTION_IMPL(MintueSDiffImpl, minutes_diff, MINUTE);
TIME_DIFF_FUNCTION_IMPL(SecondsDiffImpl, seconds_diff, SECOND);
-#define TIME_FUNCTION_TWO_ARGS_IMPL(CLASS, NAME, FUNCTION) \
- struct CLASS { \
- using ReturnType = DataTypeInt32; \
- static constexpr auto name = #NAME; \
- static constexpr auto is_nullable = false; \
- static inline int64_t execute(const Int64& t0, const Int32 mode, bool& is_null) { \
- const auto& ts0 = reinterpret_cast<const doris::vectorized::VecDateTimeValue&>(t0); \
- is_null = !ts0.is_valid_date(); \
- return ts0.FUNCTION; \
- } \
- static DataTypes get_variadic_argument_types() { \
- return {std::make_shared<DataTypeDateTime>(), std::make_shared<DataTypeInt32>()}; \
- } \
+#define TIME_FUNCTION_TWO_ARGS_IMPL(CLASS, NAME, FUNCTION) \
+ template <typename DateValueType, typename DateType, typename ArgType> \
+ struct CLASS { \
+ using ReturnType = DataTypeInt32; \
+ static constexpr auto name = #NAME; \
+ static constexpr auto is_nullable = false; \
+ static inline int64_t execute(const ArgType& t0, const Int32 mode, bool& is_null) { \
+ const auto& ts0 = reinterpret_cast<const DateValueType&>(t0); \
+ is_null = !ts0.is_valid_date(); \
+ return ts0.FUNCTION; \
+ } \
+ static DataTypes get_variadic_argument_types() { \
+ return {std::make_shared<DateType>(), std::make_shared<DataTypeInt32>()}; \
+ } \
}
TIME_FUNCTION_TWO_ARGS_IMPL(ToYearWeekTwoArgsImpl, yearweek, year_week(mysql_week_mode(mode)));
TIME_FUNCTION_TWO_ARGS_IMPL(ToWeekTwoArgsImpl, week, week(mysql_week_mode(mode)));
-template <typename FromType, typename ToType, typename Transform>
+template <typename FromType1, typename FromType2, typename ToType, typename Transform>
struct DateTimeOp {
// use for (DateTime, DateTime) -> other_type
- static void vector_vector(const PaddedPODArray<FromType>& vec_from0,
- const PaddedPODArray<FromType>& vec_from1,
+ static void vector_vector(const PaddedPODArray<FromType1>& vec_from0,
+ const PaddedPODArray<FromType2>& vec_from1,
PaddedPODArray<ToType>& vec_to, NullMap& null_map) {
size_t size = vec_from0.size();
vec_to.resize(size);
@@ -185,7 +248,7 @@ struct DateTimeOp {
}
// use for (DateTime, int32) -> other_type
- static void vector_vector(const PaddedPODArray<FromType>& vec_from0,
+ static void vector_vector(const PaddedPODArray<FromType1>& vec_from0,
const PaddedPODArray<Int32>& vec_from1,
PaddedPODArray<ToType>& vec_to, NullMap& null_map) {
size_t size = vec_from0.size();
@@ -198,7 +261,7 @@ struct DateTimeOp {
}
// use for (DateTime, const DateTime) -> other_type
- static void vector_constant(const PaddedPODArray<FromType>& vec_from,
+ static void vector_constant(const PaddedPODArray<FromType1>& vec_from,
PaddedPODArray<ToType>& vec_to, NullMap& null_map, Int128& delta) {
size_t size = vec_from.size();
vec_to.resize(size);
@@ -211,7 +274,7 @@ struct DateTimeOp {
}
// use for (DateTime, const ColumnNumber) -> other_type
- static void vector_constant(const PaddedPODArray<FromType>& vec_from,
+ static void vector_constant(const PaddedPODArray<FromType1>& vec_from,
PaddedPODArray<ToType>& vec_to, NullMap& null_map, Int64 delta) {
size_t size = vec_from.size();
vec_to.resize(size);
@@ -224,7 +287,7 @@ struct DateTimeOp {
}
// use for (const DateTime, ColumnNumber) -> other_type
- static void constant_vector(const FromType& from, PaddedPODArray<ToType>& vec_to,
+ static void constant_vector(const FromType1& from, PaddedPODArray<ToType>& vec_to,
NullMap& null_map, const IColumn& delta) {
size_t size = delta.size();
vec_to.resize(size);
@@ -236,8 +299,8 @@ struct DateTimeOp {
}
}
- static void constant_vector(const FromType& from, PaddedPODArray<ToType>& vec_to,
- NullMap& null_map, const PaddedPODArray<Int64>& delta) {
+ static void constant_vector(const FromType1& from, PaddedPODArray<ToType>& vec_to,
+ NullMap& null_map, const PaddedPODArray<FromType2>& delta) {
size_t size = delta.size();
vec_to.resize(size);
null_map.resize_fill(size, false);
@@ -248,14 +311,14 @@ struct DateTimeOp {
}
};
-template <typename FromType, typename Transform>
+template <typename FromType1, typename Transform, typename FromType2 = FromType1>
struct DateTimeAddIntervalImpl {
static Status execute(Block& block, const ColumnNumbers& arguments, size_t result) {
using ToType = typename Transform::ReturnType::FieldType;
- using Op = DateTimeOp<FromType, ToType, Transform>;
+ using Op = DateTimeOp<FromType1, FromType2, ToType, Transform>;
const ColumnPtr source_col = block.get_by_position(arguments[0]).column;
- if (const auto* sources = check_and_get_column<ColumnVector<FromType>>(source_col.get())) {
+ if (const auto* sources = check_and_get_column<ColumnVector<FromType1>>(source_col.get())) {
auto col_to = ColumnVector<ToType>::create();
auto null_map = ColumnUInt8::create();
const IColumn& delta_column = *block.get_by_position(arguments[1]).column;
@@ -276,7 +339,7 @@ struct DateTimeAddIntervalImpl {
}
} else {
if (const auto* delta_vec_column0 =
- check_and_get_column<ColumnVector<FromType>>(delta_column)) {
+ check_and_get_column<ColumnVector<FromType2>>(delta_column)) {
Op::vector_vector(sources->get_data(), delta_vec_column0->get_data(),
col_to->get_data(), null_map->get_data());
} else {
@@ -291,17 +354,17 @@ struct DateTimeAddIntervalImpl {
block.get_by_position(result).column =
ColumnNullable::create(std::move(col_to), std::move(null_map));
} else if (const auto* sources_const =
- check_and_get_column_const<ColumnVector<FromType>>(source_col.get())) {
+ check_and_get_column_const<ColumnVector<FromType1>>(source_col.get())) {
auto col_to = ColumnVector<ToType>::create();
auto null_map = ColumnUInt8::create();
- if (const auto* delta_vec_column = check_and_get_column<ColumnVector<FromType>>(
+ if (const auto* delta_vec_column = check_and_get_column<ColumnVector<FromType2>>(
*block.get_by_position(arguments[1]).column)) {
- Op::constant_vector(sources_const->template get_value<FromType>(),
+ Op::constant_vector(sources_const->template get_value<FromType1>(),
col_to->get_data(), null_map->get_data(),
delta_vec_column->get_data());
} else {
- Op::constant_vector(sources_const->template get_value<FromType>(),
+ Op::constant_vector(sources_const->template get_value<FromType2>(),
col_to->get_data(), null_map->get_data(),
*block.get_by_position(arguments[1]).column);
}
@@ -344,7 +407,8 @@ public:
}
if (arguments.size() == 2) {
- if (!is_date_or_datetime(arguments[0].type)) {
+ if (!is_date_or_datetime(arguments[0].type) &&
+ !is_date_v2_or_datetime_v2(arguments[0].type)) {
LOG(FATAL) << fmt::format(
"Illegal type {} of argument of function {}. Should be a date or a date "
"with time",
@@ -352,6 +416,7 @@ public:
}
} else {
if (!WhichDataType(arguments[0].type).is_date_time() ||
+ !WhichDataType(arguments[0].type).is_date_time_v2() ||
!WhichDataType(arguments[2].type).is_string()) {
LOG(FATAL) << fmt::format(
"Function {} supports 2 or 3 arguments. The 1st argument must be of type "
@@ -368,15 +433,56 @@ public:
Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments,
size_t result, size_t input_rows_count) override {
- const IDataType* from_type = block.get_by_position(arguments[0]).type.get();
- WhichDataType which(from_type);
-
- if (which.is_date()) {
+ const IDataType* first_arg_type = block.get_by_position(arguments[0]).type.get();
+ const IDataType* second_arg_type = block.get_by_position(arguments[1]).type.get();
+ WhichDataType which1(first_arg_type);
+ WhichDataType which2(second_arg_type);
+
+ if (which1.is_date() && which2.is_date()) {
+ return DateTimeAddIntervalImpl<DataTypeDate::FieldType, Transform,
+ DataTypeDate::FieldType>::execute(block, arguments,
+ result);
+ } else if (which1.is_date_time() && which2.is_date()) {
+ return DateTimeAddIntervalImpl<DataTypeDateTime::FieldType, Transform,
+ DataTypeDate::FieldType>::execute(block, arguments,
+ result);
+ } else if (which1.is_date_v2() && which2.is_date()) {
+ return DateTimeAddIntervalImpl<DataTypeDateV2::FieldType, Transform,
+ DataTypeDate::FieldType>::execute(block, arguments,
+ result);
+ } else if (which1.is_date() && which2.is_date_time()) {
+ return DateTimeAddIntervalImpl<DataTypeDate::FieldType, Transform,
+ DataTypeDateTime::FieldType>::execute(block, arguments,
+ result);
+ } else if (which1.is_date() && which2.is_date_v2()) {
+ return DateTimeAddIntervalImpl<DataTypeDate::FieldType, Transform,
+ DataTypeDateV2::FieldType>::execute(block, arguments,
+ result);
+ } else if (which1.is_date_v2() && which2.is_date_time()) {
+ return DateTimeAddIntervalImpl<DataTypeDateV2::FieldType, Transform,
+ DataTypeDateTime::FieldType>::execute(block, arguments,
+ result);
+ } else if (which1.is_date_v2() && which2.is_date_v2()) {
+ return DateTimeAddIntervalImpl<DataTypeDateV2::FieldType, Transform,
+ DataTypeDateV2::FieldType>::execute(block, arguments,
+ result);
+ } else if (which1.is_date_time() && which2.is_date_time()) {
+ return DateTimeAddIntervalImpl<DataTypeDateTime::FieldType, Transform,
+ DataTypeDateTime::FieldType>::execute(block, arguments,
+ result);
+ } else if (which1.is_date_time() && which2.is_date_v2()) {
+ return DateTimeAddIntervalImpl<DataTypeDateTime::FieldType, Transform,
+ DataTypeDateV2::FieldType>::execute(block, arguments,
+ result);
+ } else if (which1.is_date()) {
return DateTimeAddIntervalImpl<DataTypeDate::FieldType, Transform>::execute(
block, arguments, result);
- } else if (which.is_date_time()) {
+ } else if (which1.is_date_time()) {
return DateTimeAddIntervalImpl<DataTypeDateTime::FieldType, Transform>::execute(
block, arguments, result);
+ } else if (which1.is_date_v2()) {
+ return DateTimeAddIntervalImpl<DataTypeDateV2::FieldType, Transform>::execute(
+ block, arguments, result);
} else {
return Status::RuntimeError(
fmt::format("Illegal type {} of argument of function {}",
@@ -435,28 +541,47 @@ struct CurrentDateTimeImpl {
}
};
-template <typename FunctionName>
+template <typename FunctionName, typename DateType, typename NativeType>
struct CurrentDateImpl {
- using ReturnType = DataTypeDate;
+ using ReturnType = DateType;
static constexpr auto name = FunctionName::name;
static Status execute(FunctionContext* context, Block& block, size_t result,
size_t input_rows_count) {
- auto col_to = ColumnVector<Int64>::create();
- VecDateTimeValue dtv;
- if (dtv.from_unixtime(context->impl()->state()->timestamp_ms() / 1000,
- context->impl()->state()->timezone_obj())) {
- reinterpret_cast<VecDateTimeValue*>(&dtv)->set_type(TIME_DATE);
- auto date_packed_int = binary_cast<doris::vectorized::VecDateTimeValue, int64_t>(
- *reinterpret_cast<VecDateTimeValue*>(&dtv));
- for (int i = 0; i < input_rows_count; i++) {
- col_to->insert_data(
- const_cast<const char*>(reinterpret_cast<char*>(&date_packed_int)), 0);
+ auto col_to = ColumnVector<NativeType>::create();
+ if constexpr (std::is_same_v<DateType, DataTypeDateV2>) {
+ DateV2Value dtv;
+ if (dtv.from_unixtime(context->impl()->state()->timestamp_ms() / 1000,
+ context->impl()->state()->timezone_obj())) {
+ auto date_packed_int =
+ binary_cast<DateV2Value, uint32_t>(*reinterpret_cast<DateV2Value*>(&dtv));
+ for (int i = 0; i < input_rows_count; i++) {
+ col_to->insert_data(
+ const_cast<const char*>(reinterpret_cast<char*>(&date_packed_int)), 0);
+ }
+ } else {
+ auto invalid_val = 0;
+ for (int i = 0; i < input_rows_count; i++) {
+ col_to->insert_data(
+ const_cast<const char*>(reinterpret_cast<char*>(&invalid_val)), 0);
+ }
}
} else {
- auto invalid_val = 0;
- for (int i = 0; i < input_rows_count; i++) {
- col_to->insert_data(const_cast<const char*>(reinterpret_cast<char*>(&invalid_val)),
- 0);
+ VecDateTimeValue dtv;
+ if (dtv.from_unixtime(context->impl()->state()->timestamp_ms() / 1000,
+ context->impl()->state()->timezone_obj())) {
+ reinterpret_cast<VecDateTimeValue*>(&dtv)->set_type(TIME_DATE);
+ auto date_packed_int = binary_cast<doris::vectorized::VecDateTimeValue, int64_t>(
+ *reinterpret_cast<VecDateTimeValue*>(&dtv));
+ for (int i = 0; i < input_rows_count; i++) {
+ col_to->insert_data(
+ const_cast<const char*>(reinterpret_cast<char*>(&date_packed_int)), 0);
+ }
+ } else {
+ auto invalid_val = 0;
+ for (int i = 0; i < input_rows_count; i++) {
+ col_to->insert_data(
+ const_cast<const char*>(reinterpret_cast<char*>(&invalid_val)), 0);
+ }
}
}
block.get_by_position(result).column = std::move(col_to);
@@ -517,4 +642,38 @@ struct UtcTimestampImpl {
}
};
+template <typename FunctionName>
+class CurrentDateFunctionBuilder : public FunctionBuilderImpl {
+public:
+ explicit CurrentDateFunctionBuilder() = default;
+
+ String get_name() const override { return FunctionName::name; }
+ size_t get_number_of_arguments() const override { return 0; }
+
+protected:
+ DataTypePtr get_return_type_impl(const DataTypes& arguments) const override {
+ return make_nullable(std::make_shared<DataTypeDate>());
+ }
+ DataTypePtr get_return_type_impl(const ColumnsWithTypeAndName& arguments) const override {
+ return make_nullable(std::make_shared<DataTypeDate>());
+ }
+
+ bool use_default_implementation_for_nulls() const override { return false; }
+
+ FunctionBasePtr build_impl(const ColumnsWithTypeAndName& arguments,
+ const DataTypePtr& return_type) const override {
+ DataTypes data_types(arguments.size());
+ for (size_t i = 0; i < arguments.size(); ++i) data_types[i] = arguments[i].type;
+ if (is_date_v2(return_type)) {
+ auto function = FunctionCurrentDateOrDateTime<
+ CurrentDateImpl<FunctionName, DataTypeDateV2, UInt32>>::create();
+ return std::make_shared<DefaultFunction>(function, data_types, return_type);
+ } else {
+ auto function = FunctionCurrentDateOrDateTime<
+ CurrentDateImpl<FunctionName, DataTypeDate, Int64>>::create();
+ return std::make_shared<DefaultFunction>(function, data_types, return_type);
+ }
+ }
+};
+
} // namespace doris::vectorized
diff --git a/be/src/vec/functions/function_date_or_datetime_to_something.h b/be/src/vec/functions/function_date_or_datetime_to_something.h
index 1440126390..634f6064d1 100644
--- a/be/src/vec/functions/function_date_or_datetime_to_something.h
+++ b/be/src/vec/functions/function_date_or_datetime_to_something.h
@@ -48,14 +48,16 @@ public:
DataTypePtr get_return_type_impl(const ColumnsWithTypeAndName& arguments) const override {
if (arguments.size() == 1) {
- if (!is_date_or_datetime(arguments[0].type)) {
+ if (!is_date_or_datetime(arguments[0].type) &&
+ !is_date_v2_or_datetime_v2(arguments[0].type)) {
LOG(FATAL) << fmt::format(
"Illegal type {} of argument of function {}. Should be a date or a date "
"with time",
arguments[0].type->get_name(), get_name());
}
} else if (arguments.size() == 2) {
- if (!is_date_or_datetime(arguments[0].type)) {
+ if (!is_date_or_datetime(arguments[0].type) &&
+ !is_date_v2_or_datetime_v2(arguments[0].type)) {
LOG(FATAL) << fmt::format(
"Illegal type {} of argument of function {}. Should be a date or a date "
"with time",
@@ -92,8 +94,9 @@ public:
const IDataType* from_type = block.get_by_position(arguments[0]).type.get();
WhichDataType which(from_type);
- return DateTimeTransformImpl<Int64, typename ToDataType::FieldType, Transform>::execute(
- block, arguments, result, input_rows_count);
+ return DateTimeTransformImpl<typename Transform::ARG_TYPE, typename ToDataType::FieldType,
+ Transform>::execute(block, arguments, result,
+ input_rows_count);
}
bool has_information_about_monotonicity() const override { return true; }
diff --git a/be/src/vec/functions/function_date_or_datetime_to_string.cpp b/be/src/vec/functions/function_date_or_datetime_to_string.cpp
index 7815f41480..6f9d815fdd 100644
--- a/be/src/vec/functions/function_date_or_datetime_to_string.cpp
+++ b/be/src/vec/functions/function_date_or_datetime_to_string.cpp
@@ -24,12 +24,16 @@
namespace doris::vectorized {
-using FunctionDayName = FunctionDateOrDateTimeToString<DayNameImpl>;
-using FunctionMonthName = FunctionDateOrDateTimeToString<MonthNameImpl>;
+using FunctionDayName = FunctionDateOrDateTimeToString<DayNameImpl<VecDateTimeValue, Int64>>;
+using FunctionDayNameV2 = FunctionDateOrDateTimeToString<DayNameImpl<DateV2Value, UInt32>>;
+using FunctionMonthName = FunctionDateOrDateTimeToString<MonthNameImpl<VecDateTimeValue, Int64>>;
+using FunctionMonthNameV2 = FunctionDateOrDateTimeToString<MonthNameImpl<DateV2Value, UInt32>>;
void register_function_date_time_to_string(SimpleFunctionFactory& factory) {
factory.register_function<FunctionDayName>();
factory.register_function<FunctionMonthName>();
+ factory.register_function<FunctionDayNameV2>();
+ factory.register_function<FunctionMonthNameV2>();
}
} // namespace doris::vectorized
\ No newline at end of file
diff --git a/be/src/vec/functions/function_date_or_datetime_to_string.h b/be/src/vec/functions/function_date_or_datetime_to_string.h
index 9992f189b8..8b06869489 100644
--- a/be/src/vec/functions/function_date_or_datetime_to_string.h
+++ b/be/src/vec/functions/function_date_or_datetime_to_string.h
@@ -32,6 +32,8 @@ template <typename Transform>
class FunctionDateOrDateTimeToString : public IFunction {
public:
static constexpr auto name = Transform::name;
+ static constexpr bool has_variadic_argument =
+ !std::is_void_v<decltype(has_variadic_argument_types(std::declval<Transform>()))>;
static FunctionPtr create() { return std::make_shared<FunctionDateOrDateTimeToString>(); }
String get_name() const override { return name; }
@@ -42,13 +44,21 @@ public:
return make_nullable(std::make_shared<DataTypeString>());
}
+ bool is_variadic() const override { return true; }
+
+ DataTypes get_variadic_argument_types_impl() const override {
+ if constexpr (has_variadic_argument) return Transform::get_variadic_argument_types();
+ return {};
+ }
+
bool use_default_implementation_for_constants() const override { return true; }
ColumnNumbers get_arguments_that_are_always_constant() const override { return {1}; }
Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments,
size_t result, size_t input_rows_count) override {
const ColumnPtr source_col = block.get_by_position(arguments[0]).column;
- const auto* sources = check_and_get_column<ColumnVector<Int64>>(source_col.get());
+ const auto* sources =
+ check_and_get_column<ColumnVector<typename Transform::ARG_TYPE>>(source_col.get());
auto col_res = ColumnString::create();
auto null_map = ColumnVector<UInt8>::create();
// Support all input of datetime is valind to make sure not null return
diff --git a/be/src/vec/functions/function_datetime_string_to_string.cpp b/be/src/vec/functions/function_datetime_string_to_string.cpp
index a499773669..e4685a96a7 100644
--- a/be/src/vec/functions/function_datetime_string_to_string.cpp
+++ b/be/src/vec/functions/function_datetime_string_to_string.cpp
@@ -21,12 +21,19 @@
namespace doris::vectorized {
-using FunctionDateFormat = FunctionDateTimeStringToString<DateFormatImpl>;
-using FunctionFromUnixTime = FunctionDateTimeStringToString<FromUnixTimeImpl>;
+using FunctionDateFormat = FunctionDateTimeStringToString<DateFormatImpl<VecDateTimeValue, Int64>>;
+using FunctionDateFormatV2 = FunctionDateTimeStringToString<DateFormatImpl<DateV2Value, UInt32>>;
+using FunctionFromUnixTime = FunctionDateTimeStringToString<FromUnixTimeImpl<VecDateTimeValue>>;
+
+FunctionBuilderPtr createFromUnixTimeFunction() {
+ return std::make_shared<FromUnixTimeFunctionBuilder>();
+}
void register_function_date_time_string_to_string(SimpleFunctionFactory& factory) {
factory.register_function<FunctionDateFormat>();
+ factory.register_function<FunctionDateFormatV2>();
factory.register_function<FunctionFromUnixTime>();
+ factory.register_function("from_unixtime", &createFromUnixTimeFunction);
}
} // namespace doris::vectorized
\ No newline at end of file
diff --git a/be/src/vec/functions/function_datetime_string_to_string.h b/be/src/vec/functions/function_datetime_string_to_string.h
index 6502d0fd90..0cb2d9df96 100644
--- a/be/src/vec/functions/function_datetime_string_to_string.h
+++ b/be/src/vec/functions/function_datetime_string_to_string.h
@@ -32,12 +32,19 @@ template <typename Transform>
class FunctionDateTimeStringToString : public IFunction {
public:
static constexpr auto name = Transform::name;
+ static constexpr bool has_variadic_argument =
+ !std::is_void_v<decltype(has_variadic_argument_types(std::declval<Transform>()))>;
+
static FunctionPtr create() { return std::make_shared<FunctionDateTimeStringToString>(); }
String get_name() const override { return name; }
- size_t get_number_of_arguments() const override { return 0; }
bool is_variadic() const override { return true; }
+ size_t get_number_of_arguments() const override { return 0; }
+ DataTypes get_variadic_argument_types_impl() const override {
+ if constexpr (has_variadic_argument) return Transform::get_variadic_argument_types();
+ return {};
+ }
DataTypePtr get_return_type_impl(const ColumnsWithTypeAndName& arguments) const override {
return make_nullable(std::make_shared<DataTypeString>());
@@ -98,4 +105,35 @@ public:
}
};
+class FromUnixTimeFunctionBuilder : public FunctionBuilderImpl {
+public:
+ explicit FromUnixTimeFunctionBuilder() = default;
+
+ String get_name() const override { return "from_unixtime"; }
+ bool is_variadic() const override { return true; }
+ size_t get_number_of_arguments() const override { return 0; }
+
+ ColumnNumbers get_arguments_that_are_always_constant() const override { return {1}; }
+
+protected:
+ DataTypePtr get_return_type_impl(const DataTypes& arguments) const override {
+ return make_nullable(std::make_shared<DataTypeString>());
+ }
+ DataTypePtr get_return_type_impl(const ColumnsWithTypeAndName& arguments) const override {
+ return make_nullable(std::make_shared<DataTypeString>());
+ }
+
+ bool use_default_implementation_for_nulls() const override { return false; }
+
+ FunctionBasePtr build_impl(const ColumnsWithTypeAndName& arguments,
+ const DataTypePtr& return_type) const override {
+ DataTypes data_types(arguments.size());
+ for (size_t i = 0; i < arguments.size(); ++i) data_types[i] = arguments[i].type;
+ // TODO: we still use VecDateTimeValue to convert unix timestamp to string
+ auto function =
+ FunctionDateTimeStringToString<FromUnixTimeImpl<VecDateTimeValue>>::create();
+ return std::make_shared<DefaultFunction>(function, data_types, return_type);
+ }
+};
+
} // namespace doris::vectorized
diff --git a/be/src/vec/functions/function_timestamp.cpp b/be/src/vec/functions/function_timestamp.cpp
index 22f159bd8e..691714cb33 100644
--- a/be/src/vec/functions/function_timestamp.cpp
+++ b/be/src/vec/functions/function_timestamp.cpp
@@ -29,16 +29,17 @@
namespace doris::vectorized {
+template <typename DateType, typename NativeType>
struct StrToDate {
static constexpr auto name = "str_to_date";
- using ReturnType = DataTypeDateTime;
- using ColumnType = ColumnVector<Int64>;
+ using ReturnType = DateType;
+ using ColumnType = ColumnVector<NativeType>;
static void vector_vector(FunctionContext* context, const ColumnString::Chars& ldata,
const ColumnString::Offsets& loffsets,
const ColumnString::Chars& rdata,
- const ColumnString::Offsets& roffsets, ColumnType::Container& res,
- NullMap& null_map) {
+ const ColumnString::Offsets& roffsets,
+ PaddedPODArray<NativeType>& res, NullMap& null_map) {
size_t size = loffsets.size();
res.resize(size);
for (size_t i = 0; i < size; ++i) {
@@ -48,15 +49,23 @@ struct StrToDate {
const char* r_raw_str = reinterpret_cast<const char*>(&rdata[roffsets[i - 1]]);
int r_str_size = roffsets[i] - roffsets[i - 1] - 1;
- auto& ts_val = *reinterpret_cast<VecDateTimeValue*>(&res[i]);
- if (!ts_val.from_date_format_str(r_raw_str, r_str_size, l_raw_str, l_str_size)) {
- null_map[i] = 1;
- }
- if (context->impl()->get_return_type().type ==
- doris_udf::FunctionContext::Type::TYPE_DATETIME) {
- ts_val.to_datetime();
+ if constexpr (std::is_same_v<DateType, DataTypeDateTime> ||
+ std::is_same_v<DateType, DataTypeDate>) {
+ auto& ts_val = *reinterpret_cast<VecDateTimeValue*>(&res[i]);
+ if (!ts_val.from_date_format_str(r_raw_str, r_str_size, l_raw_str, l_str_size)) {
+ null_map[i] = 1;
+ }
+ if (context->impl()->get_return_type().type ==
+ doris_udf::FunctionContext::Type::TYPE_DATETIME) {
+ ts_val.to_datetime();
+ } else {
+ ts_val.cast_to_date();
+ }
} else {
- ts_val.cast_to_date();
+ auto& ts_val = *reinterpret_cast<DateV2Value*>(&res[i]);
+ if (!ts_val.from_date_format_str(r_raw_str, r_str_size, l_raw_str, l_str_size)) {
+ null_map[i] = 1;
+ }
}
}
}
@@ -66,16 +75,17 @@ struct NameMakeDate {
static constexpr auto name = "makedate";
};
-template <typename LeftDataType, typename RightDataType>
+template <typename LeftDataType, typename RightDataType, typename ResultDateType,
+ typename ReturnType>
struct MakeDateImpl {
- using ResultDataType = DataTypeDateTime;
+ using ResultDataType = ResultDateType;
using LeftDataColumnType = ColumnVector<typename LeftDataType::FieldType>;
using RightDataColumnType = ColumnVector<typename RightDataType::FieldType>;
- using ColumnType = ColumnVector<Int64>;
+ using ColumnType = ColumnVector<ReturnType>;
- static void vector_vector(const typename LeftDataColumnType::Container& ldata,
- const typename RightDataColumnType::Container& rdata,
- ColumnType::Container& res, NullMap& null_map) {
+ static void vector_vector(const PaddedPODArray<typename LeftDataType::FieldType>& ldata,
+ const PaddedPODArray<typename RightDataType::FieldType>& rdata,
+ PaddedPODArray<ReturnType>& res, NullMap& null_map) {
auto len = ldata.size();
res.resize(len);
@@ -87,29 +97,40 @@ struct MakeDateImpl {
continue;
}
- auto& res_val = *reinterpret_cast<VecDateTimeValue*>(&res[i]);
-
- VecDateTimeValue ts_value = VecDateTimeValue();
- ts_value.set_time(l, 1, 1, 0, 0, 0);
-
- DateTimeVal ts_val;
- ts_value.to_datetime_val(&ts_val);
- if (ts_val.is_null) {
- null_map[i] = 1;
- continue;
- }
-
- TimeInterval interval(DAY, r - 1, false);
- res_val = VecDateTimeValue::from_datetime_val(ts_val);
- if (!res_val.date_add_interval(interval, DAY)) {
- null_map[i] = 1;
- continue;
+ if constexpr (std::is_same_v<ResultDataType, DataTypeDateTime> ||
+ std::is_same_v<ResultDataType, DataTypeDate>) {
+ auto& res_val = *reinterpret_cast<VecDateTimeValue*>(&res[i]);
+
+ VecDateTimeValue ts_value = VecDateTimeValue();
+ ts_value.set_time(l, 1, 1, 0, 0, 0);
+
+ DateTimeVal ts_val;
+ ts_value.to_datetime_val(&ts_val);
+ if (ts_val.is_null) {
+ null_map[i] = 1;
+ continue;
+ }
+
+ TimeInterval interval(DAY, r - 1, false);
+ res_val = VecDateTimeValue::from_datetime_val(ts_val);
+ if (!res_val.date_add_interval(interval, DAY)) {
+ null_map[i] = 1;
+ continue;
+ }
+ res_val.cast_to_date();
+ } else {
+ DateV2Value* value = new (&res[i]) DateV2Value();
+ value->set_time(l, 1, 1);
+ TimeInterval interval(DAY, r - 1, false);
+ if (!value->date_add_interval(interval, DAY)) {
+ null_map[i] = 1;
+ }
}
- res_val.cast_to_date();
}
}
};
+template <typename DateType>
class FromDays : public IFunction {
public:
static constexpr auto name = "from_days";
@@ -125,7 +146,7 @@ public:
bool use_default_implementation_for_nulls() const override { return true; }
DataTypePtr get_return_type_impl(const DataTypes& arguments) const override {
- return make_nullable(std::make_shared<DataTypeDate>());
+ return make_nullable(std::make_shared<DateType>());
}
Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments,
@@ -138,15 +159,23 @@ public:
auto data_col = assert_cast<const ColumnVector<Int32>*>(argument_column.get());
for (int i = 0; i < input_rows_count; i++) {
- const auto& cur_data = data_col->get_data()[i];
- auto& ts_value = *reinterpret_cast<VecDateTimeValue*>(&res_data[i]);
- if (!ts_value.from_date_daynr(cur_data)) {
- null_map->get_data()[i] = 1;
- continue;
+ if constexpr (std::is_same_v<DateType, DataTypeDate>) {
+ const auto& cur_data = data_col->get_data()[i];
+ auto& ts_value = *reinterpret_cast<VecDateTimeValue*>(&res_data[i]);
+ if (!ts_value.from_date_daynr(cur_data)) {
+ null_map->get_data()[i] = 1;
+ continue;
+ }
+ DateTimeVal ts_val;
+ ts_value.to_datetime_val(&ts_val);
+ ts_value = VecDateTimeValue::from_datetime_val(ts_val);
+ } else {
+ const auto& cur_data = data_col->get_data()[i];
+ auto& ts_value = *reinterpret_cast<DateV2Value*>(&res_data[i]);
+ if (!ts_value.get_date_from_daynr(cur_data)) {
+ null_map->get_data()[i] = 1;
+ }
}
- DateTimeVal ts_val;
- ts_value.to_datetime_val(&ts_val);
- ts_value = VecDateTimeValue::from_datetime_val(ts_val);
}
block.replace_by_position(
result, ColumnNullable::create(std::move(res_column), std::move(null_map)));
@@ -154,9 +183,10 @@ public:
}
};
-using FunctionStrToDate = FunctionBinaryStringOperateToNullType<StrToDate>;
-using FunctionMakeDate =
- FunctionBinaryToNullType<DataTypeInt32, DataTypeInt32, MakeDateImpl, NameMakeDate>;
+using FunctionStrToDate = FunctionBinaryStringOperateToNullType<StrToDate<DataTypeDateTime, Int64>>;
+
+using FunctionMakeDate = FunctionBinaryToNullType<DataTypeInt32, DataTypeInt32, DataTypeDateTime,
+ Int64, MakeDateImpl, NameMakeDate>;
struct UnixTimeStampImpl {
static Int32 trim_timestamp(Int64 timestamp) {
@@ -182,8 +212,9 @@ struct UnixTimeStampImpl {
}
};
+template <typename DateType>
struct UnixTimeStampDateImpl {
- static DataTypes get_variadic_argument_types() { return {std::make_shared<DataTypeDate>()}; }
+ static DataTypes get_variadic_argument_types() { return {std::make_shared<DateType>()}; }
static DataTypePtr get_return_type_impl(const ColumnsWithTypeAndName& arguments) {
return make_nullable(std::make_shared<DataTypeInt32>());
@@ -210,14 +241,27 @@ struct UnixTimeStampDateImpl {
}
StringRef source = col_source->get_data_at(i);
- const VecDateTimeValue& ts_value =
- reinterpret_cast<const VecDateTimeValue&>(*source.data);
- int64_t timestamp;
- if (!ts_value.unix_timestamp(×tamp, context->impl()->state()->timezone_obj())) {
- null_map_data[i] = true;
+ if constexpr (std::is_same_v<DateType, DataTypeDate>) {
+ const VecDateTimeValue& ts_value =
+ reinterpret_cast<const VecDateTimeValue&>(*source.data);
+ int64_t timestamp;
+ if (!ts_value.unix_timestamp(×tamp,
+ context->impl()->state()->timezone_obj())) {
+ null_map_data[i] = true;
+ } else {
+ null_map_data[i] = false;
+ col_result_data[i] = UnixTimeStampImpl::trim_timestamp(timestamp);
+ }
} else {
- null_map_data[i] = false;
- col_result_data[i] = UnixTimeStampImpl::trim_timestamp(timestamp);
+ const DateV2Value& ts_value = reinterpret_cast<const DateV2Value&>(*source.data);
+ int64_t timestamp;
+ if (!ts_value.unix_timestamp(×tamp,
+ context->impl()->state()->timezone_obj())) {
+ null_map_data[i] = true;
+ } else {
+ null_map_data[i] = false;
+ col_result_data[i] = UnixTimeStampImpl::trim_timestamp(timestamp);
+ }
}
}
@@ -228,10 +272,9 @@ struct UnixTimeStampDateImpl {
}
};
-struct UnixTimeStampDatetimeImpl : public UnixTimeStampDateImpl {
- static DataTypes get_variadic_argument_types() {
- return {std::make_shared<DataTypeDateTime>()};
- }
+template <typename DateType>
+struct UnixTimeStampDatetimeImpl : public UnixTimeStampDateImpl<DateType> {
+ static DataTypes get_variadic_argument_types() { return {std::make_shared<DateType>()}; }
};
struct UnixTimeStampStrImpl {
@@ -320,11 +363,13 @@ public:
void register_function_timestamp(SimpleFunctionFactory& factory) {
factory.register_function<FunctionStrToDate>();
factory.register_function<FunctionMakeDate>();
- factory.register_function<FromDays>();
+ factory.register_function<FromDays<DataTypeDate>>();
factory.register_function<FunctionUnixTimestamp<UnixTimeStampImpl>>();
- factory.register_function<FunctionUnixTimestamp<UnixTimeStampDateImpl>>();
- factory.register_function<FunctionUnixTimestamp<UnixTimeStampDatetimeImpl>>();
+ factory.register_function<FunctionUnixTimestamp<UnixTimeStampDateImpl<DataTypeDate>>>();
+ factory.register_function<FunctionUnixTimestamp<UnixTimeStampDateImpl<DataTypeDateV2>>>();
+ factory.register_function<FunctionUnixTimestamp<UnixTimeStampDatetimeImpl<DataTypeDate>>>();
+ factory.register_function<FunctionUnixTimestamp<UnixTimeStampDatetimeImpl<DataTypeDateV2>>>();
factory.register_function<FunctionUnixTimestamp<UnixTimeStampStrImpl>>();
}
diff --git a/be/src/vec/functions/function_totype.h b/be/src/vec/functions/function_totype.h
index 83ddc0cead..27a96b03ad 100644
--- a/be/src/vec/functions/function_totype.h
+++ b/be/src/vec/functions/function_totype.h
@@ -267,8 +267,9 @@ private:
};
// func(type,type) -> nullable(type)
-template <typename LeftDataType, typename RightDataType,
- template <typename, typename> typename Impl, typename Name>
+template <typename LeftDataType, typename RightDataType, typename ResultDateType,
+ typename ReturnType, template <typename, typename, typename, typename> typename Impl,
+ typename Name>
class FunctionBinaryToNullType : public IFunction {
public:
static constexpr auto name = Name::name;
@@ -276,7 +277,8 @@ public:
String get_name() const override { return name; }
size_t get_number_of_arguments() const override { return 2; }
DataTypePtr get_return_type_impl(const DataTypes& arguments) const override {
- using ResultDataType = typename Impl<LeftDataType, RightDataType>::ResultDataType;
+ using ResultDataType = typename Impl<LeftDataType, RightDataType, ResultDateType,
+ ReturnType>::ResultDataType;
return make_nullable(std::make_shared<ResultDataType>());
}
@@ -300,7 +302,8 @@ public:
}
}
- using ResultDataType = typename Impl<LeftDataType, RightDataType>::ResultDataType;
+ using ResultDataType = typename Impl<LeftDataType, RightDataType, ResultDateType,
+ ReturnType>::ResultDataType;
using T0 = typename LeftDataType::FieldType;
using T1 = typename RightDataType::FieldType;
@@ -323,7 +326,7 @@ public:
if (auto col_left = check_and_get_column<ColVecLeft>(argument_columns[0].get())) {
if (auto col_right = check_and_get_column<ColVecRight>(argument_columns[1].get())) {
- Impl<LeftDataType, RightDataType>::vector_vector(
+ Impl<LeftDataType, RightDataType, ResultDateType, ReturnType>::vector_vector(
col_left->get_data(), col_right->get_data(), vec_res, null_map->get_data());
block.get_by_position(result).column =
ColumnNullable::create(std::move(col_res), std::move(null_map));
diff --git a/be/src/vec/functions/time_of_function.cpp b/be/src/vec/functions/time_of_function.cpp
index 071704826d..86aedbae48 100644
--- a/be/src/vec/functions/time_of_function.cpp
+++ b/be/src/vec/functions/time_of_function.cpp
@@ -22,12 +22,31 @@
namespace doris::vectorized {
-using FunctionWeekOfYear = FunctionDateOrDateTimeToSomething<DataTypeInt32, WeekOfYearImpl>;
-using FunctionDayOfYear = FunctionDateOrDateTimeToSomething<DataTypeInt32, DayOfYearImpl>;
-using FunctionDayOfWeek = FunctionDateOrDateTimeToSomething<DataTypeInt32, DayOfWeekImpl>;
-using FunctionDayOfMonth = FunctionDateOrDateTimeToSomething<DataTypeInt32, DayOfMonthImpl>;
-using FunctionYearWeek = FunctionDateOrDateTimeToSomething<DataTypeInt32, ToYearWeekOneArgImpl>;
-using FunctionWeekDay = FunctionDateOrDateTimeToSomething<DataTypeInt32, WeekDayImpl>;
+using FunctionWeekOfYear =
+ FunctionDateOrDateTimeToSomething<DataTypeInt32, WeekOfYearImpl<VecDateTimeValue, Int64>>;
+using FunctionWeekOfYearV2 =
+ FunctionDateOrDateTimeToSomething<DataTypeInt32, WeekOfYearImpl<DateV2Value, UInt32>>;
+using FunctionDayOfYear =
+ FunctionDateOrDateTimeToSomething<DataTypeInt32, DayOfYearImpl<VecDateTimeValue, Int64>>;
+using FunctionDayOfYearV2 =
+ FunctionDateOrDateTimeToSomething<DataTypeInt32, DayOfYearImpl<DateV2Value, UInt32>>;
+using FunctionDayOfWeek =
+ FunctionDateOrDateTimeToSomething<DataTypeInt32, DayOfWeekImpl<VecDateTimeValue, Int64>>;
+using FunctionDayOfWeekV2 =
+ FunctionDateOrDateTimeToSomething<DataTypeInt32, DayOfWeekImpl<DateV2Value, UInt32>>;
+using FunctionDayOfMonth =
+ FunctionDateOrDateTimeToSomething<DataTypeInt32, DayOfMonthImpl<VecDateTimeValue, Int64>>;
+using FunctionDayOfMonthV2 =
+ FunctionDateOrDateTimeToSomething<DataTypeInt32, DayOfMonthImpl<DateV2Value, UInt32>>;
+using FunctionYearWeek =
+ FunctionDateOrDateTimeToSomething<DataTypeInt32,
+ ToYearWeekOneArgImpl<VecDateTimeValue, Int64>>;
+using FunctionYearWeekV2 =
+ FunctionDateOrDateTimeToSomething<DataTypeInt32, ToYearWeekOneArgImpl<DateV2Value, UInt32>>;
+using FunctionWeekDay =
+ FunctionDateOrDateTimeToSomething<DataTypeInt32, WeekDayImpl<VecDateTimeValue, Int64>>;
+using FunctionWeekDayV2 =
+ FunctionDateOrDateTimeToSomething<DataTypeInt32, WeekDayImpl<DateV2Value, UInt32>>;
void register_function_time_of_function(SimpleFunctionFactory& factory) {
factory.register_function<FunctionDayOfWeek>();
@@ -36,5 +55,11 @@ void register_function_time_of_function(SimpleFunctionFactory& factory) {
factory.register_function<FunctionWeekOfYear>();
factory.register_function<FunctionYearWeek>();
factory.register_function<FunctionWeekDay>();
+ factory.register_function<FunctionDayOfWeekV2>();
+ factory.register_function<FunctionDayOfMonthV2>();
+ factory.register_function<FunctionDayOfYearV2>();
+ factory.register_function<FunctionWeekOfYearV2>();
+ factory.register_function<FunctionYearWeekV2>();
+ factory.register_function<FunctionWeekDayV2>();
}
} // namespace doris::vectorized
\ No newline at end of file
diff --git a/be/src/vec/functions/to_time_function.cpp b/be/src/vec/functions/to_time_function.cpp
index c0e862892c..3a7c6dba78 100644
--- a/be/src/vec/functions/to_time_function.cpp
+++ b/be/src/vec/functions/to_time_function.cpp
@@ -23,18 +23,53 @@
namespace doris::vectorized {
-using FunctionYear = FunctionDateOrDateTimeToSomething<DataTypeInt32, ToYearImpl>;
-using FunctionQuarter = FunctionDateOrDateTimeToSomething<DataTypeInt32, ToQuarterImpl>;
-using FunctionMonth = FunctionDateOrDateTimeToSomething<DataTypeInt32, ToMonthImpl>;
-using FunctionDay = FunctionDateOrDateTimeToSomething<DataTypeInt32, ToDayImpl>;
-using FunctionWeek = FunctionDateOrDateTimeToSomething<DataTypeInt32, ToWeekOneArgImpl>;
-using FunctionHour = FunctionDateOrDateTimeToSomething<DataTypeInt32, ToHourImpl>;
-using FunctionMinute = FunctionDateOrDateTimeToSomething<DataTypeInt32, ToMinuteImpl>;
-using FunctionSecond = FunctionDateOrDateTimeToSomething<DataTypeInt32, ToSecondImpl>;
-using FunctionToDays = FunctionDateOrDateTimeToSomething<DataTypeInt32, ToDaysImpl>;
-using FunctionToDate = FunctionDateOrDateTimeToSomething<DataTypeDateTime, ToDateImpl>;
-using FunctionDate = FunctionDateOrDateTimeToSomething<DataTypeDateTime, DateImpl>;
-using FunctionTimeStamp = FunctionDateOrDateTimeToSomething<DataTypeDateTime, TimeStampImpl>;
+using FunctionYear =
+ FunctionDateOrDateTimeToSomething<DataTypeInt32, ToYearImpl<VecDateTimeValue, Int64>>;
+using FunctionYearV2 =
+ FunctionDateOrDateTimeToSomething<DataTypeInt32, ToYearImpl<DateV2Value, UInt32>>;
+using FunctionQuarter =
+ FunctionDateOrDateTimeToSomething<DataTypeInt32, ToQuarterImpl<VecDateTimeValue, Int64>>;
+using FunctionQuarterV2 =
+ FunctionDateOrDateTimeToSomething<DataTypeInt32, ToQuarterImpl<DateV2Value, UInt32>>;
+using FunctionMonth =
+ FunctionDateOrDateTimeToSomething<DataTypeInt32, ToMonthImpl<VecDateTimeValue, Int64>>;
+using FunctionMonthV2 =
+ FunctionDateOrDateTimeToSomething<DataTypeInt32, ToMonthImpl<DateV2Value, UInt32>>;
+using FunctionDay =
+ FunctionDateOrDateTimeToSomething<DataTypeInt32, ToDayImpl<VecDateTimeValue, Int64>>;
+using FunctionDayV2 =
+ FunctionDateOrDateTimeToSomething<DataTypeInt32, ToDayImpl<DateV2Value, UInt32>>;
+using FunctionWeek =
+ FunctionDateOrDateTimeToSomething<DataTypeInt32, ToWeekOneArgImpl<VecDateTimeValue, Int64>>;
+using FunctionWeekV2 =
+ FunctionDateOrDateTimeToSomething<DataTypeInt32, ToWeekOneArgImpl<DateV2Value, UInt32>>;
+using FunctionHour =
+ FunctionDateOrDateTimeToSomething<DataTypeInt32, ToHourImpl<VecDateTimeValue, Int64>>;
+using FunctionHourV2 =
+ FunctionDateOrDateTimeToSomething<DataTypeInt32, ToHourImpl<DateV2Value, UInt32>>;
+using FunctionMinute =
+ FunctionDateOrDateTimeToSomething<DataTypeInt32, ToMinuteImpl<VecDateTimeValue, Int64>>;
+using FunctionMinuteV2 =
+ FunctionDateOrDateTimeToSomething<DataTypeInt32, ToMinuteImpl<DateV2Value, UInt32>>;
+using FunctionSecond =
+ FunctionDateOrDateTimeToSomething<DataTypeInt32, ToSecondImpl<VecDateTimeValue, Int64>>;
+using FunctionSecondV2 =
+ FunctionDateOrDateTimeToSomething<DataTypeInt32, ToSecondImpl<DateV2Value, UInt32>>;
+using FunctionToDays =
+ FunctionDateOrDateTimeToSomething<DataTypeInt32, ToDaysImpl<VecDateTimeValue, Int64>>;
+using FunctionToDaysV2 =
+ FunctionDateOrDateTimeToSomething<DataTypeInt32, ToDaysImpl<DateV2Value, UInt32>>;
+using FunctionToDate =
+ FunctionDateOrDateTimeToSomething<DataTypeDateTime, ToDateImpl<VecDateTimeValue, Int64>>;
+using FunctionToDateV2 =
+ FunctionDateOrDateTimeToSomething<DataTypeDateV2, ToDateImpl<DateV2Value, UInt32>>;
+using FunctionDate =
+ FunctionDateOrDateTimeToSomething<DataTypeDateTime, DateImpl<VecDateTimeValue, Int64>>;
+using FunctionDateV2 =
+ FunctionDateOrDateTimeToSomething<DataTypeDateV2, DateImpl<DateV2Value, UInt32>>;
+using FunctionTimeStamp = FunctionDateOrDateTimeToSomething<DataTypeDateTime, TimeStampImpl<Int64>>;
+using FunctionTimeStampV2 =
+ FunctionDateOrDateTimeToSomething<DataTypeDateV2, TimeStampImpl<UInt32>>;
void register_function_to_time_function(SimpleFunctionFactory& factory) {
factory.register_function<FunctionSecond>();
@@ -49,6 +84,18 @@ void register_function_to_time_function(SimpleFunctionFactory& factory) {
factory.register_function<FunctionToDate>();
factory.register_function<FunctionDate>();
factory.register_function<FunctionTimeStamp>();
+ factory.register_function<FunctionSecondV2>();
+ factory.register_function<FunctionMinuteV2>();
+ factory.register_function<FunctionHourV2>();
+ factory.register_function<FunctionDayV2>();
+ factory.register_function<FunctionWeekV2>();
+ factory.register_function<FunctionMonthV2>();
+ factory.register_function<FunctionYearV2>();
+ factory.register_function<FunctionQuarterV2>();
+ factory.register_function<FunctionToDaysV2>();
+ factory.register_function<FunctionToDateV2>();
+ factory.register_function<FunctionDateV2>();
+ factory.register_function<FunctionTimeStampV2>();
}
} // namespace doris::vectorized
diff --git a/be/src/vec/io/io_helper.h b/be/src/vec/io/io_helper.h
index 1633380fcc..8425c0ec4d 100644
--- a/be/src/vec/io/io_helper.h
+++ b/be/src/vec/io/io_helper.h
@@ -279,6 +279,18 @@ bool read_date_text_impl(T& x, ReadBuffer& buf) {
return ans;
}
+template <typename T>
+bool read_date_v2_text_impl(T& x, ReadBuffer& buf) {
+ static_assert(std::is_same_v<UInt32, T>);
+ auto dv = binary_cast<UInt32, DateV2Value>(x);
+ auto ans = dv.from_date_str(buf.position(), buf.count());
+
+ // only to match the is_all_read() check to prevent return null
+ buf.position() = buf.end();
+ x = binary_cast<DateV2Value, UInt32>(dv);
+ return ans;
+}
+
template <typename T>
bool read_decimal_text_impl(T& x, ReadBuffer& buf) {
static_assert(IsDecimalNumber<T>);
@@ -335,4 +347,9 @@ template <typename T>
bool try_read_date_text(T& x, ReadBuffer& in) {
return read_date_text_impl<T>(x, in);
}
+
+template <typename T>
+bool try_read_date_v2_text(T& x, ReadBuffer& in) {
+ return read_date_v2_text_impl<T>(x, in);
+}
} // namespace doris::vectorized
diff --git a/be/src/vec/olap/olap_data_convertor.cpp b/be/src/vec/olap/olap_data_convertor.cpp
index abfaaef57f..72a55e545b 100644
--- a/be/src/vec/olap/olap_data_convertor.cpp
+++ b/be/src/vec/olap/olap_data_convertor.cpp
@@ -55,6 +55,10 @@ OlapBlockDataConvertor::create_olap_column_data_convertor(const TabletColumn& co
case FieldType::OLAP_FIELD_TYPE_DATE: {
return std::make_unique<OlapColumnDataConvertorDate>();
}
+ case FieldType::OLAP_FIELD_TYPE_DATEV2: {
+ return std::make_unique<OlapColumnDataConvertorDateV2>();
+ break;
+ }
case FieldType::OLAP_FIELD_TYPE_DATETIME: {
return std::make_unique<OlapColumnDataConvertorDateTime>();
}
@@ -467,47 +471,102 @@ Status OlapBlockDataConvertor::OlapColumnDataConvertorVarChar::convert_to_olap()
return Status::OK();
}
-Status OlapBlockDataConvertor::OlapColumnDataConvertorDate::convert_to_olap() {
- assert(_typed_column.column);
- const vectorized::ColumnVector<vectorized::Int64>* column_datetime = nullptr;
- if (_nullmap) {
- auto nullable_column =
- assert_cast<const vectorized::ColumnNullable*>(_typed_column.column.get());
- column_datetime = assert_cast<const vectorized::ColumnVector<vectorized::Int64>*>(
- nullable_column->get_nested_column_ptr().get());
+void OlapBlockDataConvertor::OlapColumnDataConvertorDate::set_source_column(
+ const ColumnWithTypeAndName& typed_column, size_t row_pos, size_t num_rows) {
+ OlapBlockDataConvertor::OlapColumnDataConvertorPaddedPODArray<uint24_t>::set_source_column(
+ typed_column, row_pos, num_rows);
+ if (is_date_v2(typed_column.type)) {
+ from_date_v2_ = true;
} else {
- column_datetime = assert_cast<const vectorized::ColumnVector<vectorized::Int64>*>(
- _typed_column.column.get());
+ from_date_v2_ = false;
}
+}
- assert(column_datetime);
+Status OlapBlockDataConvertor::OlapColumnDataConvertorDate::convert_to_olap() {
+ assert(_typed_column.column);
+ if (from_date_v2_) {
+ const vectorized::ColumnVector<vectorized::UInt32>* column_datetime = nullptr;
+ if (_nullmap) {
+ auto nullable_column =
+ assert_cast<const vectorized::ColumnNullable*>(_typed_column.column.get());
+ column_datetime = assert_cast<const vectorized::ColumnVector<vectorized::UInt32>*>(
+ nullable_column->get_nested_column_ptr().get());
+ } else {
+ column_datetime = assert_cast<const vectorized::ColumnVector<vectorized::UInt32>*>(
+ _typed_column.column.get());
+ }
- const VecDateTimeValue* datetime_cur =
- (const VecDateTimeValue*)(column_datetime->get_data().data()) + _row_pos;
- const VecDateTimeValue* datetime_end = datetime_cur + _num_rows;
- uint24_t* value = _values.data();
- if (_nullmap) {
- const UInt8* nullmap_cur = _nullmap + _row_pos;
- while (datetime_cur != datetime_end) {
- if (!*nullmap_cur) {
+ assert(column_datetime);
+
+ const DateV2Value* datetime_cur =
+ (const DateV2Value*)(column_datetime->get_data().data()) + _row_pos;
+ const DateV2Value* datetime_end = datetime_cur + _num_rows;
+ uint24_t* value = _values.data();
+ if (_nullmap) {
+ const UInt8* nullmap_cur = _nullmap + _row_pos;
+ while (datetime_cur != datetime_end) {
+ if (!*nullmap_cur) {
+ *value = datetime_cur->to_olap_date();
+ } else {
+ // do nothing
+ }
+ ++value;
+ ++datetime_cur;
+ ++nullmap_cur;
+ }
+ assert(nullmap_cur == _nullmap + _row_pos + _num_rows &&
+ value == _values.get_end_ptr());
+ } else {
+ while (datetime_cur != datetime_end) {
*value = datetime_cur->to_olap_date();
- } else {
- // do nothing
+ ++value;
+ ++datetime_cur;
}
- ++value;
- ++datetime_cur;
- ++nullmap_cur;
+ assert(value == _values.get_end_ptr());
}
- assert(nullmap_cur == _nullmap + _row_pos + _num_rows && value == _values.get_end_ptr());
+ return Status::OK();
} else {
- while (datetime_cur != datetime_end) {
- *value = datetime_cur->to_olap_date();
- ++value;
- ++datetime_cur;
+ const vectorized::ColumnVector<vectorized::Int64>* column_datetime = nullptr;
+ if (_nullmap) {
+ auto nullable_column =
+ assert_cast<const vectorized::ColumnNullable*>(_typed_column.column.get());
+ column_datetime = assert_cast<const vectorized::ColumnVector<vectorized::Int64>*>(
+ nullable_column->get_nested_column_ptr().get());
+ } else {
+ column_datetime = assert_cast<const vectorized::ColumnVector<vectorized::Int64>*>(
+ _typed_column.column.get());
}
- assert(value == _values.get_end_ptr());
+
+ assert(column_datetime);
+
+ const VecDateTimeValue* datetime_cur =
+ (const VecDateTimeValue*)(column_datetime->get_data().data()) + _row_pos;
+ const VecDateTimeValue* datetime_end = datetime_cur + _num_rows;
+ uint24_t* value = _values.data();
+ if (_nullmap) {
+ const UInt8* nullmap_cur = _nullmap + _row_pos;
+ while (datetime_cur != datetime_end) {
+ if (!*nullmap_cur) {
+ *value = datetime_cur->to_olap_date();
+ } else {
+ // do nothing
+ }
+ ++value;
+ ++datetime_cur;
+ ++nullmap_cur;
+ }
+ assert(nullmap_cur == _nullmap + _row_pos + _num_rows &&
+ value == _values.get_end_ptr());
+ } else {
+ while (datetime_cur != datetime_end) {
+ *value = datetime_cur->to_olap_date();
+ ++value;
+ ++datetime_cur;
+ }
+ assert(value == _values.get_end_ptr());
+ }
+ return Status::OK();
}
- return Status::OK();
}
Status OlapBlockDataConvertor::OlapColumnDataConvertorDateTime::convert_to_olap() {
diff --git a/be/src/vec/olap/olap_data_convertor.h b/be/src/vec/olap/olap_data_convertor.h
index 2cf0f79dc3..97bf3a7532 100644
--- a/be/src/vec/olap/olap_data_convertor.h
+++ b/be/src/vec/olap/olap_data_convertor.h
@@ -192,7 +192,12 @@ private:
class OlapColumnDataConvertorDate : public OlapColumnDataConvertorPaddedPODArray<uint24_t> {
public:
+ void set_source_column(const ColumnWithTypeAndName& typed_column, size_t row_pos,
+ size_t num_rows) override;
Status convert_to_olap() override;
+
+ private:
+ bool from_date_v2_;
};
class OlapColumnDataConvertorDateTime : public OlapColumnDataConvertorPaddedPODArray<uint64_t> {
@@ -245,6 +250,96 @@ private:
const T* _values = nullptr;
};
+ class OlapColumnDataConvertorDateV2 : public OlapColumnDataConvertorBase {
+ public:
+ OlapColumnDataConvertorDateV2() = default;
+ ~OlapColumnDataConvertorDateV2() override = default;
+
+ void set_source_column(const ColumnWithTypeAndName& typed_column, size_t row_pos,
+ size_t num_rows) override {
+ OlapColumnDataConvertorBase::set_source_column(typed_column, row_pos, num_rows);
+ if (is_date(typed_column.type)) {
+ from_date_to_date_v2_ = true;
+ } else {
+ from_date_to_date_v2_ = false;
+ }
+ }
+
+ const void* get_data() const override { return values_; }
+
+ const void* get_data_at(size_t offset) const override {
+ assert(offset < _num_rows);
+ UInt8 null_flag = 0;
+ if (_nullmap) {
+ null_flag = _nullmap[offset];
+ }
+ return null_flag ? nullptr : values_ + offset;
+ }
+
+ Status convert_to_olap() override {
+ if (UNLIKELY(from_date_to_date_v2_)) {
+ const vectorized::ColumnVector<vectorized::Int64>* column_datetime = nullptr;
+ if (_nullmap) {
+ auto nullable_column = assert_cast<const vectorized::ColumnNullable*>(
+ _typed_column.column.get());
+ column_datetime =
+ assert_cast<const vectorized::ColumnVector<vectorized::Int64>*>(
+ nullable_column->get_nested_column_ptr().get());
+ } else {
+ column_datetime =
+ assert_cast<const vectorized::ColumnVector<vectorized::Int64>*>(
+ _typed_column.column.get());
+ }
+
+ assert(column_datetime);
+
+ const VecDateTimeValue* datetime_cur =
+ (const VecDateTimeValue*)(column_datetime->get_data().data()) + _row_pos;
+ const VecDateTimeValue* datetime_end = datetime_cur + _num_rows;
+ uint32_t* value = const_cast<uint32_t*>(values_);
+ if (_nullmap) {
+ const UInt8* nullmap_cur = _nullmap + _row_pos;
+ while (datetime_cur != datetime_end) {
+ if (!*nullmap_cur) {
+ *value = datetime_cur->to_date_v2();
+ } else {
+ // do nothing
+ }
+ ++value;
+ ++datetime_cur;
+ ++nullmap_cur;
+ }
+ } else {
+ while (datetime_cur != datetime_end) {
+ *value = datetime_cur->to_olap_date();
+ ++value;
+ ++datetime_cur;
+ }
+ }
+ return Status::OK();
+ } else {
+ const vectorized::ColumnVector<uint32>* column_data = nullptr;
+ if (_nullmap) {
+ auto nullable_column = assert_cast<const vectorized::ColumnNullable*>(
+ _typed_column.column.get());
+ column_data = assert_cast<const vectorized::ColumnVector<uint32>*>(
+ nullable_column->get_nested_column_ptr().get());
+ } else {
+ column_data = assert_cast<const vectorized::ColumnVector<uint32>*>(
+ _typed_column.column.get());
+ }
+
+ assert(column_data);
+ values_ = (const uint32*)(column_data->get_data().data()) + _row_pos;
+ return Status::OK();
+ }
+ }
+
+ private:
+ const uint32_t* values_ = nullptr;
+ bool from_date_to_date_v2_;
+ };
+
class OlapColumnDataConvertorArray
: public OlapColumnDataConvertorPaddedPODArray<CollectionValue> {
public:
diff --git a/be/src/vec/runtime/vdatetime_value.cpp b/be/src/vec/runtime/vdatetime_value.cpp
index 845ed46ac9..d7d8179ba4 100644
--- a/be/src/vec/runtime/vdatetime_value.cpp
+++ b/be/src/vec/runtime/vdatetime_value.cpp
@@ -44,16 +44,14 @@ uint8_t mysql_week_mode(uint32_t mode) {
return mode;
}
-static bool is_leap(uint32_t year) {
+bool is_leap(uint32_t year) {
return ((year % 4) == 0) && ((year % 100 != 0) || ((year % 400) == 0 && year));
}
-static uint32_t calc_days_in_year(uint32_t year) {
+uint32_t calc_days_in_year(uint32_t year) {
return is_leap(year) ? 366 : 365;
}
-RE2 VecDateTimeValue::time_zone_offset_format_reg("^[+-]{1}\\d{2}\\:\\d{2}$");
-
bool VecDateTimeValue::check_range(uint32_t year, uint32_t month, uint32_t day, uint32_t hour,
uint32_t minute, uint32_t second, uint16_t type) {
bool time = hour > (type == TIME_TIME ? TIME_MAX_HOUR : 23) || minute > 59 || second > 59;
@@ -480,30 +478,6 @@ bool VecDateTimeValue::from_date_daynr(uint64_t daynr) {
return true;
}
-// Following code is stolen from MySQL.
-uint64_t VecDateTimeValue::calc_daynr(uint32_t year, uint32_t month, uint32_t day) {
- uint64_t delsum = 0;
- int y = year;
-
- if (year == 0 && month == 0) {
- return 0;
- }
-
- /* Cast to int to be able to handle month == 0 */
- delsum = 365 * y + 31 * (month - 1) + day;
- if (month <= 2) {
- // No leap year
- y--;
- } else {
- // This is great!!!
- // 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12
- // 0, 0, 3, 3, 4, 4, 5, 5, 5, 6, 7, 8
- delsum -= (month * 4 + 23) / 10;
- }
- // Every 400 year has 97 leap year, 100, 200, 300 are not leap year.
- return delsum + y / 4 - y / 100 + y / 400;
-}
-
static char* int_to_str(uint64_t val, char* to) {
char buf[64];
char* ptr = buf;
@@ -931,8 +905,44 @@ uint32_t VecDateTimeValue::year_week(uint8_t mode) const {
return year * 100 + week;
}
-uint8_t VecDateTimeValue::calc_weekday(uint64_t day_nr, bool is_sunday_first_day) {
- return (day_nr + 5L + (is_sunday_first_day ? 1L : 0L)) % 7;
+bool VecDateTimeValue::operator>=(const DateV2Value& other) const {
+ int64_t ts1;
+ int64_t ts2;
+ this->unix_timestamp(&ts1, TimezoneUtils::default_time_zone);
+ other.unix_timestamp(&ts2, TimezoneUtils::default_time_zone);
+ return ts1 >= ts2;
+}
+
+bool VecDateTimeValue::operator<=(const DateV2Value& other) const {
+ int64_t ts1;
+ int64_t ts2;
+ this->unix_timestamp(&ts1, TimezoneUtils::default_time_zone);
+ other.unix_timestamp(&ts2, TimezoneUtils::default_time_zone);
+ return ts1 <= ts2;
+}
+
+bool VecDateTimeValue::operator>(const DateV2Value& other) const {
+ int64_t ts1;
+ int64_t ts2;
+ this->unix_timestamp(&ts1, TimezoneUtils::default_time_zone);
+ other.unix_timestamp(&ts2, TimezoneUtils::default_time_zone);
+ return ts1 > ts2;
+}
+
+bool VecDateTimeValue::operator<(const DateV2Value& other) const {
+ int64_t ts1;
+ int64_t ts2;
+ this->unix_timestamp(&ts1, TimezoneUtils::default_time_zone);
+ other.unix_timestamp(&ts2, TimezoneUtils::default_time_zone);
+ return ts1 < ts2;
+}
+
+bool VecDateTimeValue::operator==(const DateV2Value& other) const {
+ int64_t ts1;
+ int64_t ts2;
+ this->unix_timestamp(&ts1, TimezoneUtils::default_time_zone);
+ other.unix_timestamp(&ts2, TimezoneUtils::default_time_zone);
+ return ts1 == ts2;
}
// TODO(zhaochun): Think endptr is NULL
@@ -1430,6 +1440,12 @@ bool VecDateTimeValue::from_date_format_str(const char* format, int format_len,
return check_range_and_set_time(year, month, day, hour, minute, second, _type);
}
+int64_t VecDateTimeValue::second_diff(const DateV2Value& rhs) const {
+ int day_diff = daynr() - rhs.daynr();
+ int time_diff = hour() * 3600 + minute() * 60 + second();
+ return day_diff * 3600 * 24 + time_diff;
+}
+
bool VecDateTimeValue::date_add_interval(const TimeInterval& interval, TimeUnit unit) {
if (!is_valid_date()) return false;
@@ -1585,6 +1601,12 @@ void VecDateTimeValue::set_time(uint32_t year, uint32_t month, uint32_t day, uin
_second = second;
}
+void VecDateTimeValue::create_from_date_v2(DateV2Value& value, TimeType type) {
+ this->set_time(value.year(), value.month(), value.day(), 0, 0, 0);
+ this->set_type(type);
+ this->_neg = 0;
+}
+
void VecDateTimeValue::convert_vec_dt_to_dt(
doris::DateTimeValue* dt) { //use convert VecDateTimeValue to DateTimeValue
dt->_neg = this->_neg;
@@ -1622,8 +1644,1072 @@ std::size_t operator-(const VecDateTimeValue& v1, const VecDateTimeValue& v2) {
return v1.daynr() - v2.daynr();
}
+std::size_t operator-(const DateV2Value& v1, const VecDateTimeValue& v2) {
+ return v1.daynr() - v2.daynr();
+}
+
+std::size_t operator-(const VecDateTimeValue& v1, const DateV2Value& v2) {
+ return v1.daynr() - v2.daynr();
+}
+
std::size_t hash_value(VecDateTimeValue const& value) {
return HashUtil::hash(&value, sizeof(VecDateTimeValue), 0);
}
+bool DateV2Value::is_invalid(uint32_t year, uint32_t month, uint32_t day) {
+ if (month == 2 && day == 29 && is_leap(year)) return false;
+ if (year < MIN_YEAR || year > MAX_YEAR || month == 0 || month > 12 ||
+ day > s_days_in_month[month] || day == 0) {
+ return true;
+ }
+ return false;
+}
+
+// The interval format is that with no delimiters
+// YYYY-MM-DD HH-MM-DD.FFFFFF AM in default format
+// 0 1 2 3 4 5 6 7
+bool DateV2Value::from_date_str(const char* date_str, int len) {
+ const char* ptr = date_str;
+ const char* end = date_str + len;
+ // ONLY 2, 6 can follow by a sapce
+ const static int allow_space_mask = 4 | 64;
+ const static int MAX_DATE_PARTS = 3;
+ uint32_t date_val[MAX_DATE_PARTS];
+ int32_t date_len[MAX_DATE_PARTS] = {0, 0, 0};
+
+ // Skip space character
+ while (ptr < end && isspace(*ptr)) {
+ ptr++;
+ }
+ if (ptr == end || !isdigit(*ptr)) {
+ return false;
+ }
+ // Fix year length
+ const char* pos = ptr;
+ while (pos < end && (isdigit(*pos) || *pos == 'T')) {
+ pos++;
+ }
+ int year_len = 4;
+ int digits = pos - ptr;
+ bool is_interval_format = false;
+
+ // Compatible with MySQL. Shit!!!
+ // For YYYYMMDD/YYYYMMDDHHMMSS is 4 digits years
+ if (pos == end || *pos == '.') {
+ if (digits == 4 || digits == 8 || digits >= 14) {
+ year_len = 4;
+ } else {
+ year_len = 2;
+ }
+ is_interval_format = true;
+ }
+
+ int field_idx = 0;
+ int field_len = year_len;
+ while (ptr < end && isdigit(*ptr) && field_idx < MAX_DATE_PARTS) {
+ const char* start = ptr;
+ int temp_val = 0;
+ bool scan_to_delim = (!is_interval_format) && (field_idx != 6);
+ while (ptr < end && isdigit(*ptr) && (scan_to_delim || field_len--)) {
+ temp_val = temp_val * 10 + (*ptr++ - '0');
+ }
+ // Imposible
+ if (temp_val > 999999L) {
+ return false;
+ }
+ date_val[field_idx] = temp_val;
+ date_len[field_idx] = ptr - start;
+ field_len = 2;
+
+ if (ptr == end) {
+ field_idx++;
+ break;
+ }
+ if (field_idx == 2 && *ptr == 'T') {
+ // YYYYMMDDTHHMMDD, skip 'T' and continue
+ ptr++;
+ field_idx++;
+ continue;
+ }
+
+ // Second part
+ if (field_idx == 5) {
+ if (*ptr == '.') {
+ ptr++;
+ field_len = 6;
+ } else if (isdigit(*ptr)) {
+ field_idx++;
+ break;
+ }
+ field_idx++;
+ continue;
+ }
+ // escape separator
+ while (ptr < end && (ispunct(*ptr) || isspace(*ptr))) {
+ if (isspace(*ptr)) {
+ if (((1 << field_idx) & allow_space_mask) == 0) {
+ return false;
+ }
+ }
+ ptr++;
+ }
+ field_idx++;
+ }
+ int num_field = field_idx;
+ if (!is_interval_format) {
+ year_len = date_len[0];
+ }
+ for (; field_idx < MAX_DATE_PARTS; ++field_idx) {
+ date_len[field_idx] = 0;
+ date_val[field_idx] = 0;
+ }
+
+ if (year_len == 2) {
+ if (date_val[0] < YY_PART_YEAR) {
+ date_val[0] += 2000;
+ } else {
+ date_val[0] += 1900;
+ }
+ }
+
+ if (num_field > 3) {
+ return false;
+ }
+ return check_range_and_set_time(date_val[0], date_val[1], date_val[2]);
+}
+
+void DateV2Value::set_zero(int type) {
+ date_v2_value_.month_ = 0;
+ date_v2_value_.year_ = 0;
+ date_v2_value_.day_ = 0;
+}
+
+// this method is exactly same as fromDateFormatStr() in DateLiteral.java in FE
+// change this method should also change that.
+bool DateV2Value::from_date_format_str(const char* format, int format_len, const char* value,
+ int value_len, const char** sub_val_end) {
+ const char* ptr = format;
+ const char* end = format + format_len;
+ const char* val = value;
+ const char* val_end = value + value_len;
+
+ int day_part = 0;
+ int weekday = -1;
+ int yearday = -1;
+ int week_num = -1;
+
+ bool strict_week_number = false;
+ bool sunday_first = false;
+ bool strict_week_number_year_type = false;
+ int strict_week_number_year = -1;
+ bool usa_time = false;
+
+ // TODO: should we process [hour minute second] for datev2 type here?
+ auto [year, month, day, hour, minute, second] = std::tuple {0, 0, 0, 0, 0, 0};
+ while (ptr < end && val < val_end) {
+ // Skip space character
+ while (val < val_end && isspace(*val)) {
+ val++;
+ }
+ if (val >= val_end) {
+ break;
+ }
+ // Check switch
+ if (*ptr == '%' && ptr + 1 < end) {
+ const char* tmp = NULL;
+ int64_t int_value = 0;
+ ptr++;
+ switch (*ptr++) {
+ // Year
+ case 'y':
+ // Year, numeric (two digits)
+ tmp = val + min(2, val_end - val);
+ if (!str_to_int64(val, &tmp, &int_value)) {
+ return false;
+ }
+ int_value += int_value >= 70 ? 1900 : 2000;
+ year = int_value;
+ val = tmp;
+ break;
+ case 'Y':
+ // Year, numeric, four digits
+ tmp = val + min(4, val_end - val);
+ if (!str_to_int64(val, &tmp, &int_value)) {
+ return false;
+ }
+ if (tmp - val <= 2) {
+ int_value += int_value >= 70 ? 1900 : 2000;
+ }
+ year = int_value;
+ val = tmp;
+ break;
+ // Month
+ case 'm':
+ case 'c':
+ tmp = val + min(2, val_end - val);
+ if (!str_to_int64(val, &tmp, &int_value)) {
+ return false;
+ }
+ month = int_value;
+ val = tmp;
+ break;
+ case 'M':
+ int_value = check_word(const_cast<const char**>(s_month_name), val, val_end, &val);
+ if (int_value < 0) {
+ return false;
+ }
+ month = int_value;
+ break;
+ case 'b':
+ int_value = check_word(s_ab_month_name, val, val_end, &val);
+ if (int_value < 0) {
+ return false;
+ }
+ month = int_value;
+ break;
+ // Day
+ case 'd':
+ case 'e':
+ tmp = val + min(2, val_end - val);
+ if (!str_to_int64(val, &tmp, &int_value)) {
+ return false;
+ }
+ day = int_value;
+ val = tmp;
+ break;
+ case 'D':
+ tmp = val + min(2, val_end - val);
+ if (!str_to_int64(val, &tmp, &int_value)) {
+ return false;
+ }
+ day = int_value;
+ val = tmp + min(2, val_end - tmp);
+ break;
+ // Hour
+ case 'h':
+ case 'I':
+ case 'l':
+ usa_time = true;
+ // Fall through
+ case 'k':
+ case 'H':
+ tmp = val + min(2, val_end - val);
+ if (!str_to_int64(val, &tmp, &int_value)) {
+ return false;
+ }
+ hour = int_value;
+ val = tmp;
+ break;
+ // Minute
+ case 'i':
+ tmp = val + min(2, val_end - val);
+ if (!str_to_int64(val, &tmp, &int_value)) {
+ return false;
+ }
+ minute = int_value;
+ val = tmp;
+ break;
+ // Second
+ case 's':
+ case 'S':
+ tmp = val + min(2, val_end - val);
+ if (!str_to_int64(val, &tmp, &int_value)) {
+ return false;
+ }
+ second = int_value;
+ val = tmp;
+ break;
+ // Micro second
+ case 'f':
+ break;
+ // AM/PM
+ case 'p':
+ if ((val_end - val) < 2 || toupper(*(val + 1)) != 'M' || !usa_time) {
+ return false;
+ }
+ if (toupper(*val) == 'P') {
+ // PM
+ day_part = 12;
+ }
+ val += 2;
+ break;
+ // Weekday
+ case 'W':
+ int_value = check_word(const_cast<const char**>(s_day_name), val, val_end, &val);
+ if (int_value < 0) {
+ return false;
+ }
+ int_value++;
+ weekday = int_value;
+ break;
+ case 'a':
+ int_value = check_word(s_ab_day_name, val, val_end, &val);
+ if (int_value < 0) {
+ return false;
+ }
+ int_value++;
+ weekday = int_value;
+ break;
+ case 'w':
+ tmp = val + min(1, val_end - val);
+ if (!str_to_int64(val, &tmp, &int_value)) {
+ return false;
+ }
+ if (int_value >= 7) {
+ return false;
+ }
+ if (int_value == 0) {
+ int_value = 7;
+ }
+ weekday = int_value;
+ val = tmp;
+ break;
+ case 'j':
+ tmp = val + min(3, val_end - val);
+ if (!str_to_int64(val, &tmp, &int_value)) {
+ return false;
+ }
+ yearday = int_value;
+ val = tmp;
+ break;
+ case 'u':
+ case 'v':
+ case 'U':
+ case 'V':
+ sunday_first = (*(ptr - 1) == 'U' || *(ptr - 1) == 'V');
+ // Used to check if there is %x or %X
+ strict_week_number = (*(ptr - 1) == 'V' || *(ptr - 1) == 'v');
+ tmp = val + min(2, val_end - val);
+ if (!str_to_int64(val, &tmp, &int_value)) {
+ return false;
+ }
+ week_num = int_value;
+ if (week_num > 53 || (strict_week_number && week_num == 0)) {
+ return false;
+ }
+ val = tmp;
+ break;
+ // strict week number, must be used with %V or %v
+ case 'x':
+ case 'X':
+ strict_week_number_year_type = (*(ptr - 1) == 'X');
+ tmp = val + min(4, val_end - val);
+ if (!str_to_int64(val, &tmp, &int_value)) {
+ return false;
+ }
+ strict_week_number_year = int_value;
+ val = tmp;
+ break;
+ case 'r':
+ if (!from_date_format_str("%I:%i:%S %p", 11, val, val_end - val, &tmp)) {
+ return false;
+ }
+ val = tmp;
+ break;
+ case 'T':
+ if (!from_date_format_str("%H:%i:%S", 8, val, val_end - val, &tmp)) {
+ return false;
+ }
+ val = tmp;
+ break;
+ case '.':
+ while (val < val_end && ispunct(*val)) {
+ val++;
+ }
+ break;
+ case '@':
+ while (val < val_end && isalpha(*val)) {
+ val++;
+ }
+ break;
+ case '#':
+ while (val < val_end && isdigit(*val)) {
+ val++;
+ }
+ break;
+ case '%': // %%, escape the %
+ if ('%' != *val) {
+ return false;
+ }
+ val++;
+ break;
+ default:
+ return false;
+ }
+ } else if (!isspace(*ptr)) {
+ if (*ptr != *val) {
+ return false;
+ }
+ ptr++;
+ val++;
+ } else {
+ ptr++;
+ }
+ }
+
+ // continue to iterate pattern if has
+ // to find out if it has time part.
+ while (ptr < end) {
+ if (*ptr == '%' && ptr + 1 < end) {
+ ptr++;
+ switch (*ptr++) {
+ case 'H':
+ case 'h':
+ case 'I':
+ case 'i':
+ case 'k':
+ case 'l':
+ case 'r':
+ case 's':
+ case 'S':
+ case 'p':
+ case 'T':
+ break;
+ default:
+ break;
+ }
+ } else {
+ ptr++;
+ }
+ }
+
+ if (usa_time) {
+ if (hour > 12 || hour < 1) {
+ return false;
+ }
+ hour = (hour % 12) + day_part;
+ }
+ if (sub_val_end) {
+ *sub_val_end = val;
+ }
+
+ // Year day
+ if (yearday > 0) {
+ uint64_t days = calc_daynr(year, 1, 1) + yearday - 1;
+ if (!get_date_from_daynr(days)) {
+ return false;
+ }
+ }
+ // weekday
+ if (week_num >= 0 && weekday > 0) {
+ // Check
+ if ((strict_week_number &&
+ (strict_week_number_year < 0 || strict_week_number_year_type != sunday_first)) ||
+ (!strict_week_number && strict_week_number_year >= 0)) {
+ return false;
+ }
+ uint64_t days = calc_daynr(strict_week_number ? strict_week_number_year : year, 1, 1);
+
+ uint8_t weekday_b = calc_weekday(days, sunday_first);
+
+ if (sunday_first) {
+ days += ((weekday_b == 0) ? 0 : 7) - weekday_b + (week_num - 1) * 7 + weekday % 7;
+ } else {
+ days += ((weekday_b <= 3) ? 0 : 7) - weekday_b + (week_num - 1) * 7 + weekday - 1;
+ }
+ if (!get_date_from_daynr(days)) {
+ return false;
+ }
+ }
+ return check_range_and_set_time(year, month, day);
+}
+
+int32_t DateV2Value::to_buffer(char* buffer) const {
+ char* start = buffer;
+ uint32_t temp;
+ // Year
+ temp = date_v2_value_.year_ / 100;
+ *buffer++ = (char)('0' + (temp / 10));
+ *buffer++ = (char)('0' + (temp % 10));
+ temp = date_v2_value_.year_ % 100;
+ *buffer++ = (char)('0' + (temp / 10));
+ *buffer++ = (char)('0' + (temp % 10));
+ *buffer++ = '-';
+ // Month
+ *buffer++ = (char)('0' + (date_v2_value_.month_ / 10));
+ *buffer++ = (char)('0' + (date_v2_value_.month_ % 10));
+ *buffer++ = '-';
+ // Day
+ *buffer++ = (char)('0' + (date_v2_value_.day_ / 10));
+ *buffer++ = (char)('0' + (date_v2_value_.day_ % 10));
+ return buffer - start;
+}
+
+char* DateV2Value::to_string(char* to) const {
+ int len = to_buffer(to);
+ *(to + len) = '\0';
+ return to + len + 1;
+}
+
+uint32_t DateV2Value::to_date_uint32() const {
+ return int_val_;
+}
+
+uint32_t DateV2Value::set_date_uint32(uint32_t int_val) {
+ union DateV2UInt32Union {
+ doris::vectorized::DateV2Value dt;
+ uint32_t ui32;
+ ~DateV2UInt32Union() {}
+ };
+ DateV2UInt32Union conv = {.ui32 = int_val};
+ if (is_invalid(conv.dt.year(), conv.dt.month(), conv.dt.day())) {
+ return 0;
+ }
+ this->set_time(conv.dt.year(), conv.dt.month(), conv.dt.day());
+
+ return int_val;
+}
+
+uint8_t DateV2Value::week(uint8_t mode) const {
+ uint16_t year = 0;
+ return calc_week(this->daynr(), this->year(), this->month(), this->day(), mode, &year);
+}
+
+uint32_t DateV2Value::year_week(uint8_t mode) const {
+ uint16_t year = 0;
+ // The range of the week in the year_week is 1-53, so the mode WEEK_YEAR is always true.
+ uint8_t week = calc_week(this->daynr(), this->year(), this->month(), this->day(), mode, &year);
+ // When the mode WEEK_FIRST_WEEKDAY is not set,
+ // the week in which the last three days of the year fall may belong to the following year.
+ if (week == 53 && day() >= 29 && !(mode & 4)) {
+ uint8_t monday_first = mode & WEEK_MONDAY_FIRST;
+ uint64_t daynr_of_last_day = calc_daynr(this->year(), 12, 31);
+ uint8_t weekday_of_last_day = calc_weekday(daynr_of_last_day, !monday_first);
+
+ if (weekday_of_last_day - monday_first < 2) {
+ ++year;
+ week = 1;
+ }
+ }
+ return year * 100 + week;
+}
+
+bool DateV2Value::get_date_from_daynr(uint64_t daynr) {
+ if (daynr <= 0 || daynr > DATE_MAX_DAYNR) {
+ return false;
+ }
+
+ auto [year, month, day] = std::tuple {0, 0, 0};
+ year = daynr / 365;
+ uint32_t days_befor_year = 0;
+ while (daynr < (days_befor_year = calc_daynr(year, 1, 1))) {
+ year--;
+ }
+ uint32_t days_of_year = daynr - days_befor_year + 1;
+ int leap_day = 0;
+ if (is_leap(year)) {
+ if (days_of_year > 31 + 28) {
+ days_of_year--;
+ if (days_of_year == 31 + 28) {
+ leap_day = 1;
+ }
+ }
+ }
+ month = 1;
+ while (days_of_year > s_days_in_month[month]) {
+ days_of_year -= s_days_in_month[month];
+ month++;
+ }
+ day = days_of_year + leap_day;
+
+ if (is_invalid(year, month, day)) {
+ return false;
+ }
+ set_time(year, month, day);
+ return true;
+}
+
+bool DateV2Value::date_add_interval(const TimeInterval& interval, TimeUnit unit) {
+ if (!is_valid_date()) return false;
+
+ switch (unit) {
+ case SECOND:
+ case MINUTE:
+ case HOUR:
+ case SECOND_MICROSECOND:
+ case MINUTE_MICROSECOND:
+ case MINUTE_SECOND:
+ case HOUR_MICROSECOND:
+ case HOUR_SECOND:
+ case HOUR_MINUTE:
+ case DAY_MICROSECOND:
+ case DAY_SECOND:
+ case DAY_MINUTE:
+ case DAY_HOUR:
+ case DAY:
+ case WEEK: {
+ // This only change day information, not change second information
+ uint32_t day_nr = daynr() + interval.day;
+ if (!get_date_from_daynr(day_nr)) {
+ return false;
+ }
+ break;
+ }
+ case YEAR: {
+ // This only change year information
+ date_v2_value_.year_ += interval.year;
+ if (date_v2_value_.year_ > 9999) {
+ return false;
+ }
+ if (date_v2_value_.month_ == 2 && date_v2_value_.day_ == 29 &&
+ !is_leap(date_v2_value_.year_)) {
+ date_v2_value_.day_ = 28;
+ }
+ break;
+ }
+ case MONTH:
+ case QUARTER:
+ case YEAR_MONTH: {
+ // This will change month and year information, maybe date.
+ int64_t months = date_v2_value_.year_ * 12 + date_v2_value_.month_ - 1 +
+ 12 * interval.year + interval.month;
+ date_v2_value_.year_ = months / 12;
+ if (date_v2_value_.year_ > 9999) {
+ return false;
+ }
+ date_v2_value_.month_ = (months % 12) + 1;
+ if (date_v2_value_.day_ > s_days_in_month[date_v2_value_.month_]) {
+ date_v2_value_.day_ = s_days_in_month[date_v2_value_.month_];
+ if (date_v2_value_.month_ == 2 && is_leap(date_v2_value_.year_)) {
+ date_v2_value_.day_++;
+ }
+ }
+ break;
+ }
+ }
+ return true;
+}
+
+bool DateV2Value::unix_timestamp(int64_t* timestamp, const std::string& timezone) const {
+ cctz::time_zone ctz;
+ if (!TimezoneUtils::find_cctz_time_zone(timezone, ctz)) {
+ return false;
+ }
+ return unix_timestamp(timestamp, ctz);
+}
+
+bool DateV2Value::unix_timestamp(int64_t* timestamp, const cctz::time_zone& ctz) const {
+ const auto tp = cctz::convert(cctz::civil_second(date_v2_value_.year_, date_v2_value_.month_,
+ date_v2_value_.day_, 0, 0, 0),
+ ctz);
+ *timestamp = tp.time_since_epoch().count();
+ return true;
+}
+
+bool DateV2Value::from_unixtime(int64_t timestamp, const std::string& timezone) {
+ cctz::time_zone ctz;
+ if (!TimezoneUtils::find_cctz_time_zone(timezone, ctz)) {
+ return false;
+ }
+ return from_unixtime(timestamp, ctz);
+}
+
+bool DateV2Value::from_unixtime(int64_t timestamp, const cctz::time_zone& ctz) {
+ static const cctz::time_point<cctz::sys_seconds> epoch =
+ std::chrono::time_point_cast<cctz::sys_seconds>(
+ std::chrono::system_clock::from_time_t(0));
+ cctz::time_point<cctz::sys_seconds> t = epoch + cctz::seconds(timestamp);
+
+ const auto tp = cctz::convert(t, ctz);
+
+ set_time(tp.year(), tp.month(), tp.day());
+ return true;
+}
+
+const char* DateV2Value::month_name() const {
+ if (date_v2_value_.month_ < 1 || date_v2_value_.month_ > 12) {
+ return nullptr;
+ }
+ return s_month_name[date_v2_value_.month_];
+}
+
+const char* DateV2Value::day_name() const {
+ int day = weekday();
+ if (day < 0 || day >= 7) {
+ return nullptr;
+ }
+ return s_day_name[day];
+}
+
+void DateV2Value::set_time(uint16_t year, uint8_t month, uint8_t day) {
+ date_v2_value_.year_ = year;
+ date_v2_value_.month_ = month;
+ date_v2_value_.day_ = day;
+}
+
+void DateV2Value::convert_date_v2_to_dt(
+ doris::DateTimeValue* dt) { //use convert VecDateTimeValue to DateTimeValue
+ dt->_neg = 0;
+ dt->_type = TIME_DATE;
+ dt->_hour = 0;
+ dt->_minute = 0;
+ dt->_second = 0;
+ dt->_year = date_v2_value_.year_;
+ dt->_month = date_v2_value_.month_;
+ dt->_day = date_v2_value_.day_;
+ dt->_microsecond = 0;
+}
+
+void DateV2Value::convert_dt_to_date_v2(doris::DateTimeValue* dt) {
+ date_v2_value_.year_ = dt->_year;
+ date_v2_value_.month_ = dt->_month;
+ date_v2_value_.day_ = dt->_day;
+}
+
+bool DateV2Value::to_format_string(const char* format, int len, char* to) const {
+ char buf[64];
+ char* pos = nullptr;
+ const char* ptr = format;
+ const char* end = format + len;
+ char ch = '\0';
+
+ while (ptr < end) {
+ if (*ptr != '%' || (ptr + 1) == end) {
+ *to++ = *ptr++;
+ continue;
+ }
+ // Skip '%'
+ ptr++;
+ switch (ch = *ptr++) {
+ case 'a':
+ // Abbreviated weekday name
+ if (this->year() == 0 && this->month() == 0) {
+ return false;
+ }
+ to = append_string(s_ab_day_name[weekday()], to);
+ break;
+ case 'b':
+ // Abbreviated month name
+ if (this->month() == 0) {
+ return false;
+ }
+ to = append_string(s_ab_month_name[this->month()], to);
+ break;
+ case 'c':
+ // Month, numeric (0...12)
+ pos = int_to_str(this->month(), buf);
+ to = append_with_prefix(buf, pos - buf, '0', 1, to);
+ break;
+ case 'd':
+ // Day of month (00...31)
+ pos = int_to_str(this->day(), buf);
+ to = append_with_prefix(buf, pos - buf, '0', 2, to);
+ break;
+ case 'D':
+ // Day of the month with English suffix (0th, 1st, ...)
+ pos = int_to_str(this->day(), buf);
+ to = append_with_prefix(buf, pos - buf, '0', 1, to);
+ if (this->day() >= 10 && this->day() <= 19) {
+ to = append_string("th", to);
+ } else {
+ switch (this->day() % 10) {
+ case 1:
+ to = append_string("st", to);
+ break;
+ case 2:
+ to = append_string("nd", to);
+ break;
+ case 3:
+ to = append_string("rd", to);
+ break;
+ default:
+ to = append_string("th", to);
+ break;
+ }
+ }
+ break;
+ case 'e':
+ // Day of the month, numeric (0..31)
+ pos = int_to_str(this->day(), buf);
+ to = append_with_prefix(buf, pos - buf, '0', 1, to);
+ break;
+ case 'f':
+ // Microseconds (000000..999999)
+ pos = int_to_str(0, buf);
+ to = append_with_prefix(buf, pos - buf, '0', 6, to);
+ break;
+ case 'h':
+ case 'I':
+ // Hour (01..12)
+ pos = int_to_str(12, buf);
+ to = append_with_prefix(buf, pos - buf, '0', 2, to);
+ break;
+ case 'H':
+ // Hour (00..23)
+ pos = int_to_str(0, buf);
+ to = append_with_prefix(buf, pos - buf, '0', 2, to);
+ break;
+ case 'i':
+ // Minutes, numeric (00..59)
+ pos = int_to_str(0, buf);
+ to = append_with_prefix(buf, pos - buf, '0', 2, to);
+ break;
+ case 'j':
+ // Day of year (001..366)
+ pos = int_to_str(daynr() - calc_daynr(this->year(), 1, 1) + 1, buf);
+ to = append_with_prefix(buf, pos - buf, '0', 3, to);
+ break;
+ case 'k':
+ // Hour (0..23)
+ pos = int_to_str(0, buf);
+ to = append_with_prefix(buf, pos - buf, '0', 1, to);
+ break;
+ case 'l':
+ // Hour (1..12)
+ pos = int_to_str(12, buf);
+ to = append_with_prefix(buf, pos - buf, '0', 1, to);
+ break;
+ case 'm':
+ // Month, numeric (00..12)
+ pos = int_to_str(this->month(), buf);
+ to = append_with_prefix(buf, pos - buf, '0', 2, to);
+ break;
+ case 'M':
+ // Month name (January..December)
+ if (this->month() == 0) {
+ return false;
+ }
+ to = append_string(s_month_name[this->month()], to);
+ break;
+ case 'p':
+ // AM or PM
+ to = append_string("AM", to);
+ break;
+ case 'r': {
+ // Time, 12-hour (hh:mm:ss followed by AM or PM)
+ std::string time12_str("12:00:00 AM");
+ memcpy(to, time12_str.data(), time12_str.size());
+ to += time12_str.size();
+ break;
+ }
+ case 's':
+ case 'S':
+ // Seconds (00..59)
+ pos = int_to_str(0, buf);
+ to = append_with_prefix(buf, pos - buf, '0', 2, to);
+ break;
+ case 'T': {
+ // Time, 24-hour (hh:mm:ss)
+ std::string time24_str("00:00:00");
+ memcpy(to, time24_str.data(), time24_str.size());
+ to += time24_str.size();
+ break;
+ }
+ case 'u':
+ // Week (00..53), where Monday is the first day of the week;
+ // WEEK() mode 1
+ pos = int_to_str(week(mysql_week_mode(1)), buf);
+ to = append_with_prefix(buf, pos - buf, '0', 2, to);
+ break;
+ case 'U':
+ // Week (00..53), where Sunday is the first day of the week;
+ // WEEK() mode 0
+ pos = int_to_str(week(mysql_week_mode(0)), buf);
+ to = append_with_prefix(buf, pos - buf, '0', 2, to);
+ break;
+ case 'v':
+ // Week (01..53), where Monday is the first day of the week;
+ // WEEK() mode 3; used with %x
+ pos = int_to_str(week(mysql_week_mode(3)), buf);
+ to = append_with_prefix(buf, pos - buf, '0', 2, to);
+ break;
+ case 'V':
+ // Week (01..53), where Sunday is the first day of the week;
+ // WEEK() mode 2; used with %X
+ pos = int_to_str(week(mysql_week_mode(2)), buf);
+ to = append_with_prefix(buf, pos - buf, '0', 2, to);
+ break;
+ case 'w':
+ // Day of the week (0=Sunday..6=Saturday)
+ if (this->month() == 0 && this->year() == 0) {
+ return false;
+ }
+ pos = int_to_str(calc_weekday(daynr(), true), buf);
+ to = append_with_prefix(buf, pos - buf, '0', 1, to);
+ break;
+ case 'W':
+ // Weekday name (Sunday..Saturday)
+ to = append_string(s_day_name[weekday()], to);
+ break;
+ case 'x': {
+ // Year for the week, where Monday is the first day of the week,
+ // numeric, four digits; used with %v
+ uint16_t year = 0;
+ calc_week(this->daynr(), this->year(), this->month(), this->day(), mysql_week_mode(3),
+ &year);
+ pos = int_to_str(year, buf);
+ to = append_with_prefix(buf, pos - buf, '0', 4, to);
+ break;
+ }
+ case 'X': {
+ // Year for the week where Sunday is the first day of the week,
+ // numeric, four digits; used with %V
+ uint16_t year = 0;
+ calc_week(this->daynr(), this->year(), this->month(), this->day(), mysql_week_mode(3),
+ &year);
+ pos = int_to_str(year, buf);
+ to = append_with_prefix(buf, pos - buf, '0', 4, to);
+ break;
+ }
+ case 'y':
+ // Year, numeric (two digits)
+ pos = int_to_str(this->year() % 100, buf);
+ to = append_with_prefix(buf, pos - buf, '0', 2, to);
+ break;
+ case 'Y':
+ // Year, numeric, four digits
+ pos = int_to_str(this->year(), buf);
+ to = append_with_prefix(buf, pos - buf, '0', 4, to);
+ break;
+ default:
+ *to++ = ch;
+ break;
+ }
+ }
+ *to++ = '\0';
+ return true;
+}
+
+bool DateV2Value::from_date(uint32_t value) {
+ if (value < MIN_DATE_V2 || value > MAX_DATE_V2) {
+ return false;
+ }
+
+ return set_date_uint32(value);
+}
+
+int64_t DateV2Value::standardize_timevalue(int64_t value) {
+ if (value <= 0) {
+ return 0;
+ }
+ if (value >= 10000101000000L) {
+ // 9999-99-99 99:99:99
+ if (value > 99999999999999L) {
+ return 0;
+ }
+
+ // between 1000-01-01 00:00:00L and 9999-99-99 99:99:99
+ // all digits exist.
+ return value;
+ }
+ // 2000-01-01
+ if (value < 101) {
+ return 0;
+ }
+ // two digits year. 2000 ~ 2069
+ if (value <= (YY_PART_YEAR - 1) * 10000L + 1231L) {
+ return (value + 20000000L) * 1000000L;
+ }
+ // two digits year, invalid date
+ if (value < YY_PART_YEAR * 10000L + 101) {
+ return 0;
+ }
+ // two digits year. 1970 ~ 1999
+ if (value <= 991231L) {
+ return (value + 19000000L) * 1000000L;
+ }
+ // TODO(zhaochun): Don't allow year betwen 1000-01-01
+ if (value < 10000101) {
+ return 0;
+ }
+ // four digits years without hour.
+ if (value <= 99991231L) {
+ return value * 1000000L;
+ }
+ // below 0000-01-01
+ if (value < 101000000) {
+ return 0;
+ }
+
+ // below is with datetime, must have hh:mm:ss
+ // 2000 ~ 2069
+ if (value <= (YY_PART_YEAR - 1) * 10000000000L + 1231235959L) {
+ return value + 20000000000000L;
+ }
+ if (value < YY_PART_YEAR * 10000000000L + 101000000L) {
+ return 0;
+ }
+ // 1970 ~ 1999
+ if (value <= 991231235959L) {
+ return value + 19000000000000L;
+ }
+ return value;
+}
+
+bool DateV2Value::from_date_int64(int64_t value) {
+ value = standardize_timevalue(value);
+ if (value <= 0) {
+ return false;
+ }
+ uint64_t date = value / 1000000;
+
+ auto [year, month, day] = std::tuple {0, 0, 0};
+ year = date / 10000;
+ date %= 10000;
+ month = date / 100;
+ day = date % 100;
+
+ return check_range_and_set_time(year, month, day);
+}
+
+uint8_t DateV2Value::calc_week(const uint32_t& day_nr, const uint16_t& year, const uint8_t& month,
+ const uint8_t& day, uint8_t mode, uint16_t* to_year) {
+ bool monday_first = mode & WEEK_MONDAY_FIRST;
+ bool week_year = mode & WEEK_YEAR;
+ bool first_weekday = mode & WEEK_FIRST_WEEKDAY;
+ uint64_t daynr_first_day = calc_daynr(year, 1, 1);
+ uint8_t weekday_first_day = calc_weekday(daynr_first_day, !monday_first);
+
+ int days = 0;
+ *to_year = year;
+
+ // Check wether the first days of this year belongs to last year
+ if (month == 1 && day <= (7 - weekday_first_day)) {
+ if (!week_year && ((first_weekday && weekday_first_day != 0) ||
+ (!first_weekday && weekday_first_day > 3))) {
+ return 0;
+ }
+ (*to_year)--;
+ week_year = true;
+ daynr_first_day -= (days = calc_days_in_year(*to_year));
+ weekday_first_day = (weekday_first_day + 53 * 7 - days) % 7;
+ }
+
+ // How many days since first week
+ if ((first_weekday && weekday_first_day != 0) || (!first_weekday && weekday_first_day > 3)) {
+ // days in new year belongs to last year.
+ days = day_nr - (daynr_first_day + (7 - weekday_first_day));
+ } else {
+ // days in new year belongs to this year.
+ days = day_nr - (daynr_first_day - weekday_first_day);
+ }
+
+ if (week_year && days >= 52 * 7) {
+ weekday_first_day = (weekday_first_day + calc_days_in_year(*to_year)) % 7;
+ if ((first_weekday && weekday_first_day == 0) ||
+ (!first_weekday && weekday_first_day <= 3)) {
+ // Belong to next year.
+ (*to_year)++;
+ return 1;
+ }
+ }
+
+ return days / 7 + 1;
+}
+
+std::ostream& operator<<(std::ostream& os, const DateV2Value& value) {
+ char buf[11];
+ value.to_string(buf);
+ return os << buf;
+}
+
+// NOTE:
+// only support DATE - DATE (no support DATETIME - DATETIME)
+std::size_t operator-(const DateV2Value& v1, const DateV2Value& v2) {
+ return v1.daynr() - v2.daynr();
+}
+
+std::size_t hash_value(DateV2Value const& value) {
+ return HashUtil::hash(&value, sizeof(DateV2Value), 0);
+}
} // namespace doris::vectorized
diff --git a/be/src/vec/runtime/vdatetime_value.h b/be/src/vec/runtime/vdatetime_value.h
index 4c1f4cc137..3dd325c72f 100644
--- a/be/src/vec/runtime/vdatetime_value.h
+++ b/be/src/vec/runtime/vdatetime_value.h
@@ -142,7 +142,49 @@ static constexpr const char* s_day_name[] = {"Monday", "Tuesday", "Wednesday",
static constexpr size_t MAX_DAY_NAME_LEN = max_char_length(s_day_name, std::size(s_day_name));
static constexpr size_t MAX_MONTH_NAME_LEN = max_char_length(s_month_name, std::size(s_month_name));
+const uint32_t MAX_DATE_V2 = 31 | (12 << 8) | (9999 << 16);
+const uint32_t MIN_DATE_V2 = 1 | (1 << 8) | (1000 << 16);
+
+const uint32_t MAX_YEAR = 9999;
+const uint32_t MIN_YEAR = 1000;
+
+static RE2 time_zone_offset_format_reg("^[+-]{1}\\d{2}\\:\\d{2}$");
+
uint8_t mysql_week_mode(uint32_t mode);
+bool is_leap(uint32_t year);
+uint32_t calc_days_in_year(uint32_t year);
+
+// Calculate how many days since 0000-01-01
+// 0000-01-01 is 1st B.C.
+static uint32_t calc_daynr(uint16_t year, uint8_t month, uint8_t day) {
+ uint32_t delsum = 0;
+ int y = year;
+
+ if (year == 0 && month == 0) {
+ return 0;
+ }
+
+ /* Cast to int to be able to handle month == 0 */
+ delsum = 365 * y + 31 * (month - 1) + day;
+ if (month <= 2) {
+ // No leap year
+ y--;
+ } else {
+ // This is great!!!
+ // 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12
+ // 0, 0, 3, 3, 4, 4, 5, 5, 5, 6, 7, 8
+ delsum -= (month * 4 + 23) / 10;
+ }
+ // Every 400 year has 97 leap year, 100, 200, 300 are not leap year.
+ return delsum + y / 4 - y / 100 + y / 400;
+}
+
+//W = (D + M*2 + 3*(M+1)/5 + Y + Y/4 -Y/100 + Y/400)%7
+static uint8_t calc_weekday(uint64_t day_nr, bool is_sunday_first_day) {
+ return (day_nr + 5L + (is_sunday_first_day ? 1L : 0L)) % 7;
+}
+
+class DateV2Value;
class VecDateTimeValue { // Now this type is a temp solution with little changes, maybe large refactoring follow-up.
public:
@@ -179,6 +221,8 @@ public:
return datetime;
}
+ void create_from_date_v2(DateV2Value& value, TimeType type);
+
void set_time(uint32_t year, uint32_t month, uint32_t day, uint32_t hour, uint32_t minute,
uint32_t second);
@@ -268,6 +312,8 @@ public:
// Return true if convert success. Otherwise return false.
bool from_date_int64(int64_t value);
+ bool from_date(int64_t value) { return from_date_int64(value); };
+
// Construct time type value from int64_t value.
// Return true if convert success. Otherwise return false.
bool from_time_int64(int64_t value);
@@ -293,71 +339,6 @@ public:
static bool check_date(uint32_t year, uint32_t month, uint32_t day);
- // compute the diff between two datetime value
- template <TimeUnit unit>
- static int64_t datetime_diff(const VecDateTimeValue& ts_value1,
- const VecDateTimeValue& ts_value2) {
- switch (unit) {
- case YEAR: {
- int year = (ts_value2.year() - ts_value1.year());
- if (year > 0) {
- year -= (ts_value2.to_datetime_int64() % 10000000000 -
- ts_value1.to_datetime_int64() % 10000000000) < 0;
- } else if (year < 0) {
- year += (ts_value2.to_datetime_int64() % 10000000000 -
- ts_value1.to_datetime_int64() % 10000000000) > 0;
- }
- return year;
- }
- case MONTH: {
- int month = (ts_value2.year() - ts_value1.year()) * 12 +
- (ts_value2.month() - ts_value1.month());
- if (month > 0) {
- month -= (ts_value2.to_datetime_int64() % 100000000 -
- ts_value1.to_datetime_int64() % 100000000) < 0;
- } else if (month < 0) {
- month += (ts_value2.to_datetime_int64() % 100000000 -
- ts_value1.to_datetime_int64() % 100000000) > 0;
- }
- return month;
- }
- case WEEK: {
- int day = ts_value2.daynr() - ts_value1.daynr();
- if (day > 0) {
- day -= ts_value2.time_part_diff(ts_value1) < 0;
- } else if (day < 0) {
- day += ts_value2.time_part_diff(ts_value1) > 0;
- }
- return day / 7;
- }
- case DAY: {
- int day = ts_value2.daynr() - ts_value1.daynr();
- if (day > 0) {
- day -= ts_value2.time_part_diff(ts_value1) < 0;
- } else if (day < 0) {
- day += ts_value2.time_part_diff(ts_value1) > 0;
- }
- return day;
- }
- case HOUR: {
- int64_t second = ts_value2.second_diff(ts_value1);
- int64_t hour = second / 60 / 60;
- return hour;
- }
- case MINUTE: {
- int64_t second = ts_value2.second_diff(ts_value1);
- int64_t minute = second / 60;
- return minute;
- }
- case SECOND: {
- int64_t second = ts_value2.second_diff(ts_value1);
- return second;
- }
- }
- // Rethink the default return value
- return 0;
- }
-
// Convert this value to uint64_t
// Will check its type
int64_t to_int64() const;
@@ -373,13 +354,6 @@ public:
uint64_t daynr() const { return calc_daynr(_year, _month, _day); }
- // Calculate how many days since 0000-01-01
- // 0000-01-01 is 1st B.C.
- static uint64_t calc_daynr(uint32_t year, uint32_t month, uint32_t day);
-
- static uint8_t calc_weekday(uint64_t daynr,
- bool); //W = (D + M*2 + 3*(M+1)/5 + Y + Y/4 -Y/100 + Y/400)%7
-
int year() const { return _year; }
int month() const { return _month; }
int quarter() const { return (_month - 1) / 3 + 1; }
@@ -493,6 +467,18 @@ public:
return v1 > v2;
}
+ bool operator==(const DateV2Value& other) const;
+
+ bool operator!=(const DateV2Value& other) const { return !(*this == other); };
+
+ bool operator<=(const DateV2Value& other) const;
+
+ bool operator>=(const DateV2Value& other) const;
+
+ bool operator<(const DateV2Value& other) const;
+
+ bool operator>(const DateV2Value& other) const;
+
const char* month_name() const;
const char* day_name() const;
@@ -523,6 +509,11 @@ public:
tv->type = _type;
}
+ uint32_t to_date_v2() const {
+ CHECK(_type == TIME_DATE);
+ return (year() << 16 | month() << 8 | day());
+ };
+
static VecDateTimeValue from_datetime_val(const doris_udf::DateTimeVal& tv) {
VecDateTimeValue value;
value.from_packed_time(tv.packed_time);
@@ -562,6 +553,8 @@ public:
return day_diff * 3600 * 24 + time_diff;
}
+ int64_t second_diff(const DateV2Value& rhs) const;
+
int64_t time_part_diff(const VecDateTimeValue& rhs) const {
int time_diff = (hour() * 3600 + minute() * 60 + second()) -
(rhs.hour() * 3600 + rhs.minute() * 60 + rhs.second());
@@ -579,6 +572,7 @@ public:
void convert_vec_dt_to_dt(doris::DateTimeValue* dt);
void convert_dt_to_vec_dt(doris::DateTimeValue* dt);
+ int64_t to_datetime_int64() const;
private:
// Used to make sure sizeof VecDateTimeValue
@@ -629,8 +623,6 @@ private:
char* to_date_buffer(char* to) const;
char* to_time_buffer(char* to) const;
- // Used to convert to int64_t
- int64_t to_datetime_int64() const;
int64_t to_date_int64() const;
int64_t to_time_int64() const;
@@ -666,18 +658,532 @@ private:
_day(day),
_month(month),
_year(year) {}
+};
+
+struct DateV2ValueType {
+ uint32_t day_ : 8;
+ uint32_t month_ : 8;
+ uint32_t year_ : 16;
+
+ DateV2ValueType(uint16_t year, uint8_t month, uint8_t day)
+ : day_(day), month_(month), year_(year) {}
+};
+
+class DateV2Value { // Now this type is a temp solution with little changes, maybe large refactoring follow-up.
+public:
+ // Constructor
+ DateV2Value() : date_v2_value_(0, 0, 0) {}
+
+ DateV2Value(DateV2Value& other) { int_val_ = other.to_date_uint32(); }
+
+ DateV2Value(const DateV2Value& other) { int_val_ = other.to_date_uint32(); }
+
+ static DateV2Value create_from_olap_date(uint64_t value) {
+ DateV2Value date;
+ date.from_olap_date(value);
+ return date;
+ }
+
+ void set_time(uint16_t year, uint8_t month, uint8_t day);
+
+ // TODO(gabriel): unify execution and storage
+ bool from_olap_date(uint64_t date) {
+ auto [year, month, day] = std::tuple {0, 0, 0};
+
+ day = date & 0x1f;
+ date >>= 5;
+ month = date & 0x0f;
+ date >>= 4;
+ year = date;
+
+ return check_range_and_set_time(year, month, day);
+ }
+
+ uint64_t to_olap_date() const {
+ uint64_t val;
+ val = date_v2_value_.year_;
+ val <<= 4;
+ val |= date_v2_value_.month_;
+ val <<= 5;
+ val |= date_v2_value_.day_;
+ return val;
+ }
+
+ bool to_format_string(const char* format, int len, char* to) const;
+
+ bool from_date_format_str(const char* format, int format_len, const char* value,
+ int value_len) {
+ return from_date_format_str(format, format_len, value, value_len, nullptr);
+ }
+
+ // Construct Date/Datetime type value from string.
+ // At least the following formats are recognised (based on number of digits)
+ // 'YYMMDD', 'YYYYMMDD', 'YYMMDDHHMMSS', 'YYYYMMDDHHMMSS'
+ // 'YY-MM-DD', 'YYYY-MM-DD', 'YY-MM-DD HH.MM.SS'
+ // 'YYYYMMDDTHHMMSS'
+ bool from_date_str(const char* str, int len);
+
+ // Convert this value to string
+ // this will check type to decide which format to convert
+ // TIME: format 'hh:mm:ss.xxxxxx'
+ // DATE: format 'YYYY-MM-DD'
+ // DATETIME: format 'YYYY-MM-DD hh:mm:ss.xxxxxx'
+ int32_t to_buffer(char* buffer) const;
+
+ char* to_string(char* to) const;
+
+ // Return true if range or date is invalid
+ static bool is_invalid(uint32_t year, uint32_t month, uint32_t day);
+
+ bool check_range_and_set_time(uint16_t year, uint8_t month, uint8_t day) {
+ if (is_invalid(year, month, day)) {
+ return false;
+ }
+ set_time(year, month, day);
+ return true;
+ };
+
+ uint32_t daynr() const {
+ return calc_daynr(date_v2_value_.year_, date_v2_value_.month_, date_v2_value_.day_);
+ }
+
+ int hour() const { return 0; }
+ int minute() const { return 0; }
+ int second() const { return 0; }
+ uint16_t year() const { return date_v2_value_.year_; }
+ uint8_t month() const { return date_v2_value_.month_; }
+ int quarter() const { return (date_v2_value_.month_ - 1) / 3 + 1; }
+ int week() const { return week(mysql_week_mode(0)); } //00-53
+ uint8_t day() const { return date_v2_value_.day_; }
+
+ // Weekday, from 0(Mon) to 6(Sun)
+ uint8_t weekday() const { return calc_weekday(daynr(), false); }
+ auto day_of_week() const { return (weekday() + 1) % 7 + 1; }
+
+ // The bits in week_format has the following meaning:
+ // WEEK_MONDAY_FIRST (0)
+ // If not set:
+ // Sunday is first day of week
+ // If set:
+ // Monday is first day of week
+ //
+ // WEEK_YEAR (1)
+ // If not set:
+ // Week is in range 0-53
+ // Week 0 is returned for the last week of the previous year (for
+ // a date at start of january) In this case one can get 53 for the
+ // first week of next year. This flag ensures that the week is
+ // relevant for the given year. Note that this flag is only
+ // relevant if WEEK_JANUARY is not set.
+ // If set:
+ // Week is in range 1-53.
+ // In this case one may get week 53 for a date in January (when
+ // the week is that last week of previous year) and week 1 for a
+ // date in December.
+ //
+ // WEEK_FIRST_WEEKDAY (2)
+ // If not set
+ // Weeks are numbered according to ISO 8601:1988
+ // If set
+ // The week that contains the first 'first-day-of-week' is week 1.
+ //
+ // ISO 8601:1988 means that
+ // if the week containing January 1 has
+ // four or more days in the new year, then it is week 1;
+ // Otherwise it is the last week of the previous year, and the
+ // next week is week 1.
+ uint8_t week(uint8_t) const;
+
+ uint32_t year_week(uint8_t mode) const;
+
+ // Add interval
+ bool date_add_interval(const TimeInterval& interval, TimeUnit unit);
- // RE2 obj is thread safe
- static RE2 time_zone_offset_format_reg;
+ //unix_timestamp is called with a timezone argument,
+ //it returns seconds of the value of date literal since '1970-01-01 00:00:00' UTC
+ bool unix_timestamp(int64_t* timestamp, const std::string& timezone) const;
+ bool unix_timestamp(int64_t* timestamp, const cctz::time_zone& ctz) const;
+
+ //construct datetime_value from timestamp and timezone
+ //timestamp is an internal timestamp value representing seconds since '1970-01-01 00:00:00' UTC
+ bool from_unixtime(int64_t, const std::string& timezone);
+ bool from_unixtime(int64_t, const cctz::time_zone& ctz);
+
+ bool operator==(const DateV2Value& other) const {
+ // NOTE: This is not same with MySQL.
+ // MySQL convert both to int with left value type and then compare
+ // We think all fields equals.
+ return this->to_date_uint32() == other.to_date_uint32();
+ }
+
+ bool operator==(const VecDateTimeValue& other) const {
+ int64_t ts1;
+ int64_t ts2;
+ this->unix_timestamp(&ts1, TimezoneUtils::default_time_zone);
+ other.unix_timestamp(&ts2, TimezoneUtils::default_time_zone);
+ return ts1 == ts2;
+ }
+
+ bool operator!=(const DateV2Value& other) const {
+ return this->to_date_uint32() != other.to_date_uint32();
+ }
+
+ bool operator!=(const VecDateTimeValue& other) const { return !(*this == other); }
+
+ bool operator<=(const DateV2Value& other) const { return !(*this > other); }
+
+ bool operator<=(const VecDateTimeValue& other) const {
+ int64_t ts1;
+ int64_t ts2;
+ this->unix_timestamp(&ts1, TimezoneUtils::default_time_zone);
+ other.unix_timestamp(&ts2, TimezoneUtils::default_time_zone);
+ return ts1 <= ts2;
+ }
+
+ bool operator>=(const DateV2Value& other) const { return !(*this < other); }
+
+ bool operator>=(const VecDateTimeValue& other) const {
+ int64_t ts1;
+ int64_t ts2;
+ this->unix_timestamp(&ts1, TimezoneUtils::default_time_zone);
+ other.unix_timestamp(&ts2, TimezoneUtils::default_time_zone);
+ return ts1 >= ts2;
+ }
+
+ bool operator<(const DateV2Value& other) const {
+ return to_date_uint32() < other.to_date_uint32();
+ }
+
+ bool operator<(const VecDateTimeValue& other) const {
+ int64_t ts1;
+ int64_t ts2;
+ this->unix_timestamp(&ts1, TimezoneUtils::default_time_zone);
+ other.unix_timestamp(&ts2, TimezoneUtils::default_time_zone);
+ return ts1 < ts2;
+ }
+
+ bool operator>(const DateV2Value& other) const {
+ return to_date_uint32() > other.to_date_uint32();
+ }
+
+ bool operator>(const VecDateTimeValue& other) const {
+ int64_t ts1;
+ int64_t ts2;
+ this->unix_timestamp(&ts1, TimezoneUtils::default_time_zone);
+ other.unix_timestamp(&ts2, TimezoneUtils::default_time_zone);
+ return ts1 > ts2;
+ }
+
+ DateV2Value& operator=(const DateV2Value& other) {
+ int_val_ = other.to_date_uint32();
+ return *this;
+ }
+
+ DateV2Value& operator=(DateV2Value& other) {
+ int_val_ = other.to_date_uint32();
+ return *this;
+ }
+
+ const char* month_name() const;
+
+ const char* day_name() const;
+
+ DateV2Value& operator++() {
+ TimeInterval interval(DAY, 1, false);
+ date_add_interval(interval, DAY);
+ return *this;
+ }
+
+ uint32_t hash(int seed) const { return HashUtil::hash(this, sizeof(*this), seed); }
+
+ int day_of_year() const { return daynr() - calc_daynr(this->year(), 1, 1) + 1; }
+
+ std::string debug_string() const {
+ char buf[64];
+ char* end = to_string(buf);
+ return std::string(buf, end - buf);
+ }
+
+ bool is_valid_date() const { return !is_invalid(this->year(), this->month(), this->day()); }
+
+ int64_t second_diff(const DateV2Value& rhs) const {
+ int day_diff = daynr() - rhs.daynr();
+ return day_diff * 3600 * 24;
+ }
+
+ int64_t second_diff(const VecDateTimeValue& rhs) const {
+ int day_diff = daynr() - rhs.daynr();
+ int time_diff = rhs.hour() * 3600 + rhs.minute() * 60 + rhs.second();
+ return day_diff * 3600 * 24 - time_diff;
+ };
+
+ void convert_date_v2_to_dt(doris::DateTimeValue* dt);
+ void convert_dt_to_date_v2(doris::DateTimeValue* dt);
+
+ uint32_t to_date_uint32() const;
+
+ bool from_date(uint32_t value);
+ bool from_date(int64_t value);
+
+ bool from_date_int64(int64_t value);
+ uint32_t set_date_uint32(uint32_t int_val);
+
+ bool get_date_from_daynr(uint64_t);
+
+private:
+ static uint8_t calc_week(const uint32_t& day_nr, const uint16_t& year, const uint8_t& month,
+ const uint8_t& day, uint8_t mode, uint16_t* to_year);
+
+ // Used to construct from int value
+ int64_t standardize_timevalue(int64_t value);
+
+ // Helper to set max, min, zero
+ void set_zero(int type);
+
+ bool from_date_format_str(const char* format, int format_len, const char* value, int value_len,
+ const char** sub_val_end);
+
+ union {
+ DateV2ValueType date_v2_value_;
+ uint32_t int_val_;
+ };
+
+ DateV2Value(uint16_t year, uint8_t month, uint8_t day) : date_v2_value_(year, month, day) {}
};
// only support DATE - DATE (no support DATETIME - DATETIME)
std::size_t operator-(const VecDateTimeValue& v1, const VecDateTimeValue& v2);
+std::size_t operator-(const VecDateTimeValue& v1, const DateV2Value& v2);
+
+std::size_t operator-(const DateV2Value& v1, const VecDateTimeValue& v2);
+
std::ostream& operator<<(std::ostream& os, const VecDateTimeValue& value);
std::size_t hash_value(VecDateTimeValue const& value);
+std::size_t operator-(const DateV2Value& v1, const DateV2Value& v2);
+
+std::ostream& operator<<(std::ostream& os, const DateV2Value& value);
+
+std::size_t hash_value(DateV2Value const& value);
+
+template <TimeUnit unit>
+int64_t datetime_diff(const VecDateTimeValue& ts_value1, const VecDateTimeValue& ts_value2) {
+ switch (unit) {
+ case YEAR: {
+ int year = (ts_value2.year() - ts_value1.year());
+ if (year > 0) {
+ year -= (ts_value2.to_datetime_int64() % 10000000000 -
+ ts_value1.to_datetime_int64() % 10000000000) < 0;
+ } else if (year < 0) {
+ year += (ts_value2.to_datetime_int64() % 10000000000 -
+ ts_value1.to_datetime_int64() % 10000000000) > 0;
+ }
+ return year;
+ }
+ case MONTH: {
+ int month = (ts_value2.year() - ts_value1.year()) * 12 +
+ (ts_value2.month() - ts_value1.month());
+ if (month > 0) {
+ month -= (ts_value2.to_datetime_int64() % 100000000 -
+ ts_value1.to_datetime_int64() % 100000000) < 0;
+ } else if (month < 0) {
+ month += (ts_value2.to_datetime_int64() % 100000000 -
+ ts_value1.to_datetime_int64() % 100000000) > 0;
+ }
+ return month;
+ }
+ case WEEK: {
+ int day = ts_value2.daynr() - ts_value1.daynr();
+ if (day > 0) {
+ day -= ts_value2.time_part_diff(ts_value1) < 0;
+ } else if (day < 0) {
+ day += ts_value2.time_part_diff(ts_value1) > 0;
+ }
+ return day / 7;
+ }
+ case DAY: {
+ int day = ts_value2.daynr() - ts_value1.daynr();
+ if (day > 0) {
+ day -= ts_value2.time_part_diff(ts_value1) < 0;
+ } else if (day < 0) {
+ day += ts_value2.time_part_diff(ts_value1) > 0;
+ }
+ return day;
+ }
+ case HOUR: {
+ int64_t second = ts_value2.second_diff(ts_value1);
+ int64_t hour = second / 60 / 60;
+ return hour;
+ }
+ case MINUTE: {
+ int64_t second = ts_value2.second_diff(ts_value1);
+ int64_t minute = second / 60;
+ return minute;
+ }
+ case SECOND: {
+ int64_t second = ts_value2.second_diff(ts_value1);
+ return second;
+ }
+ }
+ // Rethink the default return value
+ return 0;
+}
+
+template <TimeUnit unit>
+int64_t datetime_diff(const DateV2Value& ts_value1, const DateV2Value& ts_value2) {
+ switch (unit) {
+ case YEAR: {
+ int year = (ts_value2.year() - ts_value1.year());
+ if (year > 0) {
+ year -= ((ts_value2.to_date_uint32() & 0x0000FFFF) -
+ (ts_value1.to_date_uint32() & 0x0000FFFF)) < 0;
+ } else if (year < 0) {
+ year += ((ts_value2.to_date_uint32() & 0x0000FFFF) -
+ (ts_value1.to_date_uint32() & 0x0000FFFF)) > 0;
+ }
+ return year;
+ }
+ case MONTH: {
+ int month = (ts_value2.year() - ts_value1.year()) * 12 +
+ (ts_value2.month() - ts_value1.month());
+ if (month > 0) {
+ month -= (ts_value2.day() - ts_value1.day()) < 0;
+ } else if (month < 0) {
+ month += (ts_value2.day() - ts_value1.day()) > 0;
+ }
+ return month;
+ }
+ case WEEK: {
+ int day = ts_value2.daynr() - ts_value1.daynr();
+ return day / 7;
+ }
+ case DAY: {
+ int day = ts_value2.daynr() - ts_value1.daynr();
+ return day;
+ }
+ case HOUR: {
+ int64_t second = ts_value2.second_diff(ts_value1);
+ int64_t hour = second / 60 / 60;
+ return hour;
+ }
+ case MINUTE: {
+ int64_t second = ts_value2.second_diff(ts_value1);
+ int64_t minute = second / 60;
+ return minute;
+ }
+ case SECOND: {
+ int64_t second = ts_value2.second_diff(ts_value1);
+ return second;
+ }
+ }
+ // Rethink the default return value
+ return 0;
+}
+
+template <TimeUnit unit>
+int64_t datetime_diff(const DateV2Value& ts_value1, const VecDateTimeValue& ts_value2) {
+ switch (unit) {
+ case YEAR: {
+ int year = (ts_value2.year() - ts_value1.year());
+ if (year > 0) {
+ year -= ((((ts_value2.to_datetime_int64() % 10000000000) / 1000000L) % 10000L) -
+ (ts_value1.to_date_uint32() & 0x0000FFFF)) < 0;
+ } else if (year < 0) {
+ year += ((((ts_value2.to_datetime_int64() % 10000000000) / 1000000L) % 10000L) -
+ (ts_value1.to_date_uint32() & 0x0000FFFF)) > 0;
+ }
+ return year;
+ }
+ case MONTH: {
+ int month = (ts_value2.year() - ts_value1.year()) * 12 +
+ (ts_value2.month() - ts_value1.month());
+ if (month > 0) {
+ month -= (ts_value2.day() - ts_value1.day()) < 0;
+ } else if (month < 0) {
+ month += (ts_value2.day() - ts_value1.day()) > 0;
+ }
+ return month;
+ }
+ case WEEK: {
+ int day = ts_value2.daynr() - ts_value1.daynr();
+ return day / 7;
+ }
+ case DAY: {
+ int day = ts_value2.daynr() - ts_value1.daynr();
+ return day;
+ }
+ case HOUR: {
+ int64_t second = ts_value2.second_diff(ts_value1);
+ int64_t hour = second / 60 / 60;
+ return hour;
+ }
+ case MINUTE: {
+ int64_t second = ts_value2.second_diff(ts_value1);
+ int64_t minute = second / 60;
+ return minute;
+ }
+ case SECOND: {
+ int64_t second = ts_value2.second_diff(ts_value1);
+ return second;
+ }
+ }
+ // Rethink the default return value
+ return 0;
+}
+
+template <TimeUnit unit>
+int64_t datetime_diff(const VecDateTimeValue& ts_value1, const DateV2Value& ts_value2) {
+ switch (unit) {
+ case YEAR: {
+ int year = (ts_value2.year() - ts_value1.year());
+ if (year > 0) {
+ year -= ((ts_value2.to_date_uint32() & 0x0000FFFF) -
+ (((ts_value1.to_datetime_int64() % 10000000000) / 1000000L) % 10000L)) < 0;
+ } else if (year < 0) {
+ year += ((ts_value2.to_date_uint32() & 0x0000FFFF) -
+ (((ts_value1.to_datetime_int64() % 10000000000) / 1000000L) % 10000L)) > 0;
+ }
+ return year;
+ }
+ case MONTH: {
+ int month = (ts_value2.year() - ts_value1.year()) * 12 +
+ (ts_value2.month() - ts_value1.month());
+ if (month > 0) {
+ month -= (ts_value2.day() - ts_value1.day()) < 0;
+ } else if (month < 0) {
+ month += (ts_value2.day() - ts_value1.day()) > 0;
+ }
+ return month;
+ }
+ case WEEK: {
+ int day = ts_value2.daynr() - ts_value1.daynr();
+ return day / 7;
+ }
+ case DAY: {
+ int day = ts_value2.daynr() - ts_value1.daynr();
+ return day;
+ }
+ case HOUR: {
+ int64_t second = ts_value2.second_diff(ts_value1);
+ int64_t hour = second / 60 / 60;
+ return hour;
+ }
+ case MINUTE: {
+ int64_t second = ts_value2.second_diff(ts_value1);
+ int64_t minute = second / 60;
+ return minute;
+ }
+ case SECOND: {
+ int64_t second = ts_value2.second_diff(ts_value1);
+ return second;
+ }
+ }
+ // Rethink the default return value
+ return 0;
+}
+
} // namespace vectorized
} // namespace doris
@@ -688,6 +1194,13 @@ struct hash<doris::vectorized::VecDateTimeValue> {
return doris::vectorized::hash_value(v);
}
};
+
+template <>
+struct hash<doris::vectorized::DateV2Value> {
+ size_t operator()(const doris::vectorized::DateV2Value& v) const {
+ return doris::vectorized::hash_value(v);
+ }
+};
} // namespace std
#endif
diff --git a/be/src/vec/runtime/vfile_result_writer.cpp b/be/src/vec/runtime/vfile_result_writer.cpp
index c1000193d8..30e32d8e7c 100644
--- a/be/src/vec/runtime/vfile_result_writer.cpp
+++ b/be/src/vec/runtime/vfile_result_writer.cpp
@@ -246,6 +246,14 @@ Status VFileResultWriter::_write_csv_file(const Block& block) {
_plain_text_outstream << buffer;
break;
}
+ case TYPE_DATEV2: {
+ char buf[64];
+ const DateV2Value* time_val =
+ (const DateV2Value*)(col.column->get_data_at(i).data);
+ time_val->to_string(buf);
+ _plain_text_outstream << buf;
+ break;
+ }
case TYPE_DATE:
case TYPE_DATETIME: {
char buf[64];
diff --git a/be/src/vec/sink/mysql_result_writer.cpp b/be/src/vec/sink/mysql_result_writer.cpp
index 1902bf9972..e9c3506dd6 100644
--- a/be/src/vec/sink/mysql_result_writer.cpp
+++ b/be/src/vec/sink/mysql_result_writer.cpp
@@ -200,7 +200,14 @@ Status VMysqlResultWriter::_add_one_column(const ColumnPtr& column_ptr,
char* pos = time_val.to_string(buf);
buf_ret = _buffer.push_string(buf, pos - buf - 1);
}
-
+ if constexpr (type == TYPE_DATEV2) {
+ char buf[64];
+ auto time_num = data[i];
+ doris::vectorized::DateV2Value date_val;
+ memcpy(static_cast<void*>(&date_val), &time_num, sizeof(UInt32));
+ char* pos = date_val.to_string(buf);
+ buf_ret = _buffer.push_string(buf, pos - buf - 1);
+ }
if constexpr (type == TYPE_DECIMALV2) {
DecimalV2Value decimal_val(data[i]);
auto decimal_str = decimal_val.to_string();
@@ -289,6 +296,14 @@ int VMysqlResultWriter::_add_one_cell(const ColumnPtr& column_ptr, size_t row_id
DecimalV2Value decimal_val(column_data[row_idx]);
auto decimal_str = decimal_val.to_string();
return buffer.push_string(decimal_str.c_str(), decimal_str.length());
+ } else if (which.is_date_v2()) {
+ auto& column_vector = assert_cast<const ColumnVector<UInt32>&>(*column);
+ auto value = column_vector[row_idx].get<UInt32>();
+ DateV2Value datev2;
+ memcpy(static_cast<void*>(&datev2), static_cast<void*>(&value), sizeof(value));
+ char buf[64];
+ char* pos = datev2.to_string(buf);
+ return buffer.push_string(buf, pos - buf - 1);
} else if (which.is_array()) {
auto& column_array = assert_cast<const ColumnArray&>(*column);
auto& offsets = column_array.get_offsets();
@@ -453,6 +468,14 @@ Status VMysqlResultWriter::append_block(Block& input_block) {
}
break;
}
+ case TYPE_DATEV2: {
+ if (type_ptr->is_nullable()) {
+ status = _add_one_column<PrimitiveType::TYPE_DATEV2, true>(column_ptr, result);
+ } else {
+ status = _add_one_column<PrimitiveType::TYPE_DATEV2, false>(column_ptr, result);
+ }
+ break;
+ }
case TYPE_HLL:
case TYPE_OBJECT: {
if (type_ptr->is_nullable()) {
diff --git a/be/src/vec/sink/vmysql_table_writer.cpp b/be/src/vec/sink/vmysql_table_writer.cpp
index 0fb1cb89e4..6c7d33830e 100644
--- a/be/src/vec/sink/vmysql_table_writer.cpp
+++ b/be/src/vec/sink/vmysql_table_writer.cpp
@@ -186,6 +186,18 @@ Status VMysqlTableWriter::insert_row(vectorized::Block& block, size_t row) {
fmt::format_to(_insert_stmt_buffer, "'{}'", str);
break;
}
+ case TYPE_DATEV2: {
+ uint32_t int_val =
+ assert_cast<const vectorized::ColumnUInt32&>(*column).get_data()[row];
+ vectorized::DateV2Value value =
+ binary_cast<uint32_t, doris::vectorized::DateV2Value>(int_val);
+
+ char buf[64];
+ char* pos = value.to_string(buf);
+ std::string str(buf, pos - buf - 1);
+ fmt::format_to(_insert_stmt_buffer, "'{}'", str);
+ break;
+ }
default: {
fmt::memory_buffer err_out;
fmt::format_to(err_out, "can't convert this type to mysql type. type = {}",
diff --git a/be/src/vec/utils/arrow_column_to_doris_column.cpp b/be/src/vec/utils/arrow_column_to_doris_column.cpp
index 76741fdb59..5923f72f16 100644
--- a/be/src/vec/utils/arrow_column_to_doris_column.cpp
+++ b/be/src/vec/utils/arrow_column_to_doris_column.cpp
@@ -30,6 +30,7 @@
#include "gutil/casts.h"
#include "vec/columns/column_array.h"
#include "vec/columns/column_nullable.h"
+#include "vec/data_types/data_type_array.h"
#include "vec/data_types/data_type_decimal.h"
#include "vec/runtime/vdatetime_value.h"
@@ -213,6 +214,35 @@ static Status convert_column_with_timestamp_data(const arrow::Array* array, size
return Status::OK();
}
+template <typename ArrowType>
+static Status convert_column_with_date_v2_data(const arrow::Array* array, size_t array_idx,
+ MutableColumnPtr& data_column, size_t num_elements,
+ const std::string& timezone) {
+ auto& column_data = static_cast<ColumnVector<UInt32>&>(*data_column).get_data();
+ auto concrete_array = down_cast<const ArrowType*>(array);
+ int64_t divisor = 1;
+ int64_t multiplier = 1;
+ if constexpr (std::is_same_v<ArrowType, arrow::TimestampArray>) {
+ const auto type = std::static_pointer_cast<arrow::TimestampType>(array->type());
+ divisor = time_unit_divisor(type->unit());
+ if (divisor == 0L) {
+ return Status::InternalError(fmt::format("Invalid Time Type:{}", type->name()));
+ }
+ } else if constexpr (std::is_same_v<ArrowType, arrow::Date32Array>) {
+ multiplier = 24 * 60 * 60; // day => secs
+ } else if constexpr (std::is_same_v<ArrowType, arrow::Date64Array>) {
+ divisor = 1000; //ms => secs
+ }
+
+ for (size_t value_i = array_idx; value_i < array_idx + num_elements; ++value_i) {
+ DateV2Value v;
+ v.from_unixtime(static_cast<Int64>(concrete_array->Value(value_i)) / divisor * multiplier,
+ timezone);
+ column_data.emplace_back(binary_cast<DateV2Value, UInt32>(v));
+ }
+ return Status::OK();
+}
+
static Status convert_column_with_decimal_data(const arrow::Array* array, size_t array_idx,
MutableColumnPtr& data_column, size_t num_elements) {
auto& column_data =
@@ -256,7 +286,8 @@ static Status convert_offset_from_list_column(const arrow::Array* array, size_t
static Status convert_column_with_list_data(const arrow::Array* array, size_t array_idx,
MutableColumnPtr& data_column, size_t num_elements,
- const std::string& timezone) {
+ const std::string& timezone,
+ const DataTypePtr& nested_type) {
size_t start_idx_of_data = 0;
size_t num_of_data = 0;
// get start idx and num of values from arrow offsets
@@ -267,23 +298,20 @@ static Status convert_column_with_list_data(const arrow::Array* array, size_t ar
std::shared_ptr<arrow::Array> arrow_data = concrete_array->values();
return arrow_column_to_doris_column(arrow_data.get(), start_idx_of_data, data_column_ptr,
- num_of_data, timezone);
+ nested_type, num_of_data, timezone);
}
Status arrow_column_to_doris_column(const arrow::Array* arrow_column, size_t arrow_batch_cur_idx,
- ColumnPtr& doirs_column, size_t num_elements,
- const std::string& timezone) {
+ ColumnPtr& doris_column, const DataTypePtr& type,
+ size_t num_elements, const std::string& timezone) {
// src column always be nullable for simpify converting
- assert(doirs_column->is_nullable());
+ CHECK(doris_column->is_nullable());
MutableColumnPtr data_column = nullptr;
- if (doirs_column->is_nullable()) {
- auto* nullable_column = reinterpret_cast<vectorized::ColumnNullable*>(
- (*std::move(doirs_column)).mutate().get());
- fill_nullable_column(arrow_column, arrow_batch_cur_idx, nullable_column, num_elements);
- data_column = nullable_column->get_nested_column_ptr();
- } else {
- data_column = (*std::move(doirs_column)).mutate();
- }
+ auto* nullable_column = reinterpret_cast<vectorized::ColumnNullable*>(
+ (*std::move(doris_column)).mutate().get());
+ fill_nullable_column(arrow_column, arrow_batch_cur_idx, nullable_column, num_elements);
+ data_column = nullable_column->get_nested_column_ptr();
+ WhichDataType which_type(type);
// process data
switch (arrow_column->type()->id()) {
case arrow::Type::STRING:
@@ -303,8 +331,13 @@ Status arrow_column_to_doris_column(const arrow::Array* arrow_column, size_t arr
return convert_column_with_boolean_data(arrow_column, arrow_batch_cur_idx, data_column,
num_elements);
case arrow::Type::DATE32:
- return convert_column_with_timestamp_data<arrow::Date32Array>(
- arrow_column, arrow_batch_cur_idx, data_column, num_elements, timezone);
+ if (which_type.is_date_v2()) {
+ return convert_column_with_date_v2_data<arrow::Date32Array>(
+ arrow_column, arrow_batch_cur_idx, data_column, num_elements, timezone);
+ } else {
+ return convert_column_with_timestamp_data<arrow::Date32Array>(
+ arrow_column, arrow_batch_cur_idx, data_column, num_elements, timezone);
+ }
case arrow::Type::DATE64:
return convert_column_with_timestamp_data<arrow::Date64Array>(
arrow_column, arrow_batch_cur_idx, data_column, num_elements, timezone);
@@ -315,8 +348,10 @@ Status arrow_column_to_doris_column(const arrow::Array* arrow_column, size_t arr
return convert_column_with_decimal_data(arrow_column, arrow_batch_cur_idx, data_column,
num_elements);
case arrow::Type::LIST:
- return convert_column_with_list_data(arrow_column, arrow_batch_cur_idx, data_column,
- num_elements, timezone);
+ CHECK(type->have_subtypes());
+ return convert_column_with_list_data(
+ arrow_column, arrow_batch_cur_idx, data_column, num_elements, timezone,
+ (reinterpret_cast<const DataTypeArray*>(type.get()))->get_nested_type());
default:
break;
}
diff --git a/be/src/vec/utils/arrow_column_to_doris_column.h b/be/src/vec/utils/arrow_column_to_doris_column.h
index 0c75f27cda..2c9189b1be 100644
--- a/be/src/vec/utils/arrow_column_to_doris_column.h
+++ b/be/src/vec/utils/arrow_column_to_doris_column.h
@@ -34,7 +34,7 @@ namespace doris::vectorized {
const PrimitiveType arrow_type_to_primitive_type(::arrow::Type::type type);
Status arrow_column_to_doris_column(const arrow::Array* arrow_column, size_t arrow_batch_cur_idx,
- ColumnPtr& doirs_column, size_t num_elements,
- const std::string& timezone);
+ ColumnPtr& doris_column, const DataTypePtr& type,
+ size_t num_elements, const std::string& timezone);
} // namespace doris::vectorized
diff --git a/be/src/vec/utils/template_helpers.hpp b/be/src/vec/utils/template_helpers.hpp
index 7fe681a8c6..afe745d1aa 100644
--- a/be/src/vec/utils/template_helpers.hpp
+++ b/be/src/vec/utils/template_helpers.hpp
@@ -43,7 +43,8 @@
#define TIME_TYPE_TO_COLUMN_TYPE(M) \
M(Date, ColumnInt64) \
- M(DateTime, ColumnInt64)
+ M(DateTime, ColumnInt64) \
+ M(DateV2, ColumnUInt32)
#define COMPLEX_TYPE_TO_COLUMN_TYPE(M) \
M(BitMap, ColumnBitmap) \
diff --git a/be/test/CMakeLists.txt b/be/test/CMakeLists.txt
index 6946e2f50c..b245a343d2 100644
--- a/be/test/CMakeLists.txt
+++ b/be/test/CMakeLists.txt
@@ -356,6 +356,7 @@ set(VEC_TEST_FILES
vec/function/function_test_util.cpp
vec/function/table_function_test.cpp
vec/runtime/vdata_stream_test.cpp
+ vec/runtime/vdatetime_value_test.cpp
vec/utils/arrow_column_to_doris_column_test.cpp
vec/olap/char_type_padding_test.cpp
)
diff --git a/be/test/olap/delta_writer_test.cpp b/be/test/olap/delta_writer_test.cpp
index 632086f891..c364af49d2 100644
--- a/be/test/olap/delta_writer_test.cpp
+++ b/be/test/olap/delta_writer_test.cpp
@@ -151,6 +151,12 @@ static void create_tablet_request(int64_t tablet_id, int32_t schema_hash,
k10.column_type.__set_scale(3);
request->tablet_schema.columns.push_back(k10);
+ TColumn k11;
+ k11.column_name = "k11";
+ k11.__set_is_key(true);
+ k11.column_type.type = TPrimitiveType::DATEV2;
+ request->tablet_schema.columns.push_back(k11);
+
TColumn v1;
v1.column_name = "v1";
v1.__set_is_key(false);
@@ -224,6 +230,13 @@ static void create_tablet_request(int64_t tablet_id, int32_t schema_hash,
v10.column_type.__set_scale(3);
v10.__set_aggregation_type(TAggregationType::SUM);
request->tablet_schema.columns.push_back(v10);
+
+ TColumn v11;
+ v11.column_name = "v11";
+ v11.__set_is_key(false);
+ v11.column_type.type = TPrimitiveType::DATEV2;
+ v11.__set_aggregation_type(TAggregationType::REPLACE);
+ request->tablet_schema.columns.push_back(v11);
}
static void create_tablet_request_with_sequence_col(int64_t tablet_id, int32_t schema_hash,
@@ -262,6 +275,13 @@ static void create_tablet_request_with_sequence_col(int64_t tablet_id, int32_t s
v1.column_type.type = TPrimitiveType::DATETIME;
v1.__set_aggregation_type(TAggregationType::REPLACE);
request->tablet_schema.columns.push_back(v1);
+
+ TColumn v2;
+ v2.column_name = "v2";
+ v2.__set_is_key(false);
+ v2.column_type.type = TPrimitiveType::DATEV2;
+ v2.__set_aggregation_type(TAggregationType::REPLACE);
+ request->tablet_schema.columns.push_back(v2);
}
static TDescriptorTable create_descriptor_tablet() {
@@ -288,27 +308,31 @@ static TDescriptorTable create_descriptor_tablet() {
TSlotDescriptorBuilder().string_type(65).column_name("k9").column_pos(8).build());
tuple_builder.add_slot(
TSlotDescriptorBuilder().decimal_type(6, 3).column_name("k10").column_pos(9).build());
+ tuple_builder.add_slot(
+ TSlotDescriptorBuilder().type(TYPE_DATEV2).column_name("k11").column_pos(10).build());
tuple_builder.add_slot(
- TSlotDescriptorBuilder().type(TYPE_TINYINT).column_name("v1").column_pos(10).build());
+ TSlotDescriptorBuilder().type(TYPE_TINYINT).column_name("v1").column_pos(11).build());
+ tuple_builder.add_slot(
+ TSlotDescriptorBuilder().type(TYPE_SMALLINT).column_name("v2").column_pos(12).build());
tuple_builder.add_slot(
- TSlotDescriptorBuilder().type(TYPE_SMALLINT).column_name("v2").column_pos(11).build());
+ TSlotDescriptorBuilder().type(TYPE_INT).column_name("v3").column_pos(13).build());
tuple_builder.add_slot(
- TSlotDescriptorBuilder().type(TYPE_INT).column_name("v3").column_pos(12).build());
+ TSlotDescriptorBuilder().type(TYPE_BIGINT).column_name("v4").column_pos(14).build());
tuple_builder.add_slot(
- TSlotDescriptorBuilder().type(TYPE_BIGINT).column_name("v4").column_pos(13).build());
+ TSlotDescriptorBuilder().type(TYPE_LARGEINT).column_name("v5").column_pos(15).build());
tuple_builder.add_slot(
- TSlotDescriptorBuilder().type(TYPE_LARGEINT).column_name("v5").column_pos(14).build());
+ TSlotDescriptorBuilder().type(TYPE_DATE).column_name("v6").column_pos(16).build());
tuple_builder.add_slot(
- TSlotDescriptorBuilder().type(TYPE_DATE).column_name("v6").column_pos(15).build());
+ TSlotDescriptorBuilder().type(TYPE_DATETIME).column_name("v7").column_pos(17).build());
tuple_builder.add_slot(
- TSlotDescriptorBuilder().type(TYPE_DATETIME).column_name("v7").column_pos(16).build());
+ TSlotDescriptorBuilder().string_type(4).column_name("v8").column_pos(18).build());
tuple_builder.add_slot(
- TSlotDescriptorBuilder().string_type(4).column_name("v8").column_pos(17).build());
+ TSlotDescriptorBuilder().string_type(65).column_name("v9").column_pos(19).build());
tuple_builder.add_slot(
- TSlotDescriptorBuilder().string_type(65).column_name("v9").column_pos(18).build());
+ TSlotDescriptorBuilder().decimal_type(6, 3).column_name("v10").column_pos(20).build());
tuple_builder.add_slot(
- TSlotDescriptorBuilder().decimal_type(6, 3).column_name("v10").column_pos(19).build());
+ TSlotDescriptorBuilder().type(TYPE_DATEV2).column_name("v11").column_pos(21).build());
tuple_builder.build(&dtb);
return dtb.desc_tbl();
@@ -329,6 +353,8 @@ static TDescriptorTable create_descriptor_tablet_with_sequence_col() {
.build());
tuple_builder.add_slot(
TSlotDescriptorBuilder().type(TYPE_DATETIME).column_name("v1").column_pos(3).build());
+ tuple_builder.add_slot(
+ TSlotDescriptorBuilder().type(TYPE_DATEV2).column_name("v2").column_pos(4).build());
tuple_builder.build(&dtb);
return dtb.desc_tbl();
@@ -439,25 +465,27 @@ TEST_F(TestDeltaWriter, write) {
DecimalV2Value decimal_value;
decimal_value.assign_from_double(1.1);
*(DecimalV2Value*)(tuple->get_slot(slots[9]->tuple_offset())) = decimal_value;
+ ((doris::vectorized::DateV2Value*)(tuple->get_slot(slots[10]->tuple_offset())))
+ ->from_date_str("2048-11-10", 10);
- *(int8_t*)(tuple->get_slot(slots[10]->tuple_offset())) = -127;
- *(int16_t*)(tuple->get_slot(slots[11]->tuple_offset())) = -32767;
- *(int32_t*)(tuple->get_slot(slots[12]->tuple_offset())) = -2147483647;
- *(int64_t*)(tuple->get_slot(slots[13]->tuple_offset())) = -9223372036854775807L;
+ *(int8_t*)(tuple->get_slot(slots[11]->tuple_offset())) = -127;
+ *(int16_t*)(tuple->get_slot(slots[12]->tuple_offset())) = -32767;
+ *(int32_t*)(tuple->get_slot(slots[13]->tuple_offset())) = -2147483647;
+ *(int64_t*)(tuple->get_slot(slots[14]->tuple_offset())) = -9223372036854775807L;
- memcpy(tuple->get_slot(slots[14]->tuple_offset()), &large_int_value, sizeof(int128_t));
+ memcpy(tuple->get_slot(slots[15]->tuple_offset()), &large_int_value, sizeof(int128_t));
- ((DateTimeValue*)(tuple->get_slot(slots[15]->tuple_offset())))
- ->from_date_str("2048-11-10", 10);
((DateTimeValue*)(tuple->get_slot(slots[16]->tuple_offset())))
+ ->from_date_str("2048-11-10", 10);
+ ((DateTimeValue*)(tuple->get_slot(slots[17]->tuple_offset())))
->from_date_str("2636-08-16 19:39:43", 19);
- char_ptr = (StringValue*)(tuple->get_slot(slots[17]->tuple_offset()));
+ char_ptr = (StringValue*)(tuple->get_slot(slots[18]->tuple_offset()));
char_ptr->ptr = (char*)pool.allocate(4);
memcpy(char_ptr->ptr, "abcd", 4);
char_ptr->len = 4;
- var_ptr = (StringValue*)(tuple->get_slot(slots[18]->tuple_offset()));
+ var_ptr = (StringValue*)(tuple->get_slot(slots[19]->tuple_offset()));
var_ptr->ptr = (char*)pool.allocate(5);
memcpy(var_ptr->ptr, "abcde", 5);
var_ptr->len = 5;
@@ -465,7 +493,10 @@ TEST_F(TestDeltaWriter, write) {
DecimalV2Value val_decimal;
val_decimal.assign_from_double(1.1);
- *(DecimalV2Value*)(tuple->get_slot(slots[19]->tuple_offset())) = val_decimal;
+ *(DecimalV2Value*)(tuple->get_slot(slots[20]->tuple_offset())) = val_decimal;
+
+ ((doris::vectorized::DateV2Value*)(tuple->get_slot(slots[21]->tuple_offset())))
+ ->from_date_str("2048-11-10", 10);
res = delta_writer->write(tuple);
EXPECT_EQ(Status::OK(), res);
@@ -567,36 +598,45 @@ TEST_F(TestDeltaWriter, vec_write) {
decimal_value.assign_from_double(1.1);
columns[9]->insert_data((const char*)&decimal_value, sizeof(decimal_value));
+ doris::vectorized::DateV2Value date_v2;
+ date_v2.from_date_str("2048-11-10", 10);
+ auto date_v2_int = date_v2.to_date_uint32();
+ columns[10]->insert_data((const char*)&date_v2_int, sizeof(date_v2_int));
+
int8_t v1 = -127;
- columns[10]->insert_data((const char*)&v1, sizeof(v1));
+ columns[11]->insert_data((const char*)&v1, sizeof(v1));
int16_t v2 = -32767;
- columns[11]->insert_data((const char*)&v2, sizeof(v2));
+ columns[12]->insert_data((const char*)&v2, sizeof(v2));
int32_t v3 = -2147483647;
- columns[12]->insert_data((const char*)&v3, sizeof(v3));
+ columns[13]->insert_data((const char*)&v3, sizeof(v3));
int64_t v4 = -9223372036854775807L;
- columns[13]->insert_data((const char*)&v4, sizeof(v4));
+ columns[14]->insert_data((const char*)&v4, sizeof(v4));
int128_t v5 = -90000;
- columns[14]->insert_data((const char*)&v5, sizeof(v5));
+ columns[15]->insert_data((const char*)&v5, sizeof(v5));
DateTimeValue v6;
v6.from_date_str("2048-11-10", 10);
auto v6_int = v6.to_int64();
- columns[15]->insert_data((const char*)&v6_int, sizeof(v6_int));
+ columns[16]->insert_data((const char*)&v6_int, sizeof(v6_int));
DateTimeValue v7;
v7.from_date_str("2636-08-16 19:39:43", 19);
auto v7_int = v7.to_int64();
- columns[16]->insert_data((const char*)&v7_int, sizeof(v7_int));
+ columns[17]->insert_data((const char*)&v7_int, sizeof(v7_int));
- columns[17]->insert_data("abcd", 4);
- columns[18]->insert_data("abcde", 5);
+ columns[18]->insert_data("abcd", 4);
+ columns[19]->insert_data("abcde", 5);
decimal_value.assign_from_double(1.1);
- columns[19]->insert_data((const char*)&decimal_value, sizeof(decimal_value));
+ columns[20]->insert_data((const char*)&decimal_value, sizeof(decimal_value));
+
+ date_v2.from_date_str("2048-11-10", 10);
+ date_v2_int = date_v2.to_date_uint32();
+ columns[21]->insert_data((const char*)&date_v2_int, sizeof(date_v2_int));
res = delta_writer->write(&block, {0});
ASSERT_TRUE(res.ok());
@@ -755,6 +795,11 @@ TEST_F(TestDeltaWriter, vec_sequence_col) {
int64_t c4_int = c4.to_int64();
columns[3]->insert_data((const char*)&c4_int, sizeof(c4));
+ doris::vectorized::DateV2Value c5;
+ c5.set_time(2022, 6, 6);
+ uint32_t c5_int = c5.to_date_uint32();
+ columns[4]->insert_data((const char*)&c5_int, sizeof(c5));
+
res = delta_writer->write(&block, {0});
ASSERT_TRUE(res.ok());
}
diff --git a/be/test/olap/in_list_predicate_test.cpp b/be/test/olap/in_list_predicate_test.cpp
index 285e6ea375..00a2280614 100644
--- a/be/test/olap/in_list_predicate_test.cpp
+++ b/be/test/olap/in_list_predicate_test.cpp
@@ -41,6 +41,15 @@ static uint24_t timestamp_from_date(const char* date_string) {
return uint24_t(value);
}
+static uint32_t timestamp_from_date_v2(const char* date_string) {
+ tm time_tm;
+ strptime(date_string, "%Y-%m-%d", &time_tm);
+
+ doris::vectorized::DateV2Value value;
+ value.set_time(time_tm.tm_year, time_tm.tm_mon, time_tm.tm_mday);
+ return binary_cast<doris::vectorized::DateV2Value, uint32_t>(value);
+}
+
static uint64_t timestamp_from_datetime(const std::string& value_string) {
tm time_tm;
strptime(value_string.c_str(), "%Y-%m-%d %H:%M:%S", &time_tm);
@@ -65,6 +74,18 @@ static std::string to_date_string(uint24_t& date_value) {
return std::string(buf);
}
+static std::string to_date_v2_string(uint32_t& date_value) {
+ tm time_tm;
+ uint32_t value = date_value;
+ memset(&time_tm, 0, sizeof(time_tm));
+ time_tm.tm_mday = static_cast<int>(value & 0x000000FF);
+ time_tm.tm_mon = static_cast<int>((value & 0x0000FF00) >> 8);
+ time_tm.tm_year = static_cast<int>((value & 0xFFFF0000) >> 16);
+ char buf[20] = {'\0'};
+ strftime(buf, sizeof(buf), "%Y-%m-%d", &time_tm);
+ return std::string(buf);
+}
+
static std::string to_datetime_string(uint64_t& datetime_value) {
tm time_tm;
int64_t part1 = (datetime_value / 1000000L);
@@ -884,6 +905,115 @@ TEST_F(TestInListPredicate, DATE_COLUMN) {
delete pred;
}
+TEST_F(TestInListPredicate, DATE_V2_COLUMN) {
+ TabletSchema tablet_schema;
+ SetTabletSchema(std::string("DATE_V2_COLUMN"), "DATEV2", "REPLACE", 1, true, true,
+ &tablet_schema);
+ int size = 6;
+ std::vector<uint32_t> return_columns;
+ for (int i = 0; i < tablet_schema.num_columns(); ++i) {
+ return_columns.push_back(i);
+ }
+ phmap::flat_hash_set<uint32_t> values;
+ uint32_t value1 = datetime::timestamp_from_date_v2("2017-09-09");
+ values.insert(value1);
+
+ uint32_t value2 = datetime::timestamp_from_date_v2("2017-09-10");
+ values.insert(value2);
+
+ uint32_t value3 = datetime::timestamp_from_date_v2("2017-09-11");
+ values.insert(value3);
+ ColumnPredicate* pred = new InListPredicate<uint32_t>(0, std::move(values));
+
+ // for VectorizedBatch no nulls
+ InitVectorizedBatch(&tablet_schema, return_columns, size);
+ ColumnVector* col_vector = _vectorized_batch->column(0);
+ col_vector->set_no_nulls(true);
+ uint32_t* col_data = reinterpret_cast<uint32_t*>(_mem_pool->allocate(size * sizeof(uint32_t)));
+ col_vector->set_col_data(col_data);
+
+ std::vector<std::string> date_array;
+ date_array.push_back("2017-09-07");
+ date_array.push_back("2017-09-08");
+ date_array.push_back("2017-09-09");
+ date_array.push_back("2017-09-10");
+ date_array.push_back("2017-09-11");
+ date_array.push_back("2017-09-12");
+ for (int i = 0; i < size; ++i) {
+ uint32_t timestamp = datetime::timestamp_from_date_v2(date_array[i].c_str());
+ *(col_data + i) = timestamp;
+ }
+ pred->evaluate(_vectorized_batch);
+ EXPECT_EQ(_vectorized_batch->size(), 3);
+ uint16_t* sel = _vectorized_batch->selected();
+ EXPECT_EQ(datetime::to_date_v2_string(*(col_data + sel[0])), "2017-09-09");
+ EXPECT_EQ(datetime::to_date_v2_string(*(col_data + sel[1])), "2017-09-10");
+ EXPECT_EQ(datetime::to_date_v2_string(*(col_data + sel[2])), "2017-09-11");
+
+ // for ColumnBlock no nulls
+ init_row_block(&tablet_schema, size);
+ ColumnBlock col_block = _row_block->column_block(0);
+ auto select_size = _row_block->selected_size();
+ ColumnBlockView col_block_view(&col_block);
+ for (int i = 0; i < size; ++i, col_block_view.advance(1)) {
+ col_block_view.set_null_bits(1, false);
+ uint32_t timestamp = datetime::timestamp_from_date_v2(date_array[i].c_str());
+ *reinterpret_cast<uint32_t*>(col_block_view.data()) = timestamp;
+ }
+ pred->evaluate(&col_block, _row_block->selection_vector(), &select_size);
+ EXPECT_EQ(select_size, 3);
+ EXPECT_EQ(datetime::to_date_v2_string(
+ *(uint32_t*)col_block.cell(_row_block->selection_vector()[0]).cell_ptr()),
+ "2017-09-09");
+ EXPECT_EQ(datetime::to_date_v2_string(
+ *(uint32_t*)col_block.cell(_row_block->selection_vector()[1]).cell_ptr()),
+ "2017-09-10");
+ EXPECT_EQ(datetime::to_date_v2_string(
+ *(uint32_t*)col_block.cell(_row_block->selection_vector()[2]).cell_ptr()),
+ "2017-09-11");
+
+ // for VectorizedBatch has nulls
+ col_vector->set_no_nulls(false);
+ bool* is_null = reinterpret_cast<bool*>(_mem_pool->allocate(size));
+ memset(is_null, 0, size);
+ col_vector->set_is_null(is_null);
+ for (int i = 0; i < size; ++i) {
+ if (i % 2 == 0) {
+ is_null[i] = true;
+ } else {
+ uint32_t timestamp = datetime::timestamp_from_date_v2(date_array[i].c_str());
+ *(col_data + i) = timestamp;
+ }
+ }
+ _vectorized_batch->set_size(size);
+ _vectorized_batch->set_selected_in_use(false);
+ pred->evaluate(_vectorized_batch);
+ EXPECT_EQ(_vectorized_batch->size(), 1);
+ sel = _vectorized_batch->selected();
+ EXPECT_EQ(datetime::to_date_v2_string(*(col_data + sel[0])), "2017-09-10");
+
+ // for ColumnBlock has nulls
+ col_block_view = ColumnBlockView(&col_block);
+ for (int i = 0; i < size; ++i, col_block_view.advance(1)) {
+ if (i % 2 == 0) {
+ col_block_view.set_null_bits(1, true);
+ } else {
+ col_block_view.set_null_bits(1, false);
+ uint32_t timestamp = datetime::timestamp_from_date_v2(date_array[i].c_str());
+ *reinterpret_cast<uint32_t*>(col_block_view.data()) = timestamp;
+ }
+ }
+ _row_block->clear();
+ select_size = _row_block->selected_size();
+ pred->evaluate(&col_block, _row_block->selection_vector(), &select_size);
+ EXPECT_EQ(select_size, 1);
+ EXPECT_EQ(datetime::to_date_v2_string(
+ *(uint32_t*)col_block.cell(_row_block->selection_vector()[0]).cell_ptr()),
+ "2017-09-10");
+
+ delete pred;
+}
+
TEST_F(TestInListPredicate, DATETIME_COLUMN) {
TabletSchema tablet_schema;
SetTabletSchema(std::string("DATETIME_COLUMN"), "DATETIME", "REPLACE", 1, true, true,
diff --git a/be/test/olap/null_predicate_test.cpp b/be/test/olap/null_predicate_test.cpp
index 47bf02306f..1fe189faf6 100644
--- a/be/test/olap/null_predicate_test.cpp
+++ b/be/test/olap/null_predicate_test.cpp
@@ -45,6 +45,13 @@ static uint24_t to_date_timestamp(const char* date_string) {
return uint24_t(value);
}
+static uint32_t to_date_v2_timestamp(const char* date_string) {
+ tm time_tm;
+ strptime(date_string, "%Y-%m-%d", &time_tm);
+
+ return ((time_tm.tm_year + 1900) << 16) | ((time_tm.tm_mon + 1) << 8) | time_tm.tm_mday;
+}
+
static uint64_t to_datetime_timestamp(const std::string& value_string) {
tm time_tm;
strptime(value_string.c_str(), "%Y-%m-%d %H:%M:%S", &time_tm);
@@ -788,4 +795,104 @@ TEST_F(TestNullPredicate, DATETIME_COLUMN) {
EXPECT_EQ(select_size, 2);
}
+TEST_F(TestNullPredicate, DATEV2_COLUMN) {
+ TabletSchema tablet_schema;
+ SetTabletSchema(std::string("DATEV2_COLUMN"), "DATEV2", "REPLACE", 4, true, true,
+ &tablet_schema);
+ int size = 6;
+ std::vector<uint32_t> return_columns;
+ for (int i = 0; i < tablet_schema.num_columns(); ++i) {
+ return_columns.push_back(i);
+ }
+ std::unique_ptr<ColumnPredicate> pred(new NullPredicate(0, true));
+
+ // for VectorizedBatch no nulls
+ InitVectorizedBatch(&tablet_schema, return_columns, size);
+ ColumnVector* col_vector = _vectorized_batch->column(0);
+ col_vector->set_no_nulls(true);
+ uint32_t* col_data = reinterpret_cast<uint32_t*>(_mem_pool->allocate(size * sizeof(uint32_t)));
+ col_vector->set_col_data(col_data);
+
+ std::vector<std::string> date_array;
+ date_array.push_back("2017-09-07");
+ date_array.push_back("2017-09-08");
+ date_array.push_back("2017-09-09");
+ date_array.push_back("2017-09-10");
+ date_array.push_back("2017-09-11");
+ date_array.push_back("2017-09-12");
+ for (int i = 0; i < size; ++i) {
+ uint32_t timestamp = datetime::to_date_v2_timestamp(date_array[i].c_str());
+ *(col_data + i) = timestamp;
+ }
+ pred->evaluate(_vectorized_batch);
+ EXPECT_EQ(_vectorized_batch->size(), 0);
+
+ // for ColumnBlock no nulls
+ init_row_block(&tablet_schema, size);
+ ColumnBlock col_block = _row_block->column_block(0);
+ auto select_size = _row_block->selected_size();
+ ColumnBlockView col_block_view(&col_block);
+ for (int i = 0; i < size; ++i, col_block_view.advance(1)) {
+ col_block_view.set_null_bits(1, false);
+ uint32_t timestamp = datetime::to_date_v2_timestamp(date_array[i].c_str());
+ *reinterpret_cast<uint32_t*>(col_block_view.data()) = timestamp;
+ }
+ pred->evaluate(&col_block, _row_block->selection_vector(), &select_size);
+ EXPECT_EQ(select_size, 0);
+
+ // for vectorized::Block no null
+ _row_block->clear();
+ select_size = _row_block->selected_size();
+ vectorized::Block vec_block = tablet_schema.create_block(return_columns);
+ _row_block->convert_to_vec_block(&vec_block);
+ ColumnPtr vec_col = vec_block.get_columns()[0];
+ select_size = pred->evaluate(const_cast<doris::vectorized::IColumn&>(*vec_col),
+ _row_block->selection_vector(), select_size);
+ EXPECT_EQ(select_size, 0);
+
+ // for VectorizedBatch has nulls
+ col_vector->set_no_nulls(false);
+ bool* is_null = reinterpret_cast<bool*>(_mem_pool->allocate(size));
+ memset(is_null, 0, size);
+ col_vector->set_is_null(is_null);
+ for (int i = 0; i < size; ++i) {
+ if (i % 3 == 0) {
+ is_null[i] = true;
+ } else {
+ uint32_t timestamp = datetime::to_date_v2_timestamp(date_array[i].c_str());
+ *(col_data + i) = timestamp;
+ }
+ }
+ _vectorized_batch->set_size(size);
+ _vectorized_batch->set_selected_in_use(false);
+ pred->evaluate(_vectorized_batch);
+ EXPECT_EQ(_vectorized_batch->size(), 2);
+
+ // for ColumnBlock has nulls
+ col_block_view = ColumnBlockView(&col_block);
+ for (int i = 0; i < size; ++i, col_block_view.advance(1)) {
+ if (i % 3 == 0) {
+ col_block_view.set_null_bits(1, true);
+ } else {
+ col_block_view.set_null_bits(1, false);
+ uint32_t timestamp = datetime::to_date_v2_timestamp(date_array[i].c_str());
+ *reinterpret_cast<uint32_t*>(col_block_view.data()) = timestamp;
+ }
+ }
+ _row_block->clear();
+ select_size = _row_block->selected_size();
+ pred->evaluate(&col_block, _row_block->selection_vector(), &select_size);
+ EXPECT_EQ(select_size, 2);
+
+ // for vectorized::Block has nulls
+ _row_block->clear();
+ select_size = _row_block->selected_size();
+ vec_block = tablet_schema.create_block(return_columns);
+ _row_block->convert_to_vec_block(&vec_block);
+ vec_col = vec_block.get_columns()[0];
+ select_size = pred->evaluate(const_cast<doris::vectorized::IColumn&>(*vec_col),
+ _row_block->selection_vector(), select_size);
+ EXPECT_EQ(select_size, 2);
+}
+
} // namespace doris
diff --git a/be/test/olap/row_cursor_test.cpp b/be/test/olap/row_cursor_test.cpp
index cdabe13387..96722c77d7 100644
--- a/be/test/olap/row_cursor_test.cpp
+++ b/be/test/olap/row_cursor_test.cpp
@@ -79,71 +79,80 @@ void set_tablet_schema_for_init(TabletSchema* tablet_schema) {
column_5->set_index_length(8);
ColumnPB* column_6 = tablet_schema_pb.add_column();
- column_6->set_unique_id(6);
+ column_6->set_unique_id(5);
column_6->set_name("column_6");
- column_6->set_type("DECIMAL");
+ column_6->set_type("DATEV2");
column_6->set_is_key(true);
column_6->set_is_nullable(true);
- column_6->set_length(12);
- column_6->set_index_length(12);
- column_6->set_frac(3);
- column_6->set_precision(6);
+ column_6->set_length(4);
+ column_6->set_index_length(4);
ColumnPB* column_7 = tablet_schema_pb.add_column();
- column_7->set_unique_id(7);
+ column_7->set_unique_id(6);
column_7->set_name("column_7");
- column_7->set_type("CHAR");
+ column_7->set_type("DECIMAL");
column_7->set_is_key(true);
column_7->set_is_nullable(true);
- column_7->set_length(4);
- column_7->set_index_length(4);
- column_7->set_default_value("char");
+ column_7->set_length(12);
+ column_7->set_index_length(12);
+ column_7->set_frac(3);
+ column_7->set_precision(6);
ColumnPB* column_8 = tablet_schema_pb.add_column();
- column_8->set_unique_id(8);
+ column_8->set_unique_id(7);
column_8->set_name("column_8");
- column_8->set_type("BIGINT");
+ column_8->set_type("CHAR");
+ column_8->set_is_key(true);
column_8->set_is_nullable(true);
- column_8->set_length(8);
- column_8->set_aggregation("SUM");
- column_8->set_is_key(false);
+ column_8->set_length(4);
+ column_8->set_index_length(4);
+ column_8->set_default_value("char");
ColumnPB* column_9 = tablet_schema_pb.add_column();
- column_9->set_unique_id(9);
+ column_9->set_unique_id(8);
column_9->set_name("column_9");
- column_9->set_type("VARCHAR");
+ column_9->set_type("BIGINT");
column_9->set_is_nullable(true);
- column_9->set_length(16 + OLAP_VARCHAR_MAX_BYTES);
- column_9->set_aggregation("REPLACE");
+ column_9->set_length(8);
+ column_9->set_aggregation("SUM");
column_9->set_is_key(false);
ColumnPB* column_10 = tablet_schema_pb.add_column();
- column_10->set_unique_id(10);
+ column_10->set_unique_id(9);
column_10->set_name("column_10");
- column_10->set_type("LARGEINT");
+ column_10->set_type("VARCHAR");
column_10->set_is_nullable(true);
- column_10->set_length(16);
- column_10->set_aggregation("MAX");
+ column_10->set_length(16 + OLAP_VARCHAR_MAX_BYTES);
+ column_10->set_aggregation("REPLACE");
column_10->set_is_key(false);
ColumnPB* column_11 = tablet_schema_pb.add_column();
- column_11->set_unique_id(11);
+ column_11->set_unique_id(10);
column_11->set_name("column_11");
- column_11->set_type("DECIMAL");
+ column_11->set_type("LARGEINT");
column_11->set_is_nullable(true);
- column_11->set_length(12);
- column_11->set_aggregation("MIN");
+ column_11->set_length(16);
+ column_11->set_aggregation("MAX");
column_11->set_is_key(false);
ColumnPB* column_12 = tablet_schema_pb.add_column();
- column_12->set_unique_id(12);
+ column_12->set_unique_id(11);
column_12->set_name("column_12");
- column_12->set_type("HLL");
+ column_12->set_type("DECIMAL");
column_12->set_is_nullable(true);
- column_12->set_length(HLL_COLUMN_DEFAULT_LEN);
- column_12->set_aggregation("HLL_UNION");
+ column_12->set_length(12);
+ column_12->set_aggregation("MIN");
column_12->set_is_key(false);
+ ColumnPB* column_13 = tablet_schema_pb.add_column();
+ column_13->set_unique_id(12);
+ column_13->set_name("column_13");
+ column_13->set_type("HLL");
+ column_13->set_is_nullable(true);
+ column_13->set_length(HLL_COLUMN_DEFAULT_LEN);
+ column_13->set_aggregation("HLL_UNION");
+ column_13->set_is_key(false);
+
tablet_schema->init_from_pb(tablet_schema_pb);
}
@@ -272,7 +281,7 @@ TEST_F(TestRowCursor, InitRowCursor) {
RowCursor row;
Status res = row.init(tablet_schema);
EXPECT_EQ(res, Status::OK());
- EXPECT_EQ(row.get_fixed_len(), 126);
+ EXPECT_EQ(row.get_fixed_len(), 131);
EXPECT_EQ(row.get_variable_len(), 20);
}
@@ -300,8 +309,8 @@ TEST_F(TestRowCursor, InitRowCursorWithColIds) {
RowCursor row;
Status res = row.init(tablet_schema, col_ids);
EXPECT_EQ(res, Status::OK());
- EXPECT_EQ(row.get_fixed_len(), 63);
- EXPECT_EQ(row.get_variable_len(), 20);
+ EXPECT_EQ(row.get_fixed_len(), 55);
+ EXPECT_EQ(row.get_variable_len(), 0);
}
TEST_F(TestRowCursor, InitRowCursorWithScanKey) {
diff --git a/be/test/olap/schema_change_test.cpp b/be/test/olap/schema_change_test.cpp
index 019c7635f8..e9383f5763 100644
--- a/be/test/olap/schema_change_test.cpp
+++ b/be/test/olap/schema_change_test.cpp
@@ -407,6 +407,96 @@ TEST_F(TestColumn, ConvertDatetimeToDate) {
EXPECT_TRUE(st == Status::OLAPInternalError(OLAP_ERR_INVALID_SCHEMA));
}
+TEST_F(TestColumn, ConvertDatetimeToDateV2) {
+ TabletSchema tablet_schema;
+ set_tablet_schema("DatetimeColumn", "DATETIME", "REPLACE", 8, false, false, &tablet_schema);
+ create_column_writer(tablet_schema);
+
+ RowCursor write_row;
+ write_row.init(tablet_schema);
+ RowBlock block(&tablet_schema);
+ RowBlockInfo block_info;
+ block_info.row_num = 10000;
+ block.init(block_info);
+
+ std::vector<std::string> val_string_array;
+ std::string origin_val = "2019-11-25 19:07:00";
+ val_string_array.emplace_back(origin_val);
+ OlapTuple tuple(val_string_array);
+ write_row.from_tuple(tuple);
+ block.set_row(0, write_row);
+ block.finalize(1);
+ EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK());
+
+ ColumnDataHeaderMessage header;
+ EXPECT_EQ(_column_writer->finalize(&header), Status::OK());
+
+ // read data
+ TabletSchema convert_tablet_schema;
+ set_tablet_schema("DateV2Column", "DATEV2", "REPLACE", 4, false, false, &convert_tablet_schema);
+ create_column_reader(tablet_schema);
+ RowCursor read_row;
+ read_row.init(convert_tablet_schema);
+
+ _col_vector.reset(new ColumnVector());
+ EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), Status::OK());
+ char* data = reinterpret_cast<char*>(_col_vector->col_data());
+ read_row.convert_from(0, data, write_row.column_schema(0)->type_info(), _mem_pool.get());
+ std::string dest_string = read_row.column_schema(0)->to_string(read_row.cell_ptr(0));
+ EXPECT_TRUE(strncmp(dest_string.c_str(), "2019-11-25", strlen("2019-11-25")) == 0);
+
+ //test not support type
+ const auto* tp = get_scalar_type_info<OLAP_FIELD_TYPE_HLL>();
+ Status st = read_row.convert_from(0, data, tp, _mem_pool.get());
+ EXPECT_TRUE(st == Status::OLAPInternalError(OLAP_ERR_INVALID_SCHEMA));
+}
+
+TEST_F(TestColumn, ConvertDateV2ToDateTime) {
+ TabletSchema tablet_schema;
+ set_tablet_schema("DateV2Column", "DATEV2", "REPLACE", 4, false, false, &tablet_schema);
+ create_column_writer(tablet_schema);
+
+ RowCursor write_row;
+ write_row.init(tablet_schema);
+ RowBlock block(&tablet_schema);
+ RowBlockInfo block_info;
+ block_info.row_num = 10000;
+ block.init(block_info);
+
+ std::vector<std::string> val_string_array;
+ std::string origin_val = "2019-11-25";
+ val_string_array.emplace_back(origin_val);
+ OlapTuple tuple(val_string_array);
+ write_row.from_tuple(tuple);
+ block.set_row(0, write_row);
+ block.finalize(1);
+ EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK());
+
+ ColumnDataHeaderMessage header;
+ EXPECT_EQ(_column_writer->finalize(&header), Status::OK());
+
+ // read data
+ TabletSchema convert_tablet_schema;
+ set_tablet_schema("DateTimeColumn", "DATETIME", "REPLACE", 8, false, false,
+ &convert_tablet_schema);
+ create_column_reader(tablet_schema);
+ RowCursor read_row;
+ read_row.init(convert_tablet_schema);
+
+ _col_vector.reset(new ColumnVector());
+ EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), Status::OK());
+ char* data = reinterpret_cast<char*>(_col_vector->col_data());
+ read_row.convert_from(0, data, write_row.column_schema(0)->type_info(), _mem_pool.get());
+ std::string dest_string = read_row.column_schema(0)->to_string(read_row.cell_ptr(0));
+ EXPECT_TRUE(strncmp(dest_string.c_str(), "2019-11-25 00:00:00",
+ strlen("2019-11-25 00:00:00")) == 0);
+
+ //test not support type
+ const auto* tp = get_scalar_type_info<OLAP_FIELD_TYPE_HLL>();
+ Status st = read_row.convert_from(0, data, tp, _mem_pool.get());
+ EXPECT_TRUE(st == Status::OLAPInternalError(OLAP_ERR_INVALID_SCHEMA));
+}
+
TEST_F(TestColumn, ConvertDateToDatetime) {
TabletSchema tablet_schema;
set_tablet_schema("DateColumn", "DATE", "REPLACE", 3, false, false, &tablet_schema);
@@ -452,6 +542,96 @@ TEST_F(TestColumn, ConvertDateToDatetime) {
EXPECT_TRUE(st == Status::OLAPInternalError(OLAP_ERR_INVALID_SCHEMA));
}
+TEST_F(TestColumn, ConvertDateToDateV2) {
+ TabletSchema tablet_schema;
+ set_tablet_schema("DateColumn", "DATE", "REPLACE", 3, false, false, &tablet_schema);
+ create_column_writer(tablet_schema);
+
+ RowCursor write_row;
+ write_row.init(tablet_schema);
+
+ RowBlock block(&tablet_schema);
+ RowBlockInfo block_info;
+ block_info.row_num = 10000;
+ block.init(block_info);
+
+ std::vector<std::string> val_string_array;
+ std::string origin_val = "2019-12-04";
+ val_string_array.emplace_back(origin_val);
+ OlapTuple tuple(val_string_array);
+ write_row.from_tuple(tuple);
+ block.set_row(0, write_row);
+ block.finalize(1);
+ EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK());
+
+ ColumnDataHeaderMessage header_message;
+ EXPECT_EQ(_column_writer->finalize(&header_message), Status::OK());
+
+ TabletSchema convert_tablet_schema;
+ set_tablet_schema("DateV2Column", "DATEV2", "REPLACE", 4, false, false, &convert_tablet_schema);
+ create_column_reader(tablet_schema);
+ RowCursor read_row;
+ read_row.init(convert_tablet_schema);
+ _col_vector.reset(new ColumnVector());
+ EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), Status::OK());
+ char* data = reinterpret_cast<char*>(_col_vector->col_data());
+ read_row.set_field_content(0, data, _mem_pool.get());
+ read_row.convert_from(0, data, write_row.column_schema(0)->type_info(), _mem_pool.get());
+ std::string expected = "2019-12-04";
+ std::string dest_string =
+ read_row.column_schema(0)->to_string(read_row.cell_ptr(0)).substr(0, expected.length());
+ EXPECT_TRUE(dest_string.compare("2019-12-04") == 0);
+
+ //test not support type
+ const auto* tp = get_scalar_type_info<OLAP_FIELD_TYPE_HLL>();
+ Status st = read_row.convert_from(0, data, tp, _mem_pool.get());
+ EXPECT_TRUE(st == Status::OLAPInternalError(OLAP_ERR_INVALID_SCHEMA));
+}
+
+TEST_F(TestColumn, ConvertDateV2ToDate) {
+ TabletSchema tablet_schema;
+ set_tablet_schema("DateV2Column", "DATEV2", "REPLACE", 4, false, false, &tablet_schema);
+ create_column_writer(tablet_schema);
+
+ RowCursor write_row;
+ write_row.init(tablet_schema);
+
+ RowBlock block(&tablet_schema);
+ RowBlockInfo block_info;
+ block_info.row_num = 10000;
+ block.init(block_info);
+
+ std::vector<std::string> val_string_array;
+ std::string origin_val = "2019-12-04";
+ val_string_array.emplace_back(origin_val);
+ OlapTuple tuple(val_string_array);
+ write_row.from_tuple(tuple);
+ block.set_row(0, write_row);
+ block.finalize(1);
+ EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK());
+
+ ColumnDataHeaderMessage header_message;
+ EXPECT_EQ(_column_writer->finalize(&header_message), Status::OK());
+
+ TabletSchema convert_tablet_schema;
+ set_tablet_schema("DateColumn", "DATE", "REPLACE", 3, false, false, &convert_tablet_schema);
+ create_column_reader(tablet_schema);
+ RowCursor read_row;
+ read_row.init(convert_tablet_schema);
+ _col_vector.reset(new ColumnVector());
+ EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), Status::OK());
+ char* data = reinterpret_cast<char*>(_col_vector->col_data());
+ read_row.set_field_content(0, data, _mem_pool.get());
+ read_row.convert_from(0, data, write_row.column_schema(0)->type_info(), _mem_pool.get());
+ std::string dest_string = read_row.column_schema(0)->to_string(read_row.cell_ptr(0));
+ EXPECT_TRUE(dest_string.compare("2019-12-04") == 0);
+
+ //test not support type
+ const auto* tp = get_scalar_type_info<OLAP_FIELD_TYPE_HLL>();
+ Status st = read_row.convert_from(0, data, tp, _mem_pool.get());
+ EXPECT_TRUE(st == Status::OLAPInternalError(OLAP_ERR_INVALID_SCHEMA));
+}
+
TEST_F(TestColumn, ConvertIntToDate) {
TabletSchema tablet_schema;
set_tablet_schema("IntColumn", "INT", "REPLACE", 4, false, false, &tablet_schema);
@@ -494,6 +674,48 @@ TEST_F(TestColumn, ConvertIntToDate) {
EXPECT_TRUE(st == Status::OLAPInternalError(OLAP_ERR_INVALID_SCHEMA));
}
+TEST_F(TestColumn, ConvertIntToDateV2) {
+ TabletSchema tablet_schema;
+ set_tablet_schema("IntColumn", "INT", "REPLACE", 4, false, false, &tablet_schema);
+ create_column_writer(tablet_schema);
+
+ RowCursor write_row;
+ write_row.init(tablet_schema);
+
+ RowBlock block(&tablet_schema);
+ RowBlockInfo block_info;
+ block_info.row_num = 10000;
+ block.init(block_info);
+
+ int time_val = 20191205;
+ write_row.set_field_content(0, reinterpret_cast<char*>(&time_val), _mem_pool.get());
+ block.set_row(0, write_row);
+ block.finalize(1);
+ EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK());
+
+ ColumnDataHeaderMessage header;
+ EXPECT_EQ(_column_writer->finalize(&header), Status::OK());
+
+ TabletSchema convert_tablet_schema;
+ set_tablet_schema("DateV2Column", "DATEV2", "REPLACE", 4, false, false, &convert_tablet_schema);
+ create_column_reader(tablet_schema);
+
+ RowCursor read_row;
+ read_row.init(convert_tablet_schema);
+
+ _col_vector.reset(new ColumnVector());
+ EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), Status::OK());
+ char* data = reinterpret_cast<char*>(_col_vector->col_data());
+ read_row.convert_from(0, data, write_row.column_schema(0)->type_info(), _mem_pool.get());
+ std::string dest_string = read_row.column_schema(0)->to_string(read_row.cell_ptr(0));
+ EXPECT_TRUE(strncmp(dest_string.c_str(), "2019-12-05", strlen("2019-12-05")) == 0);
+
+ //test not support type
+ const auto* tp = get_scalar_type_info<OLAP_FIELD_TYPE_HLL>();
+ Status st = read_row.convert_from(0, read_row.cell_ptr(0), tp, _mem_pool.get());
+ EXPECT_TRUE(st == Status::OLAPInternalError(OLAP_ERR_INVALID_SCHEMA));
+}
+
TEST_F(TestColumn, ConvertVarcharToDate) {
TabletSchema tablet_schema;
set_tablet_schema("VarcharColumn", "VARCHAR", "REPLACE", 255, false, false, &tablet_schema);
@@ -549,6 +771,64 @@ TEST_F(TestColumn, ConvertVarcharToDate) {
EXPECT_EQ(st, Status::OLAPInternalError(OLAP_ERR_INVALID_SCHEMA));
}
+TEST_F(TestColumn, ConvertVarcharToDateV2) {
+ TabletSchema tablet_schema;
+ set_tablet_schema("VarcharColumn", "VARCHAR", "REPLACE", 255, false, false, &tablet_schema);
+ create_column_writer(tablet_schema);
+
+ RowCursor write_row;
+ write_row.init(tablet_schema);
+
+ RowBlock block(&tablet_schema);
+ RowBlockInfo block_info;
+ block_info.row_num = 10000;
+ block.init(block_info);
+
+ // test valid format convert
+ std::vector<Slice> valid_src_strs = {
+ "2019-12-17", "19-12-17", "20191217", "191217", "2019/12/17", "19/12/17",
+ };
+ std::string expected_val("2019-12-17");
+ for (auto src_str : valid_src_strs) {
+ write_row.set_field_content(0, reinterpret_cast<char*>(&src_str), _mem_pool.get());
+ block.set_row(0, write_row);
+ block.finalize(1);
+ EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK());
+
+ ColumnDataHeaderMessage header;
+ EXPECT_EQ(_column_writer->finalize(&header), Status::OK());
+
+ // because file_helper is reused in this case, we should close it.
+ helper.close();
+ TabletSchema convert_tablet_schema;
+ set_tablet_schema("DateV2Column", "DATEV2", "REPLACE", 3, false, false,
+ &convert_tablet_schema);
+ create_column_reader(tablet_schema);
+ RowCursor read_row;
+ read_row.init(convert_tablet_schema);
+
+ _col_vector.reset(new ColumnVector());
+ EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), Status::OK());
+ char* data = reinterpret_cast<char*>(_col_vector->col_data());
+ read_row.convert_from(0, data, write_row.column_schema(0)->type_info(), _mem_pool.get());
+ std::string dst_str = read_row.column_schema(0)
+ ->to_string(read_row.cell_ptr(0))
+ .substr(0, expected_val.length());
+ EXPECT_EQ(expected_val, dst_str);
+ }
+ helper.close();
+ TabletSchema convert_tablet_schema;
+ set_tablet_schema("DateV2Column", "DATEV2", "REPLACE", 3, false, false, &convert_tablet_schema);
+ create_column_reader(tablet_schema);
+ RowCursor read_row;
+ read_row.init(convert_tablet_schema);
+
+ //test not support type
+ const auto* tp = get_scalar_type_info<OLAP_FIELD_TYPE_HLL>();
+ Status st = read_row.convert_from(0, read_row.cell_ptr(0), tp, _mem_pool.get());
+ EXPECT_EQ(st, Status::OLAPInternalError(OLAP_ERR_INVALID_SCHEMA));
+}
+
TEST_F(TestColumn, ConvertVarcharToTinyInt1) {
test_convert_from_varchar("TINYINT", 1, "127", Status::OK());
}
diff --git a/be/test/olap/storage_types_test.cpp b/be/test/olap/storage_types_test.cpp
index 09e7ecc8ea..ea1bd756d1 100644
--- a/be/test/olap/storage_types_test.cpp
+++ b/be/test/olap/storage_types_test.cpp
@@ -140,6 +140,8 @@ TEST(TypesTest, copy_and_equal) {
common_test<OLAP_FIELD_TYPE_DATE>((1988 << 9) | (2 << 5) | 1);
common_test<OLAP_FIELD_TYPE_DATETIME>(19880201010203L);
+ common_test<OLAP_FIELD_TYPE_DATEV2>((1988 << 16) | (2 << 8) | 1);
+
Slice slice("12345abcde");
common_test<OLAP_FIELD_TYPE_CHAR>(slice);
common_test<OLAP_FIELD_TYPE_VARCHAR>(slice);
@@ -213,6 +215,10 @@ TEST(ArrayTypeTest, copy_and_equal) {
(2008 << 9) | (2 << 5) | 1};
common_test_array<OLAP_FIELD_TYPE_DATE>(CollectionValue(date_array, 3, null_signs));
+ uint32_t date_v2_array[3] = {(1988 << 16) | (2 << 8) | 1, (1998 << 16) | (2 << 8) | 1,
+ (2008 << 16) | (2 << 8) | 1};
+ common_test_array<OLAP_FIELD_TYPE_DATEV2>(CollectionValue(date_v2_array, 3, null_signs));
+
int64_t datetime_array[3] = {19880201010203L, 19980201010203L, 20080204010203L};
common_test_array<OLAP_FIELD_TYPE_DATETIME>(CollectionValue(datetime_array, 3, null_signs));
diff --git a/be/test/vec/core/block_test.cpp b/be/test/vec/core/block_test.cpp
index fb9d2be897..d20842b017 100644
--- a/be/test/vec/core/block_test.cpp
+++ b/be/test/vec/core/block_test.cpp
@@ -54,7 +54,8 @@ TEST(BlockTest, RowBatchCovertToBlock) {
{"k4", TYPE_VARCHAR, sizeof(StringValue), false},
{"k5", TYPE_DECIMALV2, sizeof(DecimalV2Value), false},
{"k6", TYPE_LARGEINT, sizeof(__int128), false},
- {"k7", TYPE_DATETIME, sizeof(int64_t), false}};
+ {"k7", TYPE_DATETIME, sizeof(int64_t), false},
+ {"k8", TYPE_DATEV2, sizeof(uint32_t), false}};
SchemaScanner schema_scanner(column_descs,
sizeof(column_descs) / sizeof(SchemaScanner::ColumnDesc));
@@ -111,6 +112,13 @@ TEST(BlockTest, RowBatchCovertToBlock) {
k7.date_add_interval(time_interval, vectorized::TimeUnit::DAY);
memcpy(tuple->get_slot(slot_desc->tuple_offset()), &k7, column_descs[6].size);
+ slot_desc = tuple_desc->slots()[7];
+ vectorized::DateV2Value k8;
+ std::string now_date("2020-12-02");
+ k8.from_date_str(now_date.c_str(), now_date.size());
+ k8.date_add_interval(time_interval, vectorized::TimeUnit::DAY);
+ memcpy(tuple->get_slot(slot_desc->tuple_offset()), &k8, column_descs[7].size);
+
tuple_row->set_tuple(0, tuple);
row_batch.commit_last_row();
}
@@ -127,6 +135,7 @@ TEST(BlockTest, RowBatchCovertToBlock) {
vectorized::ColumnPtr column5 = block.get_columns()[4];
vectorized::ColumnPtr column6 = block.get_columns()[5];
vectorized::ColumnPtr column7 = block.get_columns()[6];
+ vectorized::ColumnPtr column8 = block.get_columns()[7];
if (i % 5 != 0) {
EXPECT_EQ((int16_t)column1->get64(i), k1);
@@ -155,6 +164,16 @@ TEST(BlockTest, RowBatchCovertToBlock) {
EXPECT_EQ(k7, date_time_value);
+ larget_int = column8->operator[](i).get<vectorized::UInt32>();
+ vectorized::DateV2Value k8;
+ memcpy(reinterpret_cast<vectorized::Int128*>(&k8), &larget_int, column_descs[7].size);
+ vectorized::DateV2Value date_v2_value;
+ std::string now_date("2020-12-02");
+ date_v2_value.from_date_str(now_date.c_str(), now_date.size());
+ date_v2_value.date_add_interval(time_interval, vectorized::TimeUnit::DAY);
+
+ EXPECT_EQ(k8, date_v2_value);
+
k1++;
k3 += 0.1;
}
@@ -393,8 +412,19 @@ TEST(BlockTest, dump_data) {
vectorized::ColumnWithTypeAndName test_datetime(column_vector_datetime->get_ptr(),
datetime_type, "test_datetime");
- vectorized::Block block(
- {test_int, test_string, test_decimal, test_nullable_int32, test_date, test_datetime});
+ auto column_vector_date_v2 = vectorized::ColumnVector<vectorized::UInt32>::create();
+ auto& date_v2_data = column_vector_date_v2->get_data();
+ for (int i = 0; i < 1024; ++i) {
+ vectorized::DateV2Value value;
+ value.from_date((uint32_t)((2022 << 16) | (6 << 8) | 6));
+ date_v2_data.push_back(*reinterpret_cast<vectorized::UInt32*>(&value));
+ }
+ vectorized::DataTypePtr date_v2_type(std::make_shared<vectorized::DataTypeDateV2>());
+ vectorized::ColumnWithTypeAndName test_date_v2(column_vector_date_v2->get_ptr(), date_v2_type,
+ "test_datev2");
+
+ vectorized::Block block({test_int, test_string, test_decimal, test_nullable_int32, test_date,
+ test_datetime, test_date_v2});
EXPECT_GT(block.dump_data().size(), 1);
}
} // namespace doris
diff --git a/be/test/vec/exprs/vexpr_test.cpp b/be/test/vec/exprs/vexpr_test.cpp
index d8e8a6d635..4ece5fd792 100644
--- a/be/test/vec/exprs/vexpr_test.cpp
+++ b/be/test/vec/exprs/vexpr_test.cpp
@@ -188,6 +188,13 @@ struct literal_traits<TYPE_DATETIME> {
using CXXType = std::string;
};
+template <>
+struct literal_traits<TYPE_DATEV2> {
+ const static TPrimitiveType::type ttype = TPrimitiveType::DATEV2;
+ const static TExprNodeType::type tnode_type = TExprNodeType::STRING_LITERAL;
+ using CXXType = std::string;
+};
+
template <>
struct literal_traits<TYPE_DECIMALV2> {
const static TPrimitiveType::type ttype = TPrimitiveType::DECIMALV2;
@@ -225,6 +232,14 @@ void set_literal(TExprNode& node, const U& value) {
node.__set_date_literal(date_literal);
}
+template <PrimitiveType T, class U = typename literal_traits<T>::CXXType,
+ std::enable_if_t<T == TYPE_DATEV2, bool> = true>
+void set_literal(TExprNode& node, const U& value) {
+ TDateLiteral date_literal;
+ date_literal.__set_value(value);
+ node.__set_date_literal(date_literal);
+}
+
template <PrimitiveType T, class U = typename literal_traits<T>::CXXType,
std::enable_if_t<std::numeric_limits<U>::is_iec559, bool> = true>
void set_literal(TExprNode& node, const U& value) {
@@ -342,6 +357,20 @@ TEST(TEST_VEXPR, LITERALTEST) {
auto v = (*ctn.column)[0].get<__int64_t>();
EXPECT_EQ(v, dt);
}
+ {
+ vectorized::DateV2Value data_time_value;
+ const char* date = "20210407";
+ data_time_value.from_date_str(date, strlen(date));
+ uint32_t dt;
+ memcpy(&dt, &data_time_value, sizeof(uint32_t));
+ VLiteral literal(create_literal<TYPE_DATEV2, std::string>(std::string(date)));
+ Block block;
+ int ret = -1;
+ literal.execute(nullptr, &block, &ret);
+ auto ctn = block.safe_get_by_position(ret);
+ auto v = (*ctn.column)[0].get<uint32_t>();
+ EXPECT_EQ(v, dt);
+ }
{
VLiteral literal(create_literal<TYPE_DECIMALV2, std::string>(std::string("1234.56")));
Block block;
diff --git a/be/test/vec/function/function_test_util.cpp b/be/test/vec/function/function_test_util.cpp
index 0e25aa1781..32e738b349 100644
--- a/be/test/vec/function/function_test_util.cpp
+++ b/be/test/vec/function/function_test_util.cpp
@@ -32,6 +32,14 @@ int64_t str_to_date_time(std::string datetime_str, bool data_time) {
}
return binary_cast<VecDateTimeValue, Int64>(v);
}
+
+uint32_t str_to_date_v2(std::string datetime_str, std::string datetime_format) {
+ DateV2Value v;
+ v.from_date_format_str(datetime_format.c_str(), datetime_format.size(), datetime_str.c_str(),
+ datetime_str.size());
+ return binary_cast<DateV2Value, UInt32>(v);
+}
+
size_t type_index_to_data_type(const std::vector<std::any>& input_types, size_t index,
ut_type::UTDataTypeDesc& ut_desc, DataTypePtr& type) {
doris_udf::FunctionContext::TypeDesc& desc = ut_desc.type_desc;
@@ -95,6 +103,14 @@ size_t type_index_to_data_type(const std::vector<std::any>& input_types, size_t
desc.type = doris_udf::FunctionContext::TYPE_DATE;
type = std::make_shared<DataTypeDate>();
return 1;
+ case TypeIndex::DateV2:
+ desc.type = doris_udf::FunctionContext::TYPE_DATEV2;
+ type = std::make_shared<DataTypeDateV2>();
+ return 1;
+ case TypeIndex::DateTimeV2:
+ desc.type = doris_udf::FunctionContext::TYPE_DATETIMEV2;
+ type = std::make_shared<DataTypeDateV2>();
+ return 1;
case TypeIndex::Array: {
desc.type = doris_udf::FunctionContext::TYPE_ARRAY;
ut_type::UTDataTypeDesc sub_desc;
@@ -200,6 +216,13 @@ bool insert_cell(MutableColumnPtr& column, DataTypePtr type_ptr, const std::any&
datetime_str.c_str(), datetime_str.size());
v.cast_to_date();
column->insert_data(reinterpret_cast<char*>(&v), 0);
+ } else if (type.is_date_v2()) {
+ static std::string date_time_format("%Y-%m-%d");
+ auto datetime_str = std::any_cast<std::string>(cell);
+ DateV2Value v;
+ v.from_date_format_str(date_time_format.c_str(), date_time_format.size(),
+ datetime_str.c_str(), datetime_str.size());
+ column->insert_data(reinterpret_cast<char*>(&v), 0);
} else if (type.is_array()) {
auto v = std::any_cast<Array>(cell);
column->insert(v);
diff --git a/be/test/vec/function/function_test_util.h b/be/test/vec/function/function_test_util.h
index 6ad30a9b6a..f29ab896c3 100644
--- a/be/test/vec/function/function_test_util.h
+++ b/be/test/vec/function/function_test_util.h
@@ -45,6 +45,7 @@ using DataSet = std::vector<Row>;
using InputTypeSet = std::vector<std::any>;
int64_t str_to_date_time(std::string datetime_str, bool data_time = true);
+uint32_t str_to_date_v2(std::string datetime_str, std::string datetime_format);
namespace ut_type {
using TINYINT = int8_t;
@@ -220,6 +221,8 @@ void check_function(const std::string& func_name, const InputTypeSet& input_type
fn_ctx_return.type = doris_udf::FunctionContext::TYPE_DOUBLE;
} else if constexpr (std::is_same_v<ReturnType, DateTime>) {
fn_ctx_return.type = doris_udf::FunctionContext::TYPE_DATETIME;
+ } else if (std::is_same_v<ReturnType, DateV2>) {
+ fn_ctx_return.type = doris_udf::FunctionContext::TYPE_DATEV2;
} else {
fn_ctx_return.type = doris_udf::FunctionContext::INVALID_TYPE;
}
diff --git a/be/test/vec/function/function_time_test.cpp b/be/test/vec/function/function_time_test.cpp
index c046ee825a..c03a572435 100644
--- a/be/test/vec/function/function_time_test.cpp
+++ b/be/test/vec/function/function_time_test.cpp
@@ -455,8 +455,7 @@ TEST(VTimestampFunctionsTest, to_days_test) {
DataSet data_set = {{{std::string("2021-01-01 00:00:00")}, 738156},
{{std::string("")}, Null()},
- {{std::string("2021-01-32 00:00:00")}, Null()},
- {{std::string("0000-01-01 00:00:00")}, 1}};
+ {{std::string("2021-01-32 00:00:00")}, Null()}};
check_function<DataTypeInt32, true>(func_name, input_types, data_set);
}
@@ -469,8 +468,7 @@ TEST(VTimestampFunctionsTest, date_test) {
DataSet data_set = {
{{std::string("2021-01-01 06:00:00")}, str_to_date_time("2021-01-01", false)},
{{std::string("")}, Null()},
- {{Null()}, Null()},
- {{std::string("0000-01-01 00:00:00")}, str_to_date_time("0000-01-01", false)}};
+ {{Null()}, Null()}};
check_function<DataTypeDate, true>(func_name, input_types, data_set);
}
@@ -566,4 +564,550 @@ TEST(VTimestampFunctionsTest, weekday_test) {
check_function<DataTypeInt32, true>(func_name, input_types, data_set);
}
+
+TEST(VTimestampFunctionsTest, day_of_week_v2_test) {
+ std::string func_name = "dayofweek";
+
+ InputTypeSet input_types = {TypeIndex::DateV2};
+
+ DataSet data_set = {{{std::string("2001-02-03")}, 7},
+ {{std::string("2020-00-01")}, Null()},
+ {{std::string("2020-01-00")}, Null()}};
+
+ check_function<DataTypeInt32, true>(func_name, input_types, data_set);
+}
+
+TEST(VTimestampFunctionsTest, day_of_month_v2_test) {
+ std::string func_name = "dayofmonth";
+
+ InputTypeSet input_types = {TypeIndex::DateV2};
+
+ DataSet data_set = {{{std::string("2020-00-01")}, Null()},
+ {{std::string("2020-01-01")}, 1},
+ {{std::string("2020-02-29")}, 29}};
+
+ check_function<DataTypeInt32, true>(func_name, input_types, data_set);
+}
+
+TEST(VTimestampFunctionsTest, day_of_year_v2_test) {
+ std::string func_name = "dayofyear";
+
+ InputTypeSet input_types = {TypeIndex::DateV2};
+
+ DataSet data_set = {{{std::string("2020-00-01")}, Null()},
+ {{std::string("2020-01-00")}, Null()},
+ {{std::string("2020-02-29")}, 60}};
+
+ check_function<DataTypeInt32, true>(func_name, input_types, data_set);
+}
+
+TEST(VTimestampFunctionsTest, week_of_year_v2_test) {
+ std::string func_name = "weekofyear";
+
+ InputTypeSet input_types = {TypeIndex::DateV2};
+
+ DataSet data_set = {{{std::string("2020-00-01")}, Null()},
+ {{std::string("2020-01-00")}, Null()},
+ {{std::string("2020-02-29")}, 9}};
+
+ check_function<DataTypeInt32, true>(func_name, input_types, data_set);
+}
+
+TEST(VTimestampFunctionsTest, year_v2_test) {
+ std::string func_name = "year";
+
+ InputTypeSet input_types = {TypeIndex::DateV2};
+
+ DataSet data_set = {{{std::string("2021-01-01")}, 2021},
+ {{std::string("2021-01-00")}, Null()},
+ {{std::string("2025-05-01")}, 2025}};
+
+ check_function<DataTypeInt32, true>(func_name, input_types, data_set);
+}
+
+TEST(VTimestampFunctionsTest, quarter_v2_test) {
+ std::string func_name = "quarter";
+
+ InputTypeSet input_types = {TypeIndex::DateV2};
+
+ DataSet data_set = {{{std::string("2021-01-01")}, 1},
+ {{std::string("")}, Null()},
+ {{std::string("2021-01-32")}, Null()},
+ {{std::string("2025-10-23")}, 4}};
+
+ check_function<DataTypeInt32, true>(func_name, input_types, data_set);
+}
+
+TEST(VTimestampFunctionsTest, month_v2_test) {
+ std::string func_name = "month";
+
+ InputTypeSet input_types = {TypeIndex::DateV2};
+
+ DataSet data_set = {{{std::string("2021-01-01")}, 1},
+ {{std::string("")}, Null()},
+ {{std::string("2021-01-32")}, Null()},
+ {{std::string("2025-05-23")}, 5}};
+
+ check_function<DataTypeInt32, true>(func_name, input_types, data_set);
+}
+
+TEST(VTimestampFunctionsTest, day_v2_test) {
+ std::string func_name = "day";
+
+ InputTypeSet input_types = {TypeIndex::DateV2};
+
+ DataSet data_set = {{{std::string("2021-01-01")}, 1},
+ {{std::string("")}, Null()},
+ {{std::string("2021-01-32")}, Null()},
+ {{std::string("2025-05-23")}, 23}};
+
+ check_function<DataTypeInt32, true>(func_name, input_types, data_set);
+}
+
+TEST(VTimestampFunctionsTest, hour_v2_test) {
+ std::string func_name = "hour";
+
+ InputTypeSet input_types = {TypeIndex::DateV2};
+
+ DataSet data_set = {{{std::string("2021-01-01")}, 0},
+ {{std::string("2021-01-13")}, 0},
+ {{std::string("")}, Null()},
+ {{std::string("2025-05-23")}, 0}};
+
+ check_function<DataTypeInt32, true>(func_name, input_types, data_set);
+}
+
+TEST(VTimestampFunctionsTest, minute_v2_test) {
+ std::string func_name = "minute";
+
+ InputTypeSet input_types = {TypeIndex::DateV2};
+
+ DataSet data_set = {{{std::string("2021-01-01")}, 0},
+ {{std::string("2021-01-13")}, 0},
+ {{std::string("")}, Null()},
+ {{std::string("2025-05-23")}, 0}};
+
+ check_function<DataTypeInt32, true>(func_name, input_types, data_set);
+}
+
+TEST(VTimestampFunctionsTest, second_v2_test) {
+ std::string func_name = "second";
+
+ InputTypeSet input_types = {TypeIndex::DateV2};
+
+ DataSet data_set = {{{std::string("2021-01-01")}, 0},
+ {{std::string("2021-01-13")}, 0},
+ {{std::string("")}, Null()},
+ {{std::string("2025-05-23")}, 0}};
+
+ check_function<DataTypeInt32, true>(func_name, input_types, data_set);
+}
+
+TEST(VTimestampFunctionsTest, timediff_v2_test) {
+ std::string func_name = "timediff";
+
+ {
+ InputTypeSet input_types = {TypeIndex::DateV2, TypeIndex::DateV2};
+
+ DataSet data_set = {{{std::string("2019-07-18"), std::string("2019-07-18")}, 0.0},
+ {{std::string("2019-07-18"), std::string("2019-07-18")}, -0.0},
+ {{std::string("2019-00-18"), std::string("2019-07-18")}, Null()},
+ {{std::string("2019-07-18"), std::string("2019-07-00")}, Null()}};
+
+ check_function<DataTypeFloat64, true>(func_name, input_types, data_set);
+ }
+
+ {
+ InputTypeSet input_types = {TypeIndex::DateV2, TypeIndex::Date};
+
+ DataSet data_set = {{{std::string("2019-07-18"), std::string("2019-07-18")}, 0.0},
+ {{std::string("2019-07-18"), std::string("2019-07-18")}, -0.0},
+ {{std::string("2019-00-18"), std::string("2019-07-18")}, Null()},
+ {{std::string("2019-07-18"), std::string("2019-07-00")}, Null()}};
+
+ check_function<DataTypeFloat64, true>(func_name, input_types, data_set);
+ }
+
+ {
+ InputTypeSet input_types = {TypeIndex::Date, TypeIndex::DateV2};
+
+ DataSet data_set = {{{std::string("2019-07-18"), std::string("2019-07-18")}, 0.0},
+ {{std::string("2019-07-18"), std::string("2019-07-18")}, -0.0},
+ {{std::string("2019-00-18"), std::string("2019-07-18")}, Null()},
+ {{std::string("2019-07-18"), std::string("2019-07-00")}, Null()}};
+
+ check_function<DataTypeFloat64, true>(func_name, input_types, data_set);
+ }
+
+ {
+ InputTypeSet input_types = {TypeIndex::DateTime, TypeIndex::DateV2};
+
+ DataSet data_set = {
+ {{std::string("2019-07-18 00:00:00"), std::string("2019-07-18")}, 0.0},
+ {{std::string("2019-07-18 00:00:00"), std::string("2019-07-18")}, -0.0},
+ {{std::string("2019-00-18 00:00:00"), std::string("2019-07-18")}, Null()},
+ {{std::string("2019-07-18 00:00:00"), std::string("2019-07-00")}, Null()}};
+
+ check_function<DataTypeFloat64, true>(func_name, input_types, data_set);
+ }
+
+ {
+ InputTypeSet input_types = {TypeIndex::DateV2, TypeIndex::DateTime};
+
+ DataSet data_set = {
+ {{std::string("2019-07-18"), std::string("2019-07-18 00:00:00")}, 0.0},
+ {{std::string("2019-07-18"), std::string("2019-07-18 00:00:00")}, -0.0},
+ {{std::string("2019-00-18"), std::string("2019-07-18 00:00:00")}, Null()},
+ {{std::string("2019-07-18"), std::string("2019-07-00 00:00:00")}, Null()}};
+
+ check_function<DataTypeFloat64, true>(func_name, input_types, data_set);
+ }
+}
+
+TEST(VTimestampFunctionsTest, datediff_v2_test) {
+ std::string func_name = "datediff";
+
+ {
+ InputTypeSet input_types = {TypeIndex::DateV2, TypeIndex::DateV2};
+
+ DataSet data_set = {{{std::string("2019-07-18"), std::string("2019-07-19")}, -1},
+ {{std::string("2019-07-18"), std::string("2019-07-17")}, 1},
+ {{std::string("2019-00-18"), std::string("2019-07-18")}, Null()},
+ {{std::string("2019-07-18"), std::string("2019-07-00")}, Null()}};
+
+ check_function<DataTypeInt32, true>(func_name, input_types, data_set);
+ }
+
+ {
+ InputTypeSet input_types = {TypeIndex::DateV2, TypeIndex::Date};
+
+ DataSet data_set = {{{std::string("2019-07-18"), std::string("2019-07-19")}, -1},
+ {{std::string("2019-07-18"), std::string("2019-07-17")}, 1},
+ {{std::string("2019-00-18"), std::string("2019-07-18")}, Null()},
+ {{std::string("2019-07-18"), std::string("2019-07-00")}, Null()}};
+
+ check_function<DataTypeInt32, true>(func_name, input_types, data_set);
+ }
+
+ {
+ InputTypeSet input_types = {TypeIndex::Date, TypeIndex::DateV2};
+
+ DataSet data_set = {{{std::string("2019-07-18"), std::string("2019-07-19")}, -1},
+ {{std::string("2019-07-18"), std::string("2019-07-17")}, 1},
+ {{std::string("2019-00-18"), std::string("2019-07-18")}, Null()},
+ {{std::string("2019-07-18"), std::string("2019-07-00")}, Null()}};
+
+ check_function<DataTypeInt32, true>(func_name, input_types, data_set);
+ }
+
+ {
+ InputTypeSet input_types = {TypeIndex::DateTime, TypeIndex::DateV2};
+
+ DataSet data_set = {
+ {{std::string("2019-07-18 00:00:00"), std::string("2019-07-19")}, -1},
+ {{std::string("2019-07-18 00:00:00"), std::string("2019-07-17")}, 1},
+ {{std::string("2019-00-18 00:00:00"), std::string("2019-07-18")}, Null()},
+ {{std::string("2019-07-18 00:00:00"), std::string("2019-07-00")}, Null()}};
+
+ check_function<DataTypeInt32, true>(func_name, input_types, data_set);
+ }
+
+ {
+ InputTypeSet input_types = {TypeIndex::DateV2, TypeIndex::DateTime};
+
+ DataSet data_set = {
+ {{std::string("2019-07-18"), std::string("2019-07-19 00:00:00")}, -1},
+ {{std::string("2019-07-18"), std::string("2019-07-17 00:00:00")}, 1},
+ {{std::string("2019-00-18"), std::string("2019-07-18 00:00:00")}, Null()},
+ {{std::string("2019-07-18"), std::string("2019-07-00 00:00:00")}, Null()}};
+
+ check_function<DataTypeInt32, true>(func_name, input_types, data_set);
+ }
+}
+
+TEST(VTimestampFunctionsTest, date_format_v2_test) {
+ std::string func_name = "date_format";
+
+ InputTypeSet input_types = {TypeIndex::DateV2, Consted {TypeIndex::String}};
+ {
+ DataSet data_set = {{{std::string("2009-10-04 22:23:00"), std::string("%W %M %Y")},
+ std::string("Sunday October 2009")}};
+
+ check_function<DataTypeString, true>(func_name, input_types, data_set);
+ }
+ {
+ DataSet data_set = {{{std::string("2007-10-04 22:23:00"), std::string("%H:%i:%s")},
+ std::string("00:00:00")}};
+
+ check_function<DataTypeString, true>(func_name, input_types, data_set);
+ }
+ {
+ DataSet data_set = {
+ {{std::string("1900-10-04 22:23:00"), std::string("%D %y %a %d %m %b %j")},
+ std::string("4th 00 Thu 04 10 Oct 277")}};
+
+ check_function<DataTypeString, true>(func_name, input_types, data_set);
+ }
+ {
+ DataSet data_set = {
+ {{std::string("1997-10-04 22:23:00"), std::string("%H %k %I %r %T %S %w")},
+ std::string("00 0 12 12:00:00 AM 00:00:00 00 6")}};
+
+ check_function<DataTypeString, true>(func_name, input_types, data_set);
+ }
+ {
+ DataSet data_set = {{{std::string("1999-01-01 00:00:00"), std::string("%X %V")},
+ std::string("1998 52")}};
+
+ check_function<DataTypeString, true>(func_name, input_types, data_set);
+ }
+ {
+ DataSet data_set = {
+ {{std::string("2006-06-01 00:00:00"), std::string("%d")}, std::string("01")}};
+
+ check_function<DataTypeString, true>(func_name, input_types, data_set);
+ }
+ {
+ DataSet data_set = {
+ {{std::string("2006-06-01 00:00:00"), std::string("%%%d")}, std::string("%01")}};
+
+ check_function<DataTypeString, true>(func_name, input_types, data_set);
+ }
+}
+
+TEST(VTimestampFunctionsTest, years_add_v2_test) {
+ std::string func_name = "years_add";
+
+ InputTypeSet input_types = {TypeIndex::DateV2, TypeIndex::Int32};
+
+ DataSet data_set = {{{std::string("2020-05-23"), 5}, str_to_date_v2("2025-05-23", "%Y-%m-%d")},
+ {{std::string("2020-05-23"), -5}, str_to_date_v2("2015-05-23", "%Y-%m-%d")},
+ {{std::string(""), 5}, Null()},
+ {{std::string("2020-05-23"), 8000}, Null()},
+ {{Null(), 5}, Null()}};
+
+ check_function<DataTypeDateV2, true>(func_name, input_types, data_set);
+}
+
+TEST(VTimestampFunctionsTest, years_sub_v2_test) {
+ std::string func_name = "years_sub";
+
+ InputTypeSet input_types = {TypeIndex::DateV2, TypeIndex::Int32};
+
+ DataSet data_set = {{{std::string("2020-05-23"), 5}, str_to_date_v2("2015-05-23", "%Y-%m-%d")},
+ {{std::string("2020-05-23"), -5}, str_to_date_v2("2025-05-23", "%Y-%m-%d")},
+ {{std::string(""), 5}, Null()},
+ {{std::string("2020-05-23"), 3000}, Null()},
+ {{Null(), 5}, Null()}};
+
+ check_function<DataTypeDateV2, true>(func_name, input_types, data_set);
+}
+
+TEST(VTimestampFunctionsTest, months_add_v2_test) {
+ std::string func_name = "months_add";
+
+ InputTypeSet input_types = {TypeIndex::DateV2, TypeIndex::Int32};
+
+ DataSet data_set = {{{std::string("2020-10-23"), -4}, str_to_date_v2("2020-06-23", "%Y-%m-%d")},
+ {{std::string("2020-05-23"), 4}, str_to_date_v2("2020-09-23", "%Y-%m-%d")},
+ {{std::string(""), 4}, Null()},
+ {{std::string("2020-05-23"), 10}, str_to_date_v2("2021-03-23", "%Y-%m-%d")},
+ {{Null(), 4}, Null()}};
+
+ check_function<DataTypeDateV2, true>(func_name, input_types, data_set);
+}
+
+TEST(VTimestampFunctionsTest, months_sub_v2_test) {
+ std::string func_name = "months_sub";
+
+ InputTypeSet input_types = {TypeIndex::DateV2, TypeIndex::Int32};
+
+ DataSet data_set = {{{std::string("2020-05-23"), 4}, str_to_date_v2("2020-01-23", "%Y-%m-%d")},
+ {{std::string("2020-05-23"), -4}, str_to_date_v2("2020-09-23", "%Y-%m-%d")},
+ {{std::string(""), 4}, Null()},
+ {{std::string("2020-05-23"), 10}, str_to_date_v2("2019-07-23", "%Y-%m-%d")},
+ {{Null(), 4}, Null()}};
+
+ check_function<DataTypeDateV2, true>(func_name, input_types, data_set);
+}
+
+TEST(VTimestampFunctionsTest, days_add_v2_test) {
+ std::string func_name = "days_add";
+
+ InputTypeSet input_types = {TypeIndex::DateV2, TypeIndex::Int32};
+
+ DataSet data_set = {{{std::string("2020-10-23"), -4}, str_to_date_v2("2020-10-19", "%Y-%m-%d")},
+ {{std::string("2020-05-23"), 4}, str_to_date_v2("2020-05-27", "%Y-%m-%d")},
+ {{std::string(""), 4}, Null()},
+ {{std::string("2020-05-23"), 10}, str_to_date_v2("2020-06-02", "%Y-%m-%d")},
+ {{Null(), 4}, Null()}};
+
+ check_function<DataTypeDateV2, true>(func_name, input_types, data_set);
+}
+
+TEST(VTimestampFunctionsTest, days_sub_v2_test) {
+ std::string func_name = "days_sub";
+
+ InputTypeSet input_types = {TypeIndex::DateV2, TypeIndex::Int32};
+
+ DataSet data_set = {{{std::string("2020-05-23"), 4}, str_to_date_v2("2020-05-19", "%Y-%m-%d")},
+ {{std::string("2020-05-23"), -4}, str_to_date_v2("2020-05-27", "%Y-%m-%d")},
+ {{std::string(""), 4}, Null()},
+ {{std::string("2020-05-23"), 31}, str_to_date_v2("2020-04-22", "%Y-%m-%d")},
+ {{Null(), 4}, Null()}};
+
+ check_function<DataTypeDateV2, true>(func_name, input_types, data_set);
+}
+
+TEST(VTimestampFunctionsTest, weeks_add_v2_test) {
+ std::string func_name = "weeks_add";
+
+ InputTypeSet input_types = {TypeIndex::DateV2, TypeIndex::Int32};
+
+ DataSet data_set = {
+ {{std::string("2020-10-23"), 5}, str_to_date_v2("2020-11-27", "%Y-%m-%d")},
+ {{std::string("2020-05-23"), -5}, str_to_date_v2("2020-04-18", "%Y-%m-%d")},
+ {{std::string(""), 4}, Null()},
+ {{std::string("2020-05-23"), 100}, str_to_date_v2("2022-04-23", "%Y-%m-%d")},
+ {{Null(), 4}, Null()}};
+
+ check_function<DataTypeDateV2, true>(func_name, input_types, data_set);
+}
+
+TEST(VTimestampFunctionsTest, weeks_sub_v2_test) {
+ std::string func_name = "weeks_sub";
+
+ InputTypeSet input_types = {TypeIndex::DateV2, TypeIndex::Int32};
+
+ DataSet data_set = {
+ {{std::string("2020-05-23"), 5}, str_to_date_v2("2020-04-18", "%Y-%m-%d")},
+ {{std::string("2020-05-23"), -5}, str_to_date_v2("2020-06-27", "%Y-%m-%d")},
+ {{std::string(""), 4}, Null()},
+ {{std::string("2020-05-23"), 100}, str_to_date_v2("2018-06-23", "%Y-%m-%d")},
+ {{Null(), 4}, Null()}};
+
+ check_function<DataTypeDateV2, true>(func_name, input_types, data_set);
+}
+
+TEST(VTimestampFunctionsTest, to_days_v2_test) {
+ std::string func_name = "to_days";
+
+ InputTypeSet input_types = {TypeIndex::DateV2};
+
+ DataSet data_set = {{{std::string("2021-01-01")}, 738156},
+ {{std::string("")}, Null()},
+ {{std::string("2021-01-32")}, Null()},
+ {{std::string("0000-01-01")}, Null()}};
+
+ check_function<DataTypeInt32, true>(func_name, input_types, data_set);
+}
+
+TEST(VTimestampFunctionsTest, date_v2_test) {
+ std::string func_name = "date";
+
+ InputTypeSet input_types = {TypeIndex::DateV2};
+
+ DataSet data_set = {{{std::string("2021-01-01")}, str_to_date_v2("2021-01-01", "%Y-%m-%d")},
+ {{std::string("")}, Null()},
+ {{Null()}, Null()},
+ {{std::string("0000-01-01")}, Null()}};
+
+ check_function<DataTypeDateV2, true>(func_name, input_types, data_set);
+}
+
+TEST(VTimestampFunctionsTest, week_v2_test) {
+ std::string func_name = "week";
+
+ InputTypeSet new_input_types = {TypeIndex::DateV2};
+ DataSet new_data_set = {{{std::string("1989-03-21")}, 12},
+ {{std::string("")}, Null()},
+ {{std::string("9999-12-12")}, 50}};
+
+ check_function<DataTypeInt32, true>(func_name, new_input_types, new_data_set);
+}
+
+TEST(VTimestampFunctionsTest, yearweek_v2_test) {
+ std::string func_name = "yearweek";
+
+ InputTypeSet new_input_types = {TypeIndex::DateV2};
+ DataSet new_data_set = {{{std::string("1989-03-21")}, 198912},
+ {{std::string("")}, Null()},
+ {{std::string("9999-12-12")}, 999950}};
+
+ check_function<DataTypeInt32, true>(func_name, new_input_types, new_data_set);
+}
+
+TEST(VTimestampFunctionsTest, str_to_date_test) {
+ std::string func_name = "str_to_date";
+
+ InputTypeSet input_types = {TypeIndex::String, TypeIndex::String};
+
+ {
+ DataSet data_set = {{{std::string("2021-01-01"), std::string("%Y-%m-%d")},
+ str_to_date_time("2021-01-01", false)},
+ {{std::string("2022-01-03"), std::string("%Y-%m-%d")},
+ str_to_date_time("2022-01-03", false)},
+ {{std::string("2021-00-01"), std::string("%Y-%m-%d")}, Null()},
+ {{std::string("2021-01-00"), std::string("%Y-%m-%d")}, Null()}};
+
+ check_function<DataTypeDate, true>(func_name, input_types, data_set);
+ }
+}
+
+TEST(VTimestampFunctionsTest, from_days_test) {
+ std::string func_name = "from_days";
+
+ InputTypeSet input_types = {TypeIndex::Int32};
+
+ {
+ DataSet data_set = {{{730669}, str_to_date_time("2000-07-03", false)}, {{0}, Null()}};
+
+ check_function<DataTypeDate, true>(func_name, input_types, data_set);
+ }
+}
+
+TEST(VTimestampFunctionsTest, weekday_v2_test) {
+ std::string func_name = "weekday";
+
+ InputTypeSet input_types = {TypeIndex::DateV2};
+
+ DataSet data_set = {{{std::string("2001-02-03")}, 5},
+ {{std::string("2019-06-25")}, 1},
+ {{std::string("2020-00-01")}, Null()},
+ {{std::string("2020-01-00")}, Null()}};
+
+ check_function<DataTypeInt32, true>(func_name, input_types, data_set);
+}
+
+TEST(VTimestampFunctionsTest, dayname_test) {
+ std::string func_name = "dayname";
+
+ {
+ InputTypeSet input_types = {TypeIndex::DateV2};
+
+ DataSet data_set = {{{std::string("2007-02-03")}, std::string("Saturday")},
+ {{std::string("2020-01-00")}, Null()}};
+
+ check_function<DataTypeString, true>(func_name, input_types, data_set);
+ }
+
+ {
+ InputTypeSet input_types = {TypeIndex::DateTime};
+
+ DataSet data_set = {{{std::string("2007-02-03 00:00:00")}, std::string("Saturday")},
+ {{std::string("2020-01-00 00:00:00")}, Null()}};
+
+ check_function<DataTypeString, true>(func_name, input_types, data_set);
+ }
+
+ {
+ InputTypeSet input_types = {TypeIndex::Date};
+
+ DataSet data_set = {{{std::string("2007-02-03")}, std::string("Saturday")},
+ {{std::string("2020-01-00")}, Null()}};
+
+ check_function<DataTypeString, true>(func_name, input_types, data_set);
+ }
+}
+
} // namespace doris::vectorized
diff --git a/be/test/vec/runtime/vdatetime_value_test.cpp b/be/test/vec/runtime/vdatetime_value_test.cpp
new file mode 100644
index 0000000000..727fdc2737
--- /dev/null
+++ b/be/test/vec/runtime/vdatetime_value_test.cpp
@@ -0,0 +1,247 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "vec/runtime/vdatetime_value.h"
+
+#include <gtest/gtest.h>
+
+#include <any>
+#include <string>
+
+namespace doris::vectorized {
+
+TEST(VDateTimeValueTest, date_v2_to_uint32_test) {
+ uint16_t year = 2022;
+ uint8_t month = 5;
+ uint8_t day = 24;
+
+ DateV2Value date_v2;
+ date_v2.set_time(year, month, day);
+
+ EXPECT_TRUE(date_v2.year() == year);
+ EXPECT_TRUE(date_v2.month() == month);
+ EXPECT_TRUE(date_v2.day() == day);
+ EXPECT_TRUE(date_v2.to_date_uint32() == ((year << 16) | (month << 8) | day));
+ EXPECT_TRUE(date_v2.hour() == 0);
+ EXPECT_TRUE(date_v2.minute() == 0);
+ EXPECT_TRUE(date_v2.second() == 0);
+}
+
+TEST(VDateTimeValueTest, date_v2_from_uint32_test) {
+ {
+ uint16_t year = 2022;
+ uint8_t month = 5;
+ uint8_t day = 24;
+
+ DateV2Value date_v2;
+ date_v2.from_date((uint32_t)((year << 16) | (month << 8) | day));
+
+ EXPECT_TRUE(date_v2.year() == year);
+ EXPECT_TRUE(date_v2.month() == month);
+ EXPECT_TRUE(date_v2.day() == day);
+ EXPECT_TRUE(date_v2.to_date_uint32() == ((year << 16) | (month << 8) | day));
+ EXPECT_TRUE(date_v2.hour() == 0);
+ EXPECT_TRUE(date_v2.minute() == 0);
+ EXPECT_TRUE(date_v2.second() == 0);
+ }
+ {
+ uint16_t year = 2022;
+ uint8_t month = 5;
+ uint8_t day = 24;
+
+ uint32_t ui32 = (uint32_t)((year << 16) | (month << 8) | day);
+ auto date_v2 = (DateV2Value&)ui32;
+
+ EXPECT_TRUE(date_v2.year() == year);
+ EXPECT_TRUE(date_v2.month() == month);
+ EXPECT_TRUE(date_v2.day() == day);
+ EXPECT_TRUE(date_v2.to_date_uint32() == ((year << 16) | (month << 8) | day));
+ EXPECT_TRUE(date_v2.hour() == 0);
+ EXPECT_TRUE(date_v2.minute() == 0);
+ EXPECT_TRUE(date_v2.second() == 0);
+ }
+}
+
+TEST(VDateTimeValueTest, date_v2_from_date_format_str_test) {
+ uint16_t year = 2022;
+ uint8_t month = 5;
+ uint8_t day = 24;
+
+ {
+ DateV2Value date_v2;
+ std::string origin_date = "2022-05-24";
+ std::string date_format = "%Y-%m-%d";
+ EXPECT_TRUE(date_v2.from_date_format_str(date_format.data(), date_format.size(),
+ origin_date.data(), origin_date.size()));
+
+ EXPECT_TRUE(date_v2.year() == year);
+ EXPECT_TRUE(date_v2.month() == month);
+ EXPECT_TRUE(date_v2.day() == day);
+ EXPECT_TRUE(date_v2.to_date_uint32() == ((year << 16) | (month << 8) | day));
+ EXPECT_TRUE(date_v2.hour() == 0);
+ EXPECT_TRUE(date_v2.minute() == 0);
+ EXPECT_TRUE(date_v2.second() == 0);
+ }
+
+ {
+ DateV2Value date_v2;
+ std::string origin_date = "2022-05-24 10:10:00";
+ std::string date_format = "%Y-%m-%d";
+ EXPECT_TRUE(date_v2.from_date_format_str(date_format.data(), date_format.size(),
+ origin_date.data(), origin_date.size()));
+
+ EXPECT_TRUE(date_v2.year() == year);
+ EXPECT_TRUE(date_v2.month() == month);
+ EXPECT_TRUE(date_v2.day() == day);
+ EXPECT_TRUE(date_v2.to_date_uint32() == ((year << 16) | (month << 8) | day));
+ EXPECT_TRUE(date_v2.hour() == 0);
+ EXPECT_TRUE(date_v2.minute() == 0);
+ EXPECT_TRUE(date_v2.second() == 0);
+ }
+}
+
+TEST(VDateTimeValueTest, date_diff_test) {
+ {
+ DateV2Value date_v2_1;
+ std::string origin_date1 = "2022-05-24";
+ std::string date_format1 = "%Y-%m-%d";
+ EXPECT_TRUE(date_v2_1.from_date_format_str(date_format1.data(), date_format1.size(),
+ origin_date1.data(), origin_date1.size()));
+
+ DateV2Value date_v2_2;
+ std::string origin_date2 = "2022-06-24";
+ std::string date_format2 = "%Y-%m-%d";
+ EXPECT_TRUE(date_v2_2.from_date_format_str(date_format2.data(), date_format2.size(),
+ origin_date2.data(), origin_date2.size()));
+
+ EXPECT_TRUE(datetime_diff<TimeUnit::DAY>(date_v2_1, date_v2_2) == 31);
+ EXPECT_TRUE(datetime_diff<TimeUnit::YEAR>(date_v2_1, date_v2_2) == 0);
+ EXPECT_TRUE(datetime_diff<TimeUnit::MONTH>(date_v2_1, date_v2_2) == 1);
+ EXPECT_TRUE(datetime_diff<TimeUnit::HOUR>(date_v2_1, date_v2_2) == 31 * 24);
+ EXPECT_TRUE(datetime_diff<TimeUnit::MINUTE>(date_v2_1, date_v2_2) == 31 * 24 * 60);
+ EXPECT_TRUE(datetime_diff<TimeUnit::SECOND>(date_v2_1, date_v2_2) == 31 * 24 * 60 * 60);
+ }
+
+ {
+ DateV2Value date_v2_1;
+ std::string origin_date1 = "2022-05-24";
+ std::string date_format1 = "%Y-%m-%d";
+ EXPECT_TRUE(date_v2_1.from_date_format_str(date_format1.data(), date_format1.size(),
+ origin_date1.data(), origin_date1.size()));
+
+ VecDateTimeValue date_v2_2;
+ std::string origin_date2 = "2022-06-24 00:00:00";
+ std::string date_format2 = "%Y-%m-%d %H:%i:%s";
+ EXPECT_TRUE(date_v2_2.from_date_format_str(date_format2.data(), date_format2.size(),
+ origin_date2.data(), origin_date2.size()));
+
+ EXPECT_TRUE(datetime_diff<TimeUnit::DAY>(date_v2_1, date_v2_2) == 31);
+ EXPECT_TRUE(datetime_diff<TimeUnit::YEAR>(date_v2_1, date_v2_2) == 0);
+ EXPECT_TRUE(datetime_diff<TimeUnit::MONTH>(date_v2_1, date_v2_2) == 1);
+ EXPECT_TRUE(datetime_diff<TimeUnit::HOUR>(date_v2_1, date_v2_2) == 31 * 24);
+ EXPECT_TRUE(datetime_diff<TimeUnit::MINUTE>(date_v2_1, date_v2_2) == 31 * 24 * 60);
+ EXPECT_TRUE(datetime_diff<TimeUnit::SECOND>(date_v2_1, date_v2_2) == 31 * 24 * 60 * 60);
+ }
+
+ {
+ VecDateTimeValue date_v2_1;
+ std::string origin_date1 = "2022-05-24 00:00:00";
+ std::string date_format1 = "%Y-%m-%d %H:%i:%s";
+ EXPECT_TRUE(date_v2_1.from_date_format_str(date_format1.data(), date_format1.size(),
+ origin_date1.data(), origin_date1.size()));
+
+ DateV2Value date_v2_2;
+ std::string origin_date2 = "2022-06-24";
+ std::string date_format2 = "%Y-%m-%d";
+ EXPECT_TRUE(date_v2_2.from_date_format_str(date_format2.data(), date_format2.size(),
+ origin_date2.data(), origin_date2.size()));
+
+ EXPECT_TRUE(datetime_diff<TimeUnit::DAY>(date_v2_1, date_v2_2) == 31);
+ EXPECT_TRUE(datetime_diff<TimeUnit::YEAR>(date_v2_1, date_v2_2) == 0);
+ EXPECT_TRUE(datetime_diff<TimeUnit::MONTH>(date_v2_1, date_v2_2) == 1);
+ EXPECT_TRUE(datetime_diff<TimeUnit::HOUR>(date_v2_1, date_v2_2) == 31 * 24);
+ EXPECT_TRUE(datetime_diff<TimeUnit::MINUTE>(date_v2_1, date_v2_2) == 31 * 24 * 60);
+ EXPECT_TRUE(datetime_diff<TimeUnit::SECOND>(date_v2_1, date_v2_2) == 31 * 24 * 60 * 60);
+ }
+
+ {
+ DateV2Value date_v2_1;
+ std::string origin_date1 = "2022-05-24";
+ std::string date_format1 = "%Y-%m-%d";
+ EXPECT_TRUE(date_v2_1.from_date_format_str(date_format1.data(), date_format1.size(),
+ origin_date1.data(), origin_date1.size()));
+
+ VecDateTimeValue date_v2_2;
+ std::string origin_date2 = "2022-06-24 06:00:00";
+ std::string date_format2 = "%Y-%m-%d %H:%i:%s";
+ EXPECT_TRUE(date_v2_2.from_date_format_str(date_format2.data(), date_format2.size(),
+ origin_date2.data(), origin_date2.size()));
+
+ EXPECT_TRUE(datetime_diff<TimeUnit::DAY>(date_v2_1, date_v2_2) == 31);
+ EXPECT_TRUE(datetime_diff<TimeUnit::YEAR>(date_v2_1, date_v2_2) == 0);
+ EXPECT_TRUE(datetime_diff<TimeUnit::MONTH>(date_v2_1, date_v2_2) == 1);
+ EXPECT_TRUE(datetime_diff<TimeUnit::HOUR>(date_v2_1, date_v2_2) == 31 * 24 + 6);
+ EXPECT_TRUE(datetime_diff<TimeUnit::MINUTE>(date_v2_1, date_v2_2) == (31 * 24 + 6) * 60);
+ EXPECT_TRUE(datetime_diff<TimeUnit::SECOND>(date_v2_1, date_v2_2) ==
+ (31 * 24 + 6) * 60 * 60);
+ }
+
+ {
+ VecDateTimeValue date_v2_1;
+ std::string origin_date1 = "2022-05-24";
+ std::string date_format1 = "%Y-%m-%d";
+ EXPECT_TRUE(date_v2_1.from_date_format_str(date_format1.data(), date_format1.size(),
+ origin_date1.data(), origin_date1.size()));
+
+ VecDateTimeValue date_v2_2;
+ std::string origin_date2 = "2022-06-24 06:00:00";
+ std::string date_format2 = "%Y-%m-%d %H:%i:%s";
+ EXPECT_TRUE(date_v2_2.from_date_format_str(date_format2.data(), date_format2.size(),
+ origin_date2.data(), origin_date2.size()));
+
+ EXPECT_TRUE(datetime_diff<TimeUnit::DAY>(date_v2_1, date_v2_2) == 31);
+ EXPECT_TRUE(datetime_diff<TimeUnit::YEAR>(date_v2_1, date_v2_2) == 0);
+ EXPECT_TRUE(datetime_diff<TimeUnit::MONTH>(date_v2_1, date_v2_2) == 1);
+ EXPECT_TRUE(datetime_diff<TimeUnit::HOUR>(date_v2_1, date_v2_2) == 31 * 24 + 6);
+ EXPECT_TRUE(datetime_diff<TimeUnit::MINUTE>(date_v2_1, date_v2_2) == (31 * 24 + 6) * 60);
+ EXPECT_TRUE(datetime_diff<TimeUnit::SECOND>(date_v2_1, date_v2_2) ==
+ (31 * 24 + 6) * 60 * 60);
+ }
+
+ {
+ VecDateTimeValue date_v2_1;
+ std::string origin_date1 = "2022-05-24 06:00:00";
+ std::string date_format1 = "%Y-%m-%d %H:%i:%s";
+ EXPECT_TRUE(date_v2_1.from_date_format_str(date_format1.data(), date_format1.size(),
+ origin_date1.data(), origin_date1.size()));
+
+ VecDateTimeValue date_v2_2;
+ std::string origin_date2 = "2022-06-24 06:00:00";
+ std::string date_format2 = "%Y-%m-%d %H:%i:%s";
+ EXPECT_TRUE(date_v2_2.from_date_format_str(date_format2.data(), date_format2.size(),
+ origin_date2.data(), origin_date2.size()));
+
+ EXPECT_TRUE(datetime_diff<TimeUnit::DAY>(date_v2_1, date_v2_2) == 31);
+ EXPECT_TRUE(datetime_diff<TimeUnit::YEAR>(date_v2_1, date_v2_2) == 0);
+ EXPECT_TRUE(datetime_diff<TimeUnit::MONTH>(date_v2_1, date_v2_2) == 1);
+ EXPECT_TRUE(datetime_diff<TimeUnit::HOUR>(date_v2_1, date_v2_2) == 31 * 24);
+ EXPECT_TRUE(datetime_diff<TimeUnit::MINUTE>(date_v2_1, date_v2_2) == 31 * 24 * 60);
+ EXPECT_TRUE(datetime_diff<TimeUnit::SECOND>(date_v2_1, date_v2_2) == 31 * 24 * 60 * 60);
+ }
+}
+
+} // namespace doris::vectorized
diff --git a/be/test/vec/utils/arrow_column_to_doris_column_test.cpp b/be/test/vec/utils/arrow_column_to_doris_column_test.cpp
index 73181a57de..606132ca9d 100644
--- a/be/test/vec/utils/arrow_column_to_doris_column_test.cpp
+++ b/be/test/vec/utils/arrow_column_to_doris_column_test.cpp
@@ -123,7 +123,8 @@ void test_arrow_to_datetime_column(std::shared_ptr<ArrowType> type, ColumnWithTy
if constexpr (std::is_same_v<ArrowType, arrow::TimestampType>) {
time_zone = type->timezone();
}
- auto ret = arrow_column_to_doris_column(array.get(), 0, column.column, num_elements, time_zone);
+ auto ret = arrow_column_to_doris_column(array.get(), 0, column.column, column.type,
+ num_elements, time_zone);
ASSERT_EQ(ret.ok(), true);
ASSERT_EQ(column.column->size(), counter);
MutableColumnPtr data_column = nullptr;
@@ -216,7 +217,8 @@ void test_arrow_to_numeric_column(std::shared_ptr<ArrowType> type, ColumnWithTyp
ASSERT_EQ(column.column->size(), counter);
auto array = create_constant_numeric_array<ArrowType, is_nullable>(num_elements, arrow_numeric,
type, counter);
- auto ret = arrow_column_to_doris_column(array.get(), 0, column.column, num_elements, "UTC");
+ auto ret = arrow_column_to_doris_column(array.get(), 0, column.column, column.type,
+ num_elements, "UTC");
ASSERT_EQ(ret.ok(), true);
ASSERT_EQ(column.column->size(), counter);
MutableColumnPtr data_column = nullptr;
@@ -351,7 +353,8 @@ void test_arrow_to_decimal_column(std::shared_ptr<arrow::Decimal128Type> type,
int128_t arrow_value, int128_t expect_value, size_t& counter) {
ASSERT_EQ(column.column->size(), counter);
auto array = create_decimal_array<is_nullable>(num_elements, arrow_value, type, counter);
- auto ret = arrow_column_to_doris_column(array.get(), 0, column.column, num_elements, "UTC");
+ auto ret = arrow_column_to_doris_column(array.get(), 0, column.column, column.type,
+ num_elements, "UTC");
ASSERT_EQ(ret.ok(), true);
ASSERT_EQ(column.column->size(), counter);
MutableColumnPtr data_column = nullptr;
@@ -452,7 +455,8 @@ void test_arrow_to_fixed_binary_column(ColumnWithTypeAndName& column, size_t num
ASSERT_EQ(column.column->size(), counter);
auto array =
create_fixed_size_binary_array<bytes_width, is_nullable>(num_elements, value, counter);
- auto ret = arrow_column_to_doris_column(array.get(), 0, column.column, num_elements, "UTC");
+ auto ret = arrow_column_to_doris_column(array.get(), 0, column.column, column.type,
+ num_elements, "UTC");
ASSERT_EQ(ret.ok(), true);
ASSERT_EQ(column.column->size(), counter);
MutableColumnPtr data_column = nullptr;
@@ -554,7 +558,8 @@ void test_arrow_to_binary_column(ColumnWithTypeAndName& column, size_t num_eleme
ArrowCppType value, size_t& counter) {
ASSERT_EQ(column.column->size(), counter);
auto array = create_binary_array<ArrowType, is_nullable>(num_elements, value, counter);
- auto ret = arrow_column_to_doris_column(array.get(), 0, column.column, num_elements, "UTC");
+ auto ret = arrow_column_to_doris_column(array.get(), 0, column.column, column.type,
+ num_elements, "UTC");
ASSERT_EQ(ret.ok(), true);
ASSERT_EQ(column.column->size(), counter);
MutableColumnPtr data_column = nullptr;
@@ -650,8 +655,8 @@ void test_arrow_to_array_column(ColumnWithTypeAndName& column,
ASSERT_EQ(column.column->size(), counter);
auto array = create_array_array<ArrowType, is_nullable>(vec_offsets, null_map, value_type,
values, counter);
- auto ret = arrow_column_to_doris_column(array.get(), 0, column.column, vec_offsets.size() - 1,
- "UTC");
+ auto ret = arrow_column_to_doris_column(array.get(), 0, column.column, column.type,
+ vec_offsets.size() - 1, "UTC");
ASSERT_EQ(ret.ok(), true);
ASSERT_EQ(column.column->size(), counter);
MutableColumnPtr data_column = nullptr;
diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup
index 3b1404fc2e..0d562e1620 100644
--- a/fe/fe-core/src/main/cup/sql_parser.cup
+++ b/fe/fe-core/src/main/cup/sql_parser.cup
@@ -4652,11 +4652,15 @@ type ::=
| KW_DOUBLE
{: RESULT = Type.DOUBLE; :}
| KW_DATE
- {: RESULT = Type.DATE; :}
+ {: RESULT = ScalarType.createDateType(); :}
+ | KW_DATETIME LPAREN INTEGER_LITERAL:precision RPAREN
+ {: RESULT = ScalarType.createDatetimeV2Type(precision.intValue()); :}
| KW_DATETIME
- {: RESULT = Type.DATETIME; :}
+ {: RESULT = ScalarType.createDatetimeType(); :}
+ | KW_TIME LPAREN INTEGER_LITERAL:precision RPAREN
+ {: RESULT = ScalarType.createTimeV2Type(precision.intValue()); :}
| KW_TIME
- {: RESULT = Type.TIME; :}
+ {: RESULT = ScalarType.createTimeType(); :}
| KW_BITMAP
{: RESULT = Type.BITMAP; :}
| KW_QUANTILE_STATE
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java
index 5b4ed33e1c..e4f8bc1ea0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java
@@ -138,8 +138,7 @@ public class CastExpr extends Expr {
private static boolean disableRegisterCastingFunction(Type fromType, Type toType) {
// Disable casting from boolean to decimal or datetime or date
- if (fromType.isBoolean()
- && (toType.equals(Type.DECIMALV2) || toType.equals(Type.DATETIME) || toType.equals(Type.DATE))) {
+ if (fromType.isBoolean() && (toType.equals(Type.DECIMALV2) || toType.isDateType())) {
return true;
}
@@ -166,7 +165,9 @@ public class CastExpr extends Expr {
beClass = "TimeOperators";
}
String typeName = Function.getUdfTypeName(toType.getPrimitiveType());
- if (toType.getPrimitiveType() == PrimitiveType.DATE) {
+ // only refactor date/datetime for vectorized engine.
+ if (toType.getPrimitiveType() == PrimitiveType.DATE
+ || toType.getPrimitiveType() == PrimitiveType.DATEV2) {
typeName = "date_val";
}
String beSymbol = "doris::" + beClass + "::cast_to_"
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java
index e98f4c2dfc..b1b471c603 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java
@@ -338,11 +338,13 @@ public class ColumnDef {
decimalLiteral.checkPrecisionAndScale(scalarType.getScalarPrecision(), scalarType.getScalarScale());
break;
case DATE:
- new DateLiteral(defaultValue, type);
+ case DATEV2:
+ new DateLiteral(defaultValue, DateLiteral.getDefaultDateType(type));
break;
case DATETIME:
+ case DATETIMEV2:
if (defaultValueExprDef == null) {
- new DateLiteral(defaultValue, type);
+ new DateLiteral(defaultValue, DateLiteral.getDefaultDateType(type));
} else {
if (defaultValueExprDef.getExprName().equals(DefaultValue.NOW)) {
break;
@@ -360,11 +362,8 @@ public class ColumnDef {
}
break;
case BITMAP:
- break;
case ARRAY:
- break;
case MAP:
- break;
case STRUCT:
break;
case BOOLEAN:
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateFunctionStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateFunctionStmt.java
index 9115a25391..74f9c29ad7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateFunctionStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateFunctionStmt.java
@@ -617,10 +617,13 @@ public class CreateFunctionStmt extends DdlStmt {
typeBuilder.setId(Types.PGenericType.TypeId.BITMAP);
break;
case DATE:
+ case DATEV2:
typeBuilder.setId(Types.PGenericType.TypeId.DATE);
break;
case DATETIME:
+ case DATETIMEV2:
case TIME:
+ case TIMEV2:
typeBuilder.setId(Types.PGenericType.TypeId.DATETIME);
break;
case DECIMALV2:
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DataDescription.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DataDescription.java
index 01cf141c71..bb4eed5615 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DataDescription.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DataDescription.java
@@ -23,7 +23,6 @@ import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.FunctionSet;
import org.apache.doris.catalog.OlapTable;
-import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
@@ -379,7 +378,7 @@ public class DataDescription {
}
private static void validateNowFunction(Column mappingColumn) throws AnalysisException {
- if (!mappingColumn.getOriginType().equals(Type.DATE) && !mappingColumn.getOriginType().equals(Type.DATETIME)) {
+ if (!mappingColumn.getOriginType().isDateType()) {
throw new AnalysisException("Now() function is only support for DATE/DATETIME column");
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java
index 24f552e100..e5953ea44d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java
@@ -21,8 +21,10 @@
package org.apache.doris.analysis;
import org.apache.doris.catalog.PrimitiveType;
+import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.Config;
import org.apache.doris.common.InvalidFormatException;
import org.apache.doris.thrift.TDateLiteral;
import org.apache.doris.thrift.TExprNode;
@@ -55,8 +57,6 @@ public class DateLiteral extends LiteralExpr {
private static final DateLiteral MAX_DATE = new DateLiteral(9999, 12, 31);
private static final DateLiteral MIN_DATETIME = new DateLiteral(0000, 1, 1, 0, 0, 0);
private static final DateLiteral MAX_DATETIME = new DateLiteral(9999, 12, 31, 23, 59, 59);
- public static final DateLiteral UNIX_EPOCH_TIME = new DateLiteral(1970, 01, 01, 00, 00, 00);
-
private static final int DATEKEY_LENGTH = 8;
private static final int MAX_MICROSECOND = 999999;
private static final int DATETIME_TO_MINUTE_STRING_LENGTH = 16;
@@ -152,7 +152,10 @@ public class DateLiteral extends LiteralExpr {
//Date Literal persist type in meta
private enum DateLiteralType {
DATETIME(0),
- DATE(1);
+ DATE(1),
+
+ DATETIMEV2(2),
+ DATEV2(3);
private final int value;
private DateLiteralType(int value) {
@@ -193,7 +196,7 @@ public class DateLiteral extends LiteralExpr {
analysisDone();
}
- public DateLiteral(long unixTimestamp, TimeZone timeZone, Type type) {
+ public DateLiteral(long unixTimestamp, TimeZone timeZone, Type type) throws AnalysisException {
DateTime dt = new DateTime(unixTimestamp, DateTimeZone.forTimeZone(timeZone));
year = dt.getYear();
month = dt.getMonthOfYear();
@@ -206,8 +209,17 @@ public class DateLiteral extends LiteralExpr {
minute = 0;
second = 0;
this.type = Type.DATE;
- } else {
+ } else if (type.equals(Type.DATETIME)) {
this.type = Type.DATETIME;
+ } else if (type.equals(Type.DATEV2)) {
+ hour = 0;
+ minute = 0;
+ second = 0;
+ this.type = Type.DATEV2;
+ } else if (type.equals(Type.DATETIMEV2)) {
+ this.type = Type.DATETIMEV2;
+ } else {
+ throw new AnalysisException("Error date literal type : " + type);
}
}
@@ -218,7 +230,23 @@ public class DateLiteral extends LiteralExpr {
this.year = year;
this.month = month;
this.day = day;
- this.type = Type.DATE;
+ try {
+ this.type = DateLiteral.getDefaultDateType(Type.DATE);
+ } catch (AnalysisException e) {
+ this.type = Type.DATE;
+ }
+ }
+
+ public DateLiteral(long year, long month, long day, Type type) {
+ this.hour = hour;
+ this.minute = minute;
+ this.second = second;
+ this.year = year;
+ this.month = month;
+ this.day = day;
+ Preconditions.checkArgument(type.getPrimitiveType().equals(Type.DATE.getPrimitiveType())
+ || type.getPrimitiveType().equals(Type.DATEV2.getPrimitiveType()));
+ this.type = type;
}
public DateLiteral(long year, long month, long day, long hour, long minute, long second) {
@@ -228,7 +256,34 @@ public class DateLiteral extends LiteralExpr {
this.year = year;
this.month = month;
this.day = day;
- this.type = Type.DATETIME;
+ try {
+ this.type = DateLiteral.getDefaultDateType(Type.DATETIME);
+ } catch (AnalysisException e) {
+ this.type = Type.DATETIME;
+ }
+ }
+
+ public DateLiteral(long year, long month, long day, long hour, long minute, long second, long microsecond) {
+ this.hour = hour;
+ this.minute = minute;
+ this.second = second;
+ this.year = year;
+ this.month = month;
+ this.day = day;
+ this.microsecond = microsecond;
+ this.type = Type.DATETIMEV2;
+ }
+
+ public DateLiteral(long year, long month, long day, long hour, long minute, long second, Type type) {
+ this.hour = hour;
+ this.minute = minute;
+ this.second = second;
+ this.year = year;
+ this.month = month;
+ this.day = day;
+ Preconditions.checkArgument(type.getPrimitiveType().equals(Type.DATETIME.getPrimitiveType())
+ || type.getPrimitiveType().equals(Type.DATETIMEV2.getPrimitiveType()));
+ this.type = type;
}
public DateLiteral(LocalDateTime dateTime, Type type) {
@@ -261,7 +316,7 @@ public class DateLiteral extends LiteralExpr {
try {
Preconditions.checkArgument(type.isDateType());
LocalDateTime dateTime;
- if (type.equals(Type.DATE)) {
+ if (type.equals(Type.DATE) || type.equals(Type.DATEV2)) {
if (s.split("-")[0].length() == 2) {
dateTime = DATE_FORMATTER_TWO_DIGIT.parseLocalDateTime(s);
} else if (s.length() == DATEKEY_LENGTH && !s.contains("-")) {
@@ -317,8 +372,10 @@ public class DateLiteral extends LiteralExpr {
public boolean isMinValue() {
switch (type.getPrimitiveType()) {
case DATE:
+ case DATEV2:
return this.getStringValue().compareTo(MIN_DATE.getStringValue()) == 0;
case DATETIME:
+ case DATETIMEV2:
return this.getStringValue().compareTo(MIN_DATETIME.getStringValue()) == 0;
default:
return false;
@@ -331,6 +388,11 @@ public class DateLiteral extends LiteralExpr {
return year * 16 * 32L + month * 32 + day;
} else if (type.equals(Type.DATETIME)) {
return (year * 10000 + month * 100 + day) * 1000000L + hour * 10000 + minute * 100 + second;
+ } else if (type.equals(Type.DATEV2)) {
+ return (year << 16) | (month << 8) | day;
+ } else if (type.equals(Type.DATETIMEV2)) {
+ return (year << 50) | (month << 46) | (day << 41) | (hour << 36)
+ | (minute << 30) | (second << 24) | microsecond;
} else {
Preconditions.checkState(false, "invalid date type: " + type);
return -1L;
@@ -340,6 +402,8 @@ public class DateLiteral extends LiteralExpr {
// Date column and Datetime column's hash value is not same.
@Override
public ByteBuffer getHashValue(PrimitiveType type) {
+ // This hash value should be computed using new String since precision is introduced to datetime.
+ // But it is hard to keep compatibility. So I don't change this function here.
String value = convertToString(type);
ByteBuffer buffer;
try {
@@ -370,12 +434,25 @@ public class DateLiteral extends LiteralExpr {
@Override
public String getStringValue() {
- return convertToString(type.getPrimitiveType());
+ if (type.isDate() || type.isDateV2()) {
+ return String.format("%04d-%02d-%02d", year, month, day);
+ } else if (type.isDatetimeV2()) {
+ String s = String.format("%04d-%02d-%02d %02d:%02d:%02d", year, month, day, hour, minute, second);
+ if (((ScalarType) type).decimalScale() == 0) {
+ return s;
+ }
+ return s + "." + microsecond / (10L * (6 - ((ScalarType) type).decimalScale()));
+ } else {
+ return String.format("%04d-%02d-%02d %02d:%02d:%02d", year, month, day, hour, minute, second);
+ }
}
private String convertToString(PrimitiveType type) {
- if (type == PrimitiveType.DATE) {
+ if (type == PrimitiveType.DATE || type == PrimitiveType.DATEV2) {
return String.format("%04d-%02d-%02d", year, month, day);
+ } else if (type == PrimitiveType.DATETIMEV2) {
+ return String.format("%04d-%02d-%02d %02d:%02d:%02d.%06d",
+ year, month, day, hour, minute, second, microsecond);
} else {
return String.format("%04d-%02d-%02d %02d:%02d:%02d", year, month, day, hour, minute, second);
}
@@ -403,10 +480,13 @@ public class DateLiteral extends LiteralExpr {
if (type.equals(targetType)) {
return this;
}
- if (targetType.equals(Type.DATE)) {
+ if (targetType.equals(Type.DATE) || targetType.equals(Type.DATEV2)) {
return new DateLiteral(this.year, this.month, this.day);
} else if (targetType.equals(Type.DATETIME)) {
return new DateLiteral(this.year, this.month, this.day, this.hour, this.minute, this.second);
+ } else if (targetType.isDatetimeV2()) {
+ return new DateLiteral(this.year, this.month, this.day, this.hour, this.minute, this.microsecond,
+ targetType);
} else {
throw new AnalysisException("Error date literal type : " + type);
}
@@ -420,7 +500,11 @@ public class DateLiteral extends LiteralExpr {
}
public void castToDate() {
- this.type = Type.DATE;
+ if (Config.use_date_v2_by_default) {
+ this.type = Type.DATEV2;
+ } else {
+ this.type = Type.DATE;
+ }
hour = 0;
minute = 0;
second = 0;
@@ -433,31 +517,43 @@ public class DateLiteral extends LiteralExpr {
return packedDatetime;
}
+ private long makePackedDatetimeV2() {
+ return (year << 50) | (month << 46) | (day << 41) | (hour << 36)
+ | (minute << 30) | (second << 24) | microsecond;
+ }
+
@Override
public void write(DataOutput out) throws IOException {
super.write(out);
//set flag bit in meta, 0 is DATETIME and 1 is DATE
if (this.type.equals(Type.DATETIME)) {
out.writeShort(DateLiteralType.DATETIME.value());
+ out.writeLong(makePackedDatetime());
} else if (this.type.equals(Type.DATE)) {
out.writeShort(DateLiteralType.DATE.value());
+ out.writeLong(makePackedDatetime());
+ } else if (this.type.equals(Type.DATETIMEV2)) {
+ out.writeShort(DateLiteralType.DATETIMEV2.value());
+ out.writeLong(makePackedDatetimeV2());
+ } else if (this.type.equals(Type.DATEV2)) {
+ out.writeShort(DateLiteralType.DATEV2.value());
+ out.writeLong(makePackedDatetimeV2());
} else {
throw new IOException("Error date literal type : " + type);
}
- out.writeLong(makePackedDatetime());
}
private void fromPackedDatetime(long packedTime) {
microsecond = (packedTime % (1L << 24));
long ymdhms = (packedTime >> 24);
long ymd = ymdhms >> 17;
- long hms = ymdhms % (1 << 17);
-
day = ymd % (1 << 5);
long ym = ymd >> 5;
month = ym % 13;
year = ym / 13;
year %= 10000;
+
+ long hms = ymdhms % (1 << 17);
second = hms % (1 << 6);
minute = (hms >> 6) % (1 << 6);
hour = (hms >> 12);
@@ -466,6 +562,24 @@ public class DateLiteral extends LiteralExpr {
this.type = Type.DATETIME;
}
+ private void fromPackedDatetimeV2(long packedTime) {
+ microsecond = (packedTime % (1L << 24));
+ long ymdhms = (packedTime >> 24);
+ long ymd = ymdhms >> 17;
+ day = ymd % (1 << 5);
+ long ym = ymd >> 5;
+ month = ym % (1 << 4);
+ year = ym >> 4;
+
+ long hms = ymdhms % (1 << 17);
+ second = hms % (1 << 6);
+ minute = (hms >> 6) % (1 << 6);
+ hour = (hms >> 12);
+ // set default date literal type to DATETIME
+ // date literal read from meta will set type by flag bit;
+ this.type = Type.DATETIMEV2;
+ }
+
public void readFields(DataInput in) throws IOException {
super.readFields(in);
short dateLiteralType = in.readShort();
@@ -474,6 +588,12 @@ public class DateLiteral extends LiteralExpr {
this.type = Type.DATETIME;
} else if (dateLiteralType == DateLiteralType.DATE.value()) {
this.type = Type.DATE;
+ } else if (dateLiteralType == DateLiteralType.DATETIMEV2.value()) {
+ fromPackedDatetime(in.readLong());
+ this.type = Type.DATETIMEV2;
+ } else if (dateLiteralType == DateLiteralType.DATEV2.value()) {
+ fromPackedDatetime(in.readLong());
+ this.type = Type.DATEV2;
} else {
throw new IOException("Error date literal type : " + type);
}
@@ -612,17 +732,17 @@ public class DateLiteral extends LiteralExpr {
case 'y': // %y Year, numeric (two digits)
builder.appendTwoDigitYear(2020);
break;
+ // TODO(Gabriel): support microseconds in date literal
case 'f': // %f Microseconds (000000..999999)
case 'w': // %w Day of the week (0=Sunday..6=Saturday)
case 'U': // %U Week (00..53), where Sunday is the first day of the week
case 'u': // %u Week (00..53), where Monday is the first day of the week
case 'V': // %V Week (01..53), where Sunday is the first day of the week; used with %X
- case 'X':
- // %X Year for the week where Sunday is the first day of the week,
+ case 'X': // %X Year for the week where Sunday is the first day of the week,
// numeric, four digits; used with %V
case 'D': // %D Day of the month with English suffix (0th, 1st, 2nd, 3rd, …)
- throw new AnalysisException(
- String.format("%%%s not supported in date format string", character));
+ throw new AnalysisException(String.format("%%%s not supported in date format string",
+ character));
case '%': // %% A literal "%" character
builder.appendLiteral('%');
break;
@@ -641,9 +761,9 @@ public class DateLiteral extends LiteralExpr {
}
public LocalDateTime getTimeFormatter() throws AnalysisException {
- if (type.equals(Type.DATE)) {
+ if (type.equals(Type.DATE) || type.equals(Type.DATEV2)) {
return DATE_FORMATTER.parseLocalDateTime(getStringValue());
- } else if (type.equals(Type.DATETIME)) {
+ } else if (type.equals(Type.DATETIME) || type.equals(Type.DATETIMEV2)) {
return DATE_TIME_FORMATTER.parseLocalDateTime(getStringValue());
} else {
throw new AnalysisException("Not support date literal type");
@@ -706,7 +826,6 @@ public class DateLiteral extends LiteralExpr {
private long second;
private long microsecond;
-
@Override
public int hashCode() {
return 31 * super.hashCode() + Objects.hashCode(unixTimestamp(TimeZone.getDefault()));
@@ -963,8 +1082,8 @@ public class DateLiteral extends LiteralExpr {
}
} else if (format.charAt(fp) != ' ') {
if (format.charAt(fp) != value.charAt(vp)) {
- throw new InvalidFormatException("Invalid char: " + value.charAt(vp)
- + ", expected: " + format.charAt(fp));
+ throw new InvalidFormatException("Invalid char: " + value.charAt(vp) + ", expected: "
+ + format.charAt(fp));
}
fp++;
vp++;
@@ -1041,6 +1160,8 @@ public class DateLiteral extends LiteralExpr {
}
// Compute timestamp type
+ // TODO(Gabriel): we still use old version datetime/date and change this to new version when
+ // we think it's stable enough
if (datePartUsed) {
if (timePartUsed) {
this.type = Type.DATETIME;
@@ -1055,6 +1176,31 @@ public class DateLiteral extends LiteralExpr {
return 0;
}
+ public int fromDateFormatStr(String format, String value, boolean hasSubVal, Type type)
+ throws InvalidFormatException {
+ switch (type.getPrimitiveType()) {
+ case DATETIME:
+ case DATE:
+ return fromDateFormatStr(format, value, hasSubVal);
+ default:
+ int val = fromDateFormatStr(format, value, hasSubVal);
+ convertTypeToV2();
+ return val;
+ }
+ }
+
+ private void convertTypeToV2() {
+ switch (type.getPrimitiveType()) {
+ case DATETIME:
+ this.type = Type.DATETIMEV2;
+ break;
+ case DATE:
+ this.type = Type.DATEV2;
+ break;
+ default:
+ }
+ }
+
private boolean checkRange() {
return year > MAX_DATETIME.year || month > MAX_DATETIME.month || day > MAX_DATETIME.day
|| hour > MAX_DATETIME.hour || minute > MAX_DATETIME.minute || second > MAX_DATETIME.second
@@ -1285,4 +1431,38 @@ public class DateLiteral extends LiteralExpr {
throw new AnalysisException("Datetime value is out of range: " + dateStr);
}
}
+
+ public void fromDateStr(String dateStr, Type type) throws AnalysisException {
+ switch (type.getPrimitiveType()) {
+ case DATETIME:
+ case DATE:
+ fromDateStr(dateStr);
+ break;
+ default:
+ fromDateStr(dateStr);
+ convertTypeToV2();
+ }
+ }
+
+ public static Type getDefaultDateType(Type type) throws AnalysisException {
+ switch (type.getPrimitiveType()) {
+ case DATE:
+ if (Config.use_date_v2_by_default) {
+ return Type.DATEV2;
+ } else {
+ return Type.DATE;
+ }
+ case DATETIME:
+ if (Config.use_date_v2_by_default) {
+ return Type.DATETIMEV2;
+ } else {
+ return Type.DATETIME;
+ }
+ case DATEV2:
+ case DATETIMEV2:
+ return type;
+ default:
+ throw new AnalysisException("Invalid date type: " + type);
+ }
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/FloatLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/FloatLiteral.java
index f9b79d7ccc..37b405390f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FloatLiteral.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FloatLiteral.java
@@ -20,6 +20,7 @@ package org.apache.doris.analysis;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.Config;
import org.apache.doris.common.NotImplementedException;
import org.apache.doris.thrift.TExprNode;
import org.apache.doris.thrift.TExprNodeType;
@@ -129,12 +130,27 @@ public class FloatLiteral extends LiteralExpr {
public String getStringValue() {
// TODO: Here is weird use float to represent TIME type
// rethink whether it is reasonable to use this way
- if (type.equals(Type.TIME)) {
+ if (type.equals(Type.TIME) || type.equals(Type.TIMEV2)) {
return timeStrFromFloat(value);
}
return Double.toString(value);
}
+ public static Type getDefaultTimeType(Type type) throws AnalysisException {
+ switch (type.getPrimitiveType()) {
+ case TIME:
+ if (Config.use_date_v2_by_default) {
+ return Type.TIMEV2;
+ } else {
+ return Type.TIME;
+ }
+ case TIMEV2:
+ return type;
+ default:
+ throw new AnalysisException("Invalid time type: " + type);
+ }
+ }
+
@Override
public long getLongValue() {
return (long) value;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
index 03c4cbe6af..6b4ee81dc7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
@@ -1042,12 +1042,12 @@ public class FunctionCallExpr extends Expr {
Expr child1Result = getChild(1).getResultValue();
if (child1Result instanceof StringLiteral) {
if (DateLiteral.hasTimePart(((StringLiteral) child1Result).getStringValue())) {
- this.type = Type.DATETIME;
+ this.type = DateLiteral.getDefaultDateType(Type.DATETIME);
} else {
- this.type = Type.DATE;
+ this.type = DateLiteral.getDefaultDateType(Type.DATE);
}
} else {
- this.type = Type.DATETIME;
+ this.type = DateLiteral.getDefaultDateType(Type.DATETIME);
}
} else {
this.type = fn.getReturnType();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java
index 6d40c80acb..3d3574dd35 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java
@@ -80,6 +80,8 @@ public abstract class LiteralExpr extends Expr implements Comparable<LiteralExpr
break;
case DATE:
case DATETIME:
+ case DATEV2:
+ case DATETIMEV2:
literalExpr = new DateLiteral(value, type);
break;
default:
@@ -140,6 +142,8 @@ public abstract class LiteralExpr extends Expr implements Comparable<LiteralExpr
return LargeIntLiteral.createMinValue();
case DATE:
case DATETIME:
+ case DATEV2:
+ case DATETIMEV2:
return DateLiteral.createMinValue(type);
default:
throw new AnalysisException("Invalid data type for creating infinity: " + type);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAlterStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAlterStmt.java
index 4bcb72d9d8..75747d5022 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAlterStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAlterStmt.java
@@ -120,7 +120,8 @@ public class ShowAlterStmt extends ShowStmt {
throw new AnalysisException("Where clause : CreateTime/FinishTime =|>=|<=|>|<|!= "
+ "\"2019-12-02|2019-12-02 14:54:00\"");
}
- subExpr.setChild(1, ((StringLiteral) subExpr.getChild(1)).castTo(Type.DATETIME));
+ subExpr.setChild(1, (subExpr.getChild(1)).castTo(
+ DateLiteral.getDefaultDateType(Type.DATETIME)));
} else {
throw new AnalysisException("The columns of TableName/CreateTime/FinishTime/State are supported.");
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java
index a215cc3132..800f26fcaf 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java
@@ -175,11 +175,11 @@ public class StringLiteral extends LiteralExpr {
public LiteralExpr convertToDate(Type targetType) throws AnalysisException {
LiteralExpr newLiteral = null;
try {
- newLiteral = new DateLiteral(value, targetType);
+ newLiteral = new DateLiteral(value, DateLiteral.getDefaultDateType(targetType));
} catch (AnalysisException e) {
if (targetType.isScalarType(PrimitiveType.DATETIME)) {
- newLiteral = new DateLiteral(value, Type.DATE);
- newLiteral.setType(Type.DATETIME);
+ newLiteral = new DateLiteral(value, DateLiteral.getDefaultDateType(Type.DATE));
+ newLiteral.setType(DateLiteral.getDefaultDateType(Type.DATETIME));
} else {
throw e;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/TypeDef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/TypeDef.java
index bdbb4c88fc..21b3eea072 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/TypeDef.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/TypeDef.java
@@ -29,8 +29,6 @@ import org.apache.doris.catalog.StructType;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
-import com.google.common.base.Preconditions;
-
import java.util.ArrayList;
/**
@@ -52,6 +50,14 @@ public class TypeDef implements ParseNode {
return new TypeDef(ScalarType.createDecimalV2Type(precision, scale));
}
+ public static TypeDef createDatetimeV2(int scale) {
+ return new TypeDef(ScalarType.createDatetimeV2Type(scale));
+ }
+
+ public static TypeDef createTimeV2(int scale) {
+ return new TypeDef(ScalarType.createTimeV2Type(scale));
+ }
+
public static TypeDef createVarchar(int len) {
return new TypeDef(ScalarType.createVarchar(len));
}
@@ -68,9 +74,9 @@ public class TypeDef implements ParseNode {
// Check the max nesting depth before calling the recursive analyze() to avoid
// a stack overflow.
if (parsedType.exceedsMaxNestingDepth()) {
- throw new AnalysisException(
- String.format("Type exceeds the maximum nesting depth of %s:\n%s", Type.MAX_NESTING_DEPTH,
- parsedType.toSql()));
+ throw new AnalysisException(String.format(
+ "Type exceeds the maximum nesting depth of %s:\n%s",
+ Type.MAX_NESTING_DEPTH, parsedType.toSql()));
}
analyze(parsedType);
isAnalyzed = true;
@@ -111,6 +117,10 @@ public class TypeDef implements ParseNode {
if (type.isNull()) {
throw new AnalysisException("Unsupported data type: " + type.toSql());
}
+ if (!type.getPrimitiveType().isIntegerType()
+ && !type.getPrimitiveType().isCharFamily()) {
+ throw new AnalysisException("Array column just support INT/VARCHAR sub-type");
+ }
if (type.getPrimitiveType().isStringType()
&& !type.isAssignedStrLenInColDefinition()) {
type.setLength(1);
@@ -129,11 +139,9 @@ public class TypeDef implements ParseNode {
if (type == PrimitiveType.VARCHAR) {
name = "VARCHAR";
maxLen = ScalarType.MAX_VARCHAR_LENGTH;
- } else if (type == PrimitiveType.CHAR) {
+ } else {
name = "CHAR";
maxLen = ScalarType.MAX_CHAR_LENGTH;
- } else {
- Preconditions.checkState(false);
return;
}
int len = scalarType.getLength();
@@ -168,10 +176,25 @@ public class TypeDef implements ParseNode {
}
break;
}
+ case TIMEV2:
+ case DATETIMEV2: {
+ int precision = scalarType.decimalPrecision();
+ int scale = scalarType.decimalScale();
+ // precision: [1, 27]
+ if (precision != ScalarType.DATETIME_PRECISION) {
+ throw new AnalysisException("Precision of Datetime/Time must be " + ScalarType.DATETIME_PRECISION
+ + "." + " Precision was set to: " + precision + ".");
+ }
+ // scale: [0, 9]
+ if (scale < 0 || scale > 6) {
+ throw new AnalysisException("Scale of Datetime/Time must between 0 and 6."
+ + " Scale was set to: " + scale + ".");
+ }
+ break;
+ }
case INVALID_TYPE:
throw new AnalysisException("Invalid type.");
- default:
- break;
+ default: break;
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
index 02892573bc..a68ca2ce44 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
@@ -455,14 +455,6 @@ public class Column implements Writable {
|| other.getDataType() == PrimitiveType.STRING)) {
return;
}
-
- if (this.getPrecision() != other.getPrecision()) {
- throw new DdlException("Cannot change precision");
- }
-
- if (this.getScale() != other.getScale()) {
- throw new DdlException("Cannot change scale");
- }
}
public boolean nameEquals(String otherColName, boolean ignorePrefix) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnType.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnType.java
index 81bd0e3238..ae82bfeaa8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnType.java
@@ -111,8 +111,21 @@ public abstract class ColumnType {
schemaChangeMatrix[PrimitiveType.DECIMALV2.ordinal()][PrimitiveType.STRING.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.DATETIME.ordinal()][PrimitiveType.DATE.ordinal()] = true;
-
schemaChangeMatrix[PrimitiveType.DATE.ordinal()][PrimitiveType.DATETIME.ordinal()] = true;
+ schemaChangeMatrix[PrimitiveType.DATETIME.ordinal()][PrimitiveType.DATEV2.ordinal()] = true;
+ schemaChangeMatrix[PrimitiveType.DATE.ordinal()][PrimitiveType.DATETIMEV2.ordinal()] = true;
+ schemaChangeMatrix[PrimitiveType.DATETIME.ordinal()][PrimitiveType.DATETIMEV2.ordinal()] = true;
+ schemaChangeMatrix[PrimitiveType.DATE.ordinal()][PrimitiveType.DATEV2.ordinal()] = true;
+
+ schemaChangeMatrix[PrimitiveType.DATETIMEV2.ordinal()][PrimitiveType.DATE.ordinal()] = true;
+ schemaChangeMatrix[PrimitiveType.DATEV2.ordinal()][PrimitiveType.DATETIME.ordinal()] = true;
+ schemaChangeMatrix[PrimitiveType.DATETIMEV2.ordinal()][PrimitiveType.DATEV2.ordinal()] = true;
+ schemaChangeMatrix[PrimitiveType.DATEV2.ordinal()][PrimitiveType.DATETIMEV2.ordinal()] = true;
+ schemaChangeMatrix[PrimitiveType.DATETIMEV2.ordinal()][PrimitiveType.DATETIME.ordinal()] = true;
+ schemaChangeMatrix[PrimitiveType.DATEV2.ordinal()][PrimitiveType.DATE.ordinal()] = true;
+
+ // we should support schema change between different precision
+ schemaChangeMatrix[PrimitiveType.DATETIMEV2.ordinal()][PrimitiveType.DATETIMEV2.ordinal()] = true;
}
static boolean isSchemaChangeAllowed(Type lhs, Type rhs) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Function.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Function.java
index 9079deaa2d..93c218d0b4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Function.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Function.java
@@ -496,6 +496,7 @@ public class Function implements Writable {
return "float_val";
case DOUBLE:
case TIME:
+ case TIMEV2:
return "double_val";
case VARCHAR:
case CHAR:
@@ -506,6 +507,8 @@ public class Function implements Writable {
return "string_val";
case DATE:
case DATETIME:
+ case DATEV2:
+ case DATETIMEV2:
return "datetime_val";
case DECIMALV2:
return "decimalv2_val";
@@ -535,6 +538,7 @@ public class Function implements Writable {
return "FloatVal";
case DOUBLE:
case TIME:
+ case TIMEV2:
return "DoubleVal";
case VARCHAR:
case CHAR:
@@ -545,6 +549,8 @@ public class Function implements Writable {
return "StringVal";
case DATE:
case DATETIME:
+ case DATEV2:
+ case DATETIMEV2:
return "DateTimeVal";
case DECIMALV2:
return "DecimalV2Val";
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java
index 3b44365626..d8bcb961e2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java
@@ -310,6 +310,8 @@ public class PartitionKey implements Comparable<PartitionKey>, Writable {
break;
case DATE:
case DATETIME:
+ case DATEV2:
+ case DATETIMEV2:
literal = DateLiteral.read(in);
break;
case CHAR:
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/PrimitiveType.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/PrimitiveType.java
index 32dc2bec55..7f01110cba 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/PrimitiveType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/PrimitiveType.java
@@ -54,6 +54,9 @@ public enum PrimitiveType {
HLL("HLL", 16, TPrimitiveType.HLL),
BITMAP("BITMAP", 16, TPrimitiveType.OBJECT),
QUANTILE_STATE("QUANTILE_STATE", 16, TPrimitiveType.QUANTILE_STATE),
+ DATEV2("DATEV2", 4, TPrimitiveType.DATEV2),
+ DATETIMEV2("DATETIMEV2", 8, TPrimitiveType.DATETIMEV2),
+ TIMEV2("TIMEV2", 8, TPrimitiveType.TIMEV2),
// sizeof(CollectionValue)
ARRAY("ARRAY", 32, TPrimitiveType.ARRAY),
@@ -85,12 +88,15 @@ public enum PrimitiveType {
builder.put(NULL_TYPE, DOUBLE);
builder.put(NULL_TYPE, DATE);
builder.put(NULL_TYPE, DATETIME);
+ builder.put(NULL_TYPE, DATEV2);
+ builder.put(NULL_TYPE, DATETIMEV2);
builder.put(NULL_TYPE, DECIMALV2);
builder.put(NULL_TYPE, CHAR);
builder.put(NULL_TYPE, VARCHAR);
builder.put(NULL_TYPE, STRING);
builder.put(NULL_TYPE, BITMAP); //TODO(weixiang):why null type can cast to bitmap?
builder.put(NULL_TYPE, TIME);
+ builder.put(NULL_TYPE, TIMEV2);
// Boolean
builder.put(BOOLEAN, BOOLEAN);
builder.put(BOOLEAN, TINYINT);
@@ -102,6 +108,8 @@ public enum PrimitiveType {
builder.put(BOOLEAN, DOUBLE);
builder.put(BOOLEAN, DATE);
builder.put(BOOLEAN, DATETIME);
+ builder.put(BOOLEAN, DATEV2);
+ builder.put(BOOLEAN, DATETIMEV2);
builder.put(BOOLEAN, DECIMALV2);
builder.put(BOOLEAN, VARCHAR);
builder.put(BOOLEAN, STRING);
@@ -116,6 +124,8 @@ public enum PrimitiveType {
builder.put(TINYINT, DOUBLE);
builder.put(TINYINT, DATE);
builder.put(TINYINT, DATETIME);
+ builder.put(TINYINT, DATEV2);
+ builder.put(TINYINT, DATETIMEV2);
builder.put(TINYINT, DECIMALV2);
builder.put(TINYINT, VARCHAR);
builder.put(TINYINT, STRING);
@@ -130,6 +140,8 @@ public enum PrimitiveType {
builder.put(SMALLINT, DOUBLE);
builder.put(SMALLINT, DATE);
builder.put(SMALLINT, DATETIME);
+ builder.put(SMALLINT, DATEV2);
+ builder.put(SMALLINT, DATETIMEV2);
builder.put(SMALLINT, DECIMALV2);
builder.put(SMALLINT, VARCHAR);
builder.put(SMALLINT, STRING);
@@ -144,6 +156,8 @@ public enum PrimitiveType {
builder.put(INT, DOUBLE);
builder.put(INT, DATE);
builder.put(INT, DATETIME);
+ builder.put(INT, DATEV2);
+ builder.put(INT, DATETIMEV2);
builder.put(INT, DECIMALV2);
builder.put(INT, VARCHAR);
builder.put(INT, STRING);
@@ -158,6 +172,8 @@ public enum PrimitiveType {
builder.put(BIGINT, DOUBLE);
builder.put(BIGINT, DATE);
builder.put(BIGINT, DATETIME);
+ builder.put(BIGINT, DATEV2);
+ builder.put(BIGINT, DATETIMEV2);
builder.put(BIGINT, DECIMALV2);
builder.put(BIGINT, VARCHAR);
builder.put(BIGINT, STRING);
@@ -172,6 +188,8 @@ public enum PrimitiveType {
builder.put(LARGEINT, DOUBLE);
builder.put(LARGEINT, DATE);
builder.put(LARGEINT, DATETIME);
+ builder.put(LARGEINT, DATEV2);
+ builder.put(LARGEINT, DATETIMEV2);
builder.put(LARGEINT, DECIMALV2);
builder.put(LARGEINT, VARCHAR);
builder.put(LARGEINT, STRING);
@@ -186,6 +204,8 @@ public enum PrimitiveType {
builder.put(FLOAT, DOUBLE);
builder.put(FLOAT, DATE);
builder.put(FLOAT, DATETIME);
+ builder.put(FLOAT, DATEV2);
+ builder.put(FLOAT, DATETIMEV2);
builder.put(FLOAT, DECIMALV2);
builder.put(FLOAT, VARCHAR);
builder.put(FLOAT, STRING);
@@ -200,6 +220,8 @@ public enum PrimitiveType {
builder.put(DOUBLE, DOUBLE);
builder.put(DOUBLE, DATE);
builder.put(DOUBLE, DATETIME);
+ builder.put(DOUBLE, DATEV2);
+ builder.put(DOUBLE, DATETIMEV2);
builder.put(DOUBLE, DECIMALV2);
builder.put(DOUBLE, VARCHAR);
builder.put(DOUBLE, STRING);
@@ -214,6 +236,8 @@ public enum PrimitiveType {
builder.put(DATE, DOUBLE);
builder.put(DATE, DATE);
builder.put(DATE, DATETIME);
+ builder.put(DATE, DATEV2);
+ builder.put(DATE, DATETIMEV2);
builder.put(DATE, DECIMALV2);
builder.put(DATE, VARCHAR);
builder.put(DATE, STRING);
@@ -228,9 +252,43 @@ public enum PrimitiveType {
builder.put(DATETIME, DOUBLE);
builder.put(DATETIME, DATE);
builder.put(DATETIME, DATETIME);
+ builder.put(DATETIME, DATEV2);
+ builder.put(DATETIME, DATETIMEV2);
builder.put(DATETIME, DECIMALV2);
builder.put(DATETIME, VARCHAR);
builder.put(DATETIME, STRING);
+ // DateV2
+ builder.put(DATEV2, BOOLEAN);
+ builder.put(DATEV2, TINYINT);
+ builder.put(DATEV2, SMALLINT);
+ builder.put(DATEV2, INT);
+ builder.put(DATEV2, BIGINT);
+ builder.put(DATEV2, LARGEINT);
+ builder.put(DATEV2, FLOAT);
+ builder.put(DATEV2, DOUBLE);
+ builder.put(DATEV2, DATE);
+ builder.put(DATEV2, DATETIME);
+ builder.put(DATEV2, DATEV2);
+ builder.put(DATEV2, DATETIMEV2);
+ builder.put(DATEV2, DECIMALV2);
+ builder.put(DATEV2, VARCHAR);
+ builder.put(DATEV2, STRING);
+ // DatetimeV2
+ builder.put(DATETIMEV2, BOOLEAN);
+ builder.put(DATETIMEV2, TINYINT);
+ builder.put(DATETIMEV2, SMALLINT);
+ builder.put(DATETIMEV2, INT);
+ builder.put(DATETIMEV2, BIGINT);
+ builder.put(DATETIMEV2, LARGEINT);
+ builder.put(DATETIMEV2, FLOAT);
+ builder.put(DATETIMEV2, DOUBLE);
+ builder.put(DATETIMEV2, DATE);
+ builder.put(DATETIMEV2, DATETIME);
+ builder.put(DATETIMEV2, DATEV2);
+ builder.put(DATETIMEV2, DATETIMEV2);
+ builder.put(DATETIMEV2, DECIMALV2);
+ builder.put(DATETIMEV2, VARCHAR);
+ builder.put(DATETIMEV2, STRING);
// Char
builder.put(CHAR, BOOLEAN);
builder.put(CHAR, TINYINT);
@@ -243,6 +301,8 @@ public enum PrimitiveType {
builder.put(CHAR, DOUBLE);
builder.put(CHAR, DATE);
builder.put(CHAR, DATETIME);
+ builder.put(CHAR, DATEV2);
+ builder.put(CHAR, DATETIMEV2);
builder.put(CHAR, DECIMALV2);
builder.put(CHAR, VARCHAR);
builder.put(CHAR, STRING);
@@ -257,6 +317,8 @@ public enum PrimitiveType {
builder.put(VARCHAR, DOUBLE);
builder.put(VARCHAR, DATE);
builder.put(VARCHAR, DATETIME);
+ builder.put(VARCHAR, DATEV2);
+ builder.put(VARCHAR, DATETIMEV2);
builder.put(VARCHAR, DECIMALV2);
builder.put(VARCHAR, VARCHAR);
builder.put(VARCHAR, STRING);
@@ -272,6 +334,8 @@ public enum PrimitiveType {
builder.put(STRING, DOUBLE);
builder.put(STRING, DATE);
builder.put(STRING, DATETIME);
+ builder.put(STRING, DATEV2);
+ builder.put(STRING, DATETIMEV2);
builder.put(STRING, DECIMALV2);
builder.put(STRING, VARCHAR);
builder.put(STRING, STRING);
@@ -300,8 +364,14 @@ public enum PrimitiveType {
// TIME
builder.put(TIME, TIME);
+ builder.put(TIME, TIMEV2);
builder.put(TIME, DOUBLE);
+ //TIMEV2
+ builder.put(TIMEV2, TIME);
+ builder.put(TIMEV2, TIMEV2);
+ builder.put(TIMEV2, DOUBLE);
+
implicitCastMap = builder.build();
}
@@ -344,6 +414,9 @@ public enum PrimitiveType {
supportedTypes.add(DATE);
supportedTypes.add(DATETIME);
supportedTypes.add(TIME);
+ supportedTypes.add(DATEV2);
+ supportedTypes.add(DATETIMEV2);
+ supportedTypes.add(TIMEV2);
supportedTypes.add(DECIMALV2);
supportedTypes.add(BITMAP);
supportedTypes.add(ARRAY);
@@ -398,14 +471,15 @@ public enum PrimitiveType {
compatibilityMatrix[NULL_TYPE.ordinal()][DOUBLE.ordinal()] = DOUBLE;
compatibilityMatrix[NULL_TYPE.ordinal()][DATE.ordinal()] = DATE;
compatibilityMatrix[NULL_TYPE.ordinal()][DATETIME.ordinal()] = DATETIME;
+ compatibilityMatrix[NULL_TYPE.ordinal()][DATEV2.ordinal()] = DATEV2;
+ compatibilityMatrix[NULL_TYPE.ordinal()][DATETIMEV2.ordinal()] = DATETIMEV2;
compatibilityMatrix[NULL_TYPE.ordinal()][CHAR.ordinal()] = CHAR;
compatibilityMatrix[NULL_TYPE.ordinal()][VARCHAR.ordinal()] = VARCHAR;
compatibilityMatrix[NULL_TYPE.ordinal()][STRING.ordinal()] = STRING;
compatibilityMatrix[NULL_TYPE.ordinal()][DECIMALV2.ordinal()] = DECIMALV2;
compatibilityMatrix[NULL_TYPE.ordinal()][TIME.ordinal()] = TIME;
- //TODO(weixiang): bitmap can be null?
+ compatibilityMatrix[NULL_TYPE.ordinal()][TIMEV2.ordinal()] = TIMEV2;
compatibilityMatrix[NULL_TYPE.ordinal()][BITMAP.ordinal()] = BITMAP;
- //TODO(weixiang): QUANTILE_STATE can be null?
compatibilityMatrix[NULL_TYPE.ordinal()][QUANTILE_STATE.ordinal()] = QUANTILE_STATE;
compatibilityMatrix[BOOLEAN.ordinal()][BOOLEAN.ordinal()] = BOOLEAN;
@@ -418,11 +492,14 @@ public enum PrimitiveType {
compatibilityMatrix[BOOLEAN.ordinal()][DOUBLE.ordinal()] = DOUBLE;
compatibilityMatrix[BOOLEAN.ordinal()][DATE.ordinal()] = INVALID_TYPE;
compatibilityMatrix[BOOLEAN.ordinal()][DATETIME.ordinal()] = INVALID_TYPE;
+ compatibilityMatrix[BOOLEAN.ordinal()][DATEV2.ordinal()] = INVALID_TYPE;
+ compatibilityMatrix[BOOLEAN.ordinal()][DATETIMEV2.ordinal()] = INVALID_TYPE;
compatibilityMatrix[BOOLEAN.ordinal()][CHAR.ordinal()] = INVALID_TYPE;
compatibilityMatrix[BOOLEAN.ordinal()][VARCHAR.ordinal()] = INVALID_TYPE;
compatibilityMatrix[BOOLEAN.ordinal()][STRING.ordinal()] = INVALID_TYPE;
compatibilityMatrix[BOOLEAN.ordinal()][DECIMALV2.ordinal()] = DECIMALV2;
compatibilityMatrix[BOOLEAN.ordinal()][TIME.ordinal()] = TIME;
+ compatibilityMatrix[BOOLEAN.ordinal()][TIMEV2.ordinal()] = TIMEV2;
compatibilityMatrix[TINYINT.ordinal()][TINYINT.ordinal()] = TINYINT;
compatibilityMatrix[TINYINT.ordinal()][SMALLINT.ordinal()] = SMALLINT;
@@ -433,11 +510,14 @@ public enum PrimitiveType {
compatibilityMatrix[TINYINT.ordinal()][DOUBLE.ordinal()] = DOUBLE;
compatibilityMatrix[TINYINT.ordinal()][DATE.ordinal()] = INVALID_TYPE;
compatibilityMatrix[TINYINT.ordinal()][DATETIME.ordinal()] = INVALID_TYPE;
+ compatibilityMatrix[TINYINT.ordinal()][DATEV2.ordinal()] = INVALID_TYPE;
+ compatibilityMatrix[TINYINT.ordinal()][DATETIMEV2.ordinal()] = INVALID_TYPE;
compatibilityMatrix[TINYINT.ordinal()][CHAR.ordinal()] = INVALID_TYPE;
compatibilityMatrix[TINYINT.ordinal()][VARCHAR.ordinal()] = INVALID_TYPE;
compatibilityMatrix[TINYINT.ordinal()][STRING.ordinal()] = INVALID_TYPE;
compatibilityMatrix[TINYINT.ordinal()][DECIMALV2.ordinal()] = DECIMALV2;
compatibilityMatrix[TINYINT.ordinal()][TIME.ordinal()] = TIME;
+ compatibilityMatrix[TINYINT.ordinal()][TIMEV2.ordinal()] = TIMEV2;
compatibilityMatrix[SMALLINT.ordinal()][SMALLINT.ordinal()] = SMALLINT;
compatibilityMatrix[SMALLINT.ordinal()][INT.ordinal()] = INT;
@@ -447,11 +527,14 @@ public enum PrimitiveType {
compatibilityMatrix[SMALLINT.ordinal()][DOUBLE.ordinal()] = DOUBLE;
compatibilityMatrix[SMALLINT.ordinal()][DATE.ordinal()] = INVALID_TYPE;
compatibilityMatrix[SMALLINT.ordinal()][DATETIME.ordinal()] = INVALID_TYPE;
+ compatibilityMatrix[SMALLINT.ordinal()][DATEV2.ordinal()] = INVALID_TYPE;
+ compatibilityMatrix[SMALLINT.ordinal()][DATETIMEV2.ordinal()] = INVALID_TYPE;
compatibilityMatrix[SMALLINT.ordinal()][CHAR.ordinal()] = INVALID_TYPE;
compatibilityMatrix[SMALLINT.ordinal()][VARCHAR.ordinal()] = INVALID_TYPE;
compatibilityMatrix[SMALLINT.ordinal()][STRING.ordinal()] = INVALID_TYPE;
compatibilityMatrix[SMALLINT.ordinal()][DECIMALV2.ordinal()] = DECIMALV2;
compatibilityMatrix[SMALLINT.ordinal()][TIME.ordinal()] = TIME;
+ compatibilityMatrix[SMALLINT.ordinal()][TIMEV2.ordinal()] = TIMEV2;
compatibilityMatrix[INT.ordinal()][INT.ordinal()] = INT;
compatibilityMatrix[INT.ordinal()][BIGINT.ordinal()] = BIGINT;
@@ -460,11 +543,14 @@ public enum PrimitiveType {
compatibilityMatrix[INT.ordinal()][DOUBLE.ordinal()] = DOUBLE;
compatibilityMatrix[INT.ordinal()][DATE.ordinal()] = INVALID_TYPE;
compatibilityMatrix[INT.ordinal()][DATETIME.ordinal()] = INVALID_TYPE;
+ compatibilityMatrix[INT.ordinal()][DATEV2.ordinal()] = INVALID_TYPE;
+ compatibilityMatrix[INT.ordinal()][DATETIMEV2.ordinal()] = INVALID_TYPE;
compatibilityMatrix[INT.ordinal()][CHAR.ordinal()] = INVALID_TYPE;
... 3976 lines suppressed ...
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org