You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by ja...@apache.org on 2026/10/09 01:44:39 UTC
(doris) branch master updated: [fix](be) Skip NULL-masked payload in strict cast (#67981)
This is an automated email from the ASF dual-hosted git repository.
jacktengg pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new d43784149f1 [fix](be) Skip NULL-masked payload in strict cast (#67981)
d43784149f1 is described below
commit d43784149f19cee27aeacdeacf4218d53b9b1adf
Author: TengJianPing <te...@selectdb.com>
AuthorDate: Fri Oct 9 09:44:31 2026 +0800
[fix](be) Skip NULL-masked payload in strict cast (#67981)
### What problem does this PR solve?
Issue Number: close #xxx
Related PR: #xxx
Problem Summary:
### Release note
None
### Check List (For Author)
- Test <!-- At least one of them must be included. -->
- [ ] Regression test
- [ ] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason <!-- Add your reason? -->
- Behavior changed:
- [ ] No.
- [ ] Yes. <!-- Explain the behavior change -->
- Does this need documentation?
- [ ] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---
.../data_type_date_or_datetime_serde.cpp | 96 +++++---
.../data_type_date_or_datetime_serde.h | 9 +-
.../data_type_serde/data_type_datetimev2_serde.cpp | 54 +++--
.../data_type_serde/data_type_datetimev2_serde.h | 10 +-
.../data_type_serde/data_type_datev2_serde.cpp | 54 +++--
.../core/data_type_serde/data_type_datev2_serde.h | 10 +-
.../data_type_serde/data_type_decimal_serde.cpp | 18 +-
.../core/data_type_serde/data_type_decimal_serde.h | 11 +-
.../data_type_serde/data_type_number_serde.cpp | 18 +-
.../core/data_type_serde/data_type_number_serde.h | 11 +-
be/src/core/data_type_serde/data_type_serde.cpp | 19 +-
be/src/core/data_type_serde/data_type_serde.h | 13 +-
.../data_type_serde/data_type_string_serde.cpp | 9 +-
.../core/data_type_serde/data_type_string_serde.h | 6 +-
.../core/data_type_serde/data_type_time_serde.cpp | 54 +++--
be/src/core/data_type_serde/data_type_time_serde.h | 9 +-
.../data_type_timestamp_ns_serde.cpp | 61 +++--
.../data_type_serde/data_type_timestamp_ns_serde.h | 16 +-
be/src/exprs/function/cast/cast_base.cpp | 53 ++++-
be/src/exprs/function/cast/cast_base.h | 26 +++
be/src/exprs/function/cast/cast_to_array.h | 9 +-
be/src/exprs/function/cast/cast_to_date.h | 6 +-
be/src/exprs/function/cast/cast_to_decimal.h | 16 ++
be/src/exprs/function/cast/cast_to_int.h | 9 +
be/src/exprs/function/cast/cast_to_jsonb.h | 17 +-
be/src/exprs/function/cast/cast_to_map.h | 28 ++-
be/src/exprs/function/cast/cast_to_struct.h | 35 ++-
be/src/exprs/function/cast/cast_to_timestamp_ns.h | 15 +-
be/src/exprs/function/cast/function_cast.cpp | 5 +-
.../data_type_serde_datelike_batch_test.cpp | 98 ++++++++
.../function/cast/cast_jsonb_null_mask_test.cpp | 212 ++++++++++++++++++
.../exprs/function/cast/cast_timestamp_ns_test.cpp | 40 ++++
be/test/exprs/function/cast/cast_to_array_test.cpp | 140 ++++++++++++
be/test/exprs/function/cast/cast_to_date_test.cpp | 52 +++++
be/test/exprs/function/cast/cast_to_decimal.cpp | 41 ++++
be/test/exprs/function/cast/cast_to_integer.cpp | 62 +++++-
be/test/exprs/function/cast/cast_to_map_test.cpp | 97 ++++++++
.../exprs/function/cast/cast_to_struct_test.cpp | 87 ++++++++
.../test_cast_complex_null_covered_payload.out | 97 ++++++++
...yint_from_int_nullable_null_covered_payload.out | 49 ++++
.../test_cast_complex_null_covered_payload.groovy | 248 +++++++++++++++++++++
...t_from_int_nullable_null_covered_payload.groovy | 153 +++++++++++++
42 files changed, 1895 insertions(+), 178 deletions(-)
diff --git a/be/src/core/data_type_serde/data_type_date_or_datetime_serde.cpp b/be/src/core/data_type_serde/data_type_date_or_datetime_serde.cpp
index 1ff362c6d5e..04ebbc364f6 100644
--- a/be/src/core/data_type_serde/data_type_date_or_datetime_serde.cpp
+++ b/be/src/core/data_type_serde/data_type_date_or_datetime_serde.cpp
@@ -505,12 +505,16 @@ Status DataTypeDateSerDe<T>::from_int_batch(const typename IntDataType::ColumnTy
template <PrimitiveType T>
template <typename IntDataType>
Status DataTypeDateSerDe<T>::from_int_strict_mode_batch(
- const typename IntDataType::ColumnType& int_col, IColumn& target_col) const {
+ const typename IntDataType::ColumnType& int_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const {
auto& col_data = assert_cast<ColumnType&>(target_col);
col_data.resize(int_col.size());
CastParameters params {.status = Status::OK(), .is_strict = true};
for (size_t i = 0; i < int_col.size(); ++i) {
+ if (null_map && null_map[i]) {
+ continue;
+ }
CppType val;
CastToDateOrDatetime::from_integer<DatelikeParseMode::STRICT,
IsDatetime ? DatelikeTargetType::DATE_TIME
@@ -556,12 +560,16 @@ Status DataTypeDateSerDe<T>::from_float_batch(const typename FloatDataType::Colu
template <PrimitiveType T>
template <typename FloatDataType>
Status DataTypeDateSerDe<T>::from_float_strict_mode_batch(
- const typename FloatDataType::ColumnType& float_col, IColumn& target_col) const {
+ const typename FloatDataType::ColumnType& float_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const {
auto& col_data = assert_cast<ColumnType&>(target_col);
col_data.resize(float_col.size());
CastParameters params {.status = Status::OK(), .is_strict = true};
for (size_t i = 0; i < float_col.size(); ++i) {
+ if (null_map && null_map[i]) {
+ continue;
+ }
CppType val;
CastToDateOrDatetime::from_float<DatelikeParseMode::STRICT,
IsDatetime ? DatelikeTargetType::DATE_TIME
@@ -609,12 +617,16 @@ Status DataTypeDateSerDe<T>::from_decimal_batch(
template <PrimitiveType T>
template <typename DecimalDataType>
Status DataTypeDateSerDe<T>::from_decimal_strict_mode_batch(
- const typename DecimalDataType::ColumnType& decimal_col, IColumn& target_col) const {
+ const typename DecimalDataType::ColumnType& decimal_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const {
auto& col_data = assert_cast<ColumnType&>(target_col);
col_data.resize(decimal_col.size());
CastParameters params {.status = Status::OK(), .is_strict = true};
for (size_t i = 0; i < decimal_col.size(); ++i) {
+ if (null_map && null_map[i]) {
+ continue;
+ }
CppType val;
CastToDateOrDatetime::from_decimal<DatelikeParseMode::STRICT,
IsDatetime ? DatelikeTargetType::DATE_TIME
@@ -665,23 +677,30 @@ template Status DataTypeDateSerDe<TYPE_DATE>::from_int_batch<DataTypeInt64>(
template Status DataTypeDateSerDe<TYPE_DATE>::from_int_batch<DataTypeInt128>(
const DataTypeInt128::ColumnType& int_col, ColumnNullable& target_col) const;
template Status DataTypeDateSerDe<TYPE_DATE>::from_int_strict_mode_batch<DataTypeInt8>(
- const DataTypeInt8::ColumnType& int_col, IColumn& target_col) const;
+ const DataTypeInt8::ColumnType& int_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
template Status DataTypeDateSerDe<TYPE_DATE>::from_int_strict_mode_batch<DataTypeInt16>(
- const DataTypeInt16::ColumnType& int_col, IColumn& target_col) const;
+ const DataTypeInt16::ColumnType& int_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
template Status DataTypeDateSerDe<TYPE_DATE>::from_int_strict_mode_batch<DataTypeInt32>(
- const DataTypeInt32::ColumnType& int_col, IColumn& target_col) const;
+ const DataTypeInt32::ColumnType& int_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
template Status DataTypeDateSerDe<TYPE_DATE>::from_int_strict_mode_batch<DataTypeInt64>(
- const DataTypeInt64::ColumnType& int_col, IColumn& target_col) const;
+ const DataTypeInt64::ColumnType& int_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
template Status DataTypeDateSerDe<TYPE_DATE>::from_int_strict_mode_batch<DataTypeInt128>(
- const DataTypeInt128::ColumnType& int_col, IColumn& target_col) const;
+ const DataTypeInt128::ColumnType& int_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
template Status DataTypeDateSerDe<TYPE_DATE>::from_float_batch<DataTypeFloat32>(
const DataTypeFloat32::ColumnType& float_col, ColumnNullable& target_col) const;
template Status DataTypeDateSerDe<TYPE_DATE>::from_float_batch<DataTypeFloat64>(
const DataTypeFloat64::ColumnType& float_col, ColumnNullable& target_col) const;
template Status DataTypeDateSerDe<TYPE_DATE>::from_float_strict_mode_batch<DataTypeFloat32>(
- const DataTypeFloat32::ColumnType& float_col, IColumn& target_col) const;
+ const DataTypeFloat32::ColumnType& float_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
template Status DataTypeDateSerDe<TYPE_DATE>::from_float_strict_mode_batch<DataTypeFloat64>(
- const DataTypeFloat64::ColumnType& float_col, IColumn& target_col) const;
+ const DataTypeFloat64::ColumnType& float_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
template Status DataTypeDateSerDe<TYPE_DATE>::from_decimal_batch<DataTypeDecimal32>(
const DataTypeDecimal32::ColumnType& decimal_col, ColumnNullable& target_col) const;
template Status DataTypeDateSerDe<TYPE_DATE>::from_decimal_batch<DataTypeDecimal64>(
@@ -693,15 +712,20 @@ template Status DataTypeDateSerDe<TYPE_DATE>::from_decimal_batch<DataTypeDecimal
template Status DataTypeDateSerDe<TYPE_DATE>::from_decimal_batch<DataTypeDecimal256>(
const DataTypeDecimal256::ColumnType& decimal_col, ColumnNullable& target_col) const;
template Status DataTypeDateSerDe<TYPE_DATE>::from_decimal_strict_mode_batch<DataTypeDecimal32>(
- const DataTypeDecimal32::ColumnType& decimal_col, IColumn& target_col) const;
+ const DataTypeDecimal32::ColumnType& decimal_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
template Status DataTypeDateSerDe<TYPE_DATE>::from_decimal_strict_mode_batch<DataTypeDecimal64>(
- const DataTypeDecimal64::ColumnType& decimal_col, IColumn& target_col) const;
+ const DataTypeDecimal64::ColumnType& decimal_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
template Status DataTypeDateSerDe<TYPE_DATE>::from_decimal_strict_mode_batch<DataTypeDecimalV2>(
- const DataTypeDecimalV2::ColumnType& decimal_col, IColumn& target_col) const;
+ const DataTypeDecimalV2::ColumnType& decimal_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
template Status DataTypeDateSerDe<TYPE_DATE>::from_decimal_strict_mode_batch<DataTypeDecimal128>(
- const DataTypeDecimal128::ColumnType& decimal_col, IColumn& target_col) const;
+ const DataTypeDecimal128::ColumnType& decimal_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
template Status DataTypeDateSerDe<TYPE_DATE>::from_decimal_strict_mode_batch<DataTypeDecimal256>(
- const DataTypeDecimal256::ColumnType& decimal_col, IColumn& target_col) const;
+ const DataTypeDecimal256::ColumnType& decimal_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
template Status DataTypeDateSerDe<TYPE_DATETIME>::from_int_batch<DataTypeInt8>(
const DataTypeInt8::ColumnType& int_col, ColumnNullable& target_col) const;
@@ -714,23 +738,30 @@ template Status DataTypeDateSerDe<TYPE_DATETIME>::from_int_batch<DataTypeInt64>(
template Status DataTypeDateSerDe<TYPE_DATETIME>::from_int_batch<DataTypeInt128>(
const DataTypeInt128::ColumnType& int_col, ColumnNullable& target_col) const;
template Status DataTypeDateSerDe<TYPE_DATETIME>::from_int_strict_mode_batch<DataTypeInt8>(
- const DataTypeInt8::ColumnType& int_col, IColumn& target_col) const;
+ const DataTypeInt8::ColumnType& int_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
template Status DataTypeDateSerDe<TYPE_DATETIME>::from_int_strict_mode_batch<DataTypeInt16>(
- const DataTypeInt16::ColumnType& int_col, IColumn& target_col) const;
+ const DataTypeInt16::ColumnType& int_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
template Status DataTypeDateSerDe<TYPE_DATETIME>::from_int_strict_mode_batch<DataTypeInt32>(
- const DataTypeInt32::ColumnType& int_col, IColumn& target_col) const;
+ const DataTypeInt32::ColumnType& int_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
template Status DataTypeDateSerDe<TYPE_DATETIME>::from_int_strict_mode_batch<DataTypeInt64>(
- const DataTypeInt64::ColumnType& int_col, IColumn& target_col) const;
+ const DataTypeInt64::ColumnType& int_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
template Status DataTypeDateSerDe<TYPE_DATETIME>::from_int_strict_mode_batch<DataTypeInt128>(
- const DataTypeInt128::ColumnType& int_col, IColumn& target_col) const;
+ const DataTypeInt128::ColumnType& int_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
template Status DataTypeDateSerDe<TYPE_DATETIME>::from_float_batch<DataTypeFloat32>(
const DataTypeFloat32::ColumnType& float_col, ColumnNullable& target_col) const;
template Status DataTypeDateSerDe<TYPE_DATETIME>::from_float_batch<DataTypeFloat64>(
const DataTypeFloat64::ColumnType& float_col, ColumnNullable& target_col) const;
template Status DataTypeDateSerDe<TYPE_DATETIME>::from_float_strict_mode_batch<DataTypeFloat32>(
- const DataTypeFloat32::ColumnType& float_col, IColumn& target_col) const;
+ const DataTypeFloat32::ColumnType& float_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
template Status DataTypeDateSerDe<TYPE_DATETIME>::from_float_strict_mode_batch<DataTypeFloat64>(
- const DataTypeFloat64::ColumnType& float_col, IColumn& target_col) const;
+ const DataTypeFloat64::ColumnType& float_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
template Status DataTypeDateSerDe<TYPE_DATETIME>::from_decimal_batch<DataTypeDecimal32>(
const DataTypeDecimal32::ColumnType& decimal_col, ColumnNullable& target_col) const;
template Status DataTypeDateSerDe<TYPE_DATETIME>::from_decimal_batch<DataTypeDecimal64>(
@@ -742,15 +773,18 @@ template Status DataTypeDateSerDe<TYPE_DATETIME>::from_decimal_batch<DataTypeDec
template Status DataTypeDateSerDe<TYPE_DATETIME>::from_decimal_batch<DataTypeDecimal256>(
const DataTypeDecimal256::ColumnType& decimal_col, ColumnNullable& target_col) const;
template Status DataTypeDateSerDe<TYPE_DATETIME>::from_decimal_strict_mode_batch<DataTypeDecimal32>(
- const DataTypeDecimal32::ColumnType& decimal_col, IColumn& target_col) const;
+ const DataTypeDecimal32::ColumnType& decimal_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
template Status DataTypeDateSerDe<TYPE_DATETIME>::from_decimal_strict_mode_batch<DataTypeDecimal64>(
- const DataTypeDecimal64::ColumnType& decimal_col, IColumn& target_col) const;
+ const DataTypeDecimal64::ColumnType& decimal_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
template Status DataTypeDateSerDe<TYPE_DATETIME>::from_decimal_strict_mode_batch<DataTypeDecimalV2>(
- const DataTypeDecimalV2::ColumnType& decimal_col, IColumn& target_col) const;
-template Status
-DataTypeDateSerDe<TYPE_DATETIME>::from_decimal_strict_mode_batch<DataTypeDecimal128>(
- const DataTypeDecimal128::ColumnType& decimal_col, IColumn& target_col) const;
-template Status
-DataTypeDateSerDe<TYPE_DATETIME>::from_decimal_strict_mode_batch<DataTypeDecimal256>(
- const DataTypeDecimal256::ColumnType& decimal_col, IColumn& target_col) const;
+ const DataTypeDecimalV2::ColumnType& decimal_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
+template Status DataTypeDateSerDe<TYPE_DATETIME>::from_decimal_strict_mode_batch<
+ DataTypeDecimal128>(const DataTypeDecimal128::ColumnType& decimal_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
+template Status DataTypeDateSerDe<TYPE_DATETIME>::from_decimal_strict_mode_batch<
+ DataTypeDecimal256>(const DataTypeDecimal256::ColumnType& decimal_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
} // namespace doris
diff --git a/be/src/core/data_type_serde/data_type_date_or_datetime_serde.h b/be/src/core/data_type_serde/data_type_date_or_datetime_serde.h
index e8ce3f41069..016d40c90ee 100644
--- a/be/src/core/data_type_serde/data_type_date_or_datetime_serde.h
+++ b/be/src/core/data_type_serde/data_type_date_or_datetime_serde.h
@@ -69,21 +69,24 @@ public:
ColumnNullable& target_col) const;
template <typename IntDataType>
Status from_int_strict_mode_batch(const typename IntDataType::ColumnType& int_col,
- IColumn& target_col) const;
+ IColumn& target_col,
+ const NullMap::value_type* null_map = nullptr) const;
template <typename FloatDataType>
Status from_float_batch(const typename FloatDataType::ColumnType& float_col,
ColumnNullable& target_col) const;
template <typename FloatDataType>
Status from_float_strict_mode_batch(const typename FloatDataType::ColumnType& float_col,
- IColumn& target_col) const;
+ IColumn& target_col,
+ const NullMap::value_type* null_map = nullptr) const;
template <typename DecimalDataType>
Status from_decimal_batch(const typename DecimalDataType::ColumnType& decimal_col,
ColumnNullable& target_col) const;
template <typename DecimalDataType>
Status from_decimal_strict_mode_batch(const typename DecimalDataType::ColumnType& decimal_col,
- IColumn& target_col) const;
+ IColumn& target_col,
+ const NullMap::value_type* null_map = nullptr) const;
Status serialize_one_cell_to_json(
const IColumn& column, int64_t row_num, BufferWritable& bw,
diff --git a/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp b/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp
index 89e388f9039..9d543400c9c 100644
--- a/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp
+++ b/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp
@@ -430,12 +430,16 @@ Status DataTypeDateTimeV2SerDe::from_int_batch(const typename IntDataType::Colum
template <typename IntDataType>
Status DataTypeDateTimeV2SerDe::from_int_strict_mode_batch(
- const typename IntDataType::ColumnType& int_col, IColumn& target_col) const {
+ const typename IntDataType::ColumnType& int_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const {
auto& col_data = assert_cast<ColumnDateTimeV2&>(target_col);
col_data.resize(int_col.size());
CastParameters params {.status = Status::OK(), .is_strict = true};
for (size_t i = 0; i < int_col.size(); ++i) {
+ if (null_map && null_map[i]) {
+ continue;
+ }
DateV2Value<DateTimeV2ValueType> val;
CastToDatetimeV2::from_integer<DatelikeParseMode::STRICT>(int_col.get_element(i), val,
params);
@@ -475,12 +479,16 @@ Status DataTypeDateTimeV2SerDe::from_float_batch(
template <typename FloatDataType>
Status DataTypeDateTimeV2SerDe::from_float_strict_mode_batch(
- const typename FloatDataType::ColumnType& float_col, IColumn& target_col) const {
+ const typename FloatDataType::ColumnType& float_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const {
auto& col_data = assert_cast<ColumnDateTimeV2&>(target_col);
col_data.resize(float_col.size());
CastParameters params {.status = Status::OK(), .is_strict = true};
for (size_t i = 0; i < float_col.size(); ++i) {
+ if (null_map && null_map[i]) {
+ continue;
+ }
DateV2Value<DateTimeV2ValueType> val;
CastToDatetimeV2::from_float<DatelikeParseMode::STRICT>(float_col.get_data()[i], val,
_scale, params);
@@ -521,12 +529,16 @@ Status DataTypeDateTimeV2SerDe::from_decimal_batch(
template <typename DecimalDataType>
Status DataTypeDateTimeV2SerDe::from_decimal_strict_mode_batch(
- const typename DecimalDataType::ColumnType& decimal_col, IColumn& target_col) const {
+ const typename DecimalDataType::ColumnType& decimal_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const {
auto& col_data = assert_cast<ColumnDateTimeV2&>(target_col);
col_data.resize(decimal_col.size());
CastParameters params {.status = Status::OK(), .is_strict = true};
for (size_t i = 0; i < decimal_col.size(); ++i) {
+ if (null_map && null_map[i]) {
+ continue;
+ }
DateV2Value<DateTimeV2ValueType> val;
CastToDatetimeV2::from_decimal<DatelikeParseMode::STRICT>(
decimal_col.get_intergral_part(i), decimal_col.get_fractional_part(i),
@@ -954,23 +966,30 @@ template Status DataTypeDateTimeV2SerDe::from_int_batch<DataTypeInt64>(
template Status DataTypeDateTimeV2SerDe::from_int_batch<DataTypeInt128>(
const DataTypeInt128::ColumnType& int_col, ColumnNullable& target_col) const;
template Status DataTypeDateTimeV2SerDe::from_int_strict_mode_batch<DataTypeInt8>(
- const DataTypeInt8::ColumnType& int_col, IColumn& target_col) const;
+ const DataTypeInt8::ColumnType& int_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
template Status DataTypeDateTimeV2SerDe::from_int_strict_mode_batch<DataTypeInt16>(
- const DataTypeInt16::ColumnType& int_col, IColumn& target_col) const;
+ const DataTypeInt16::ColumnType& int_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
template Status DataTypeDateTimeV2SerDe::from_int_strict_mode_batch<DataTypeInt32>(
- const DataTypeInt32::ColumnType& int_col, IColumn& target_col) const;
+ const DataTypeInt32::ColumnType& int_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
template Status DataTypeDateTimeV2SerDe::from_int_strict_mode_batch<DataTypeInt64>(
- const DataTypeInt64::ColumnType& int_col, IColumn& target_col) const;
+ const DataTypeInt64::ColumnType& int_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
template Status DataTypeDateTimeV2SerDe::from_int_strict_mode_batch<DataTypeInt128>(
- const DataTypeInt128::ColumnType& int_col, IColumn& target_col) const;
+ const DataTypeInt128::ColumnType& int_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
template Status DataTypeDateTimeV2SerDe::from_float_batch<DataTypeFloat32>(
const DataTypeFloat32::ColumnType& float_col, ColumnNullable& target_col) const;
template Status DataTypeDateTimeV2SerDe::from_float_batch<DataTypeFloat64>(
const DataTypeFloat64::ColumnType& float_col, ColumnNullable& target_col) const;
template Status DataTypeDateTimeV2SerDe::from_float_strict_mode_batch<DataTypeFloat32>(
- const DataTypeFloat32::ColumnType& float_col, IColumn& target_col) const;
+ const DataTypeFloat32::ColumnType& float_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
template Status DataTypeDateTimeV2SerDe::from_float_strict_mode_batch<DataTypeFloat64>(
- const DataTypeFloat64::ColumnType& float_col, IColumn& target_col) const;
+ const DataTypeFloat64::ColumnType& float_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
template Status DataTypeDateTimeV2SerDe::from_decimal_batch<DataTypeDecimal32>(
const DataTypeDecimal32::ColumnType& decimal_col, ColumnNullable& target_col) const;
template Status DataTypeDateTimeV2SerDe::from_decimal_batch<DataTypeDecimal64>(
@@ -982,15 +1001,20 @@ template Status DataTypeDateTimeV2SerDe::from_decimal_batch<DataTypeDecimal128>(
template Status DataTypeDateTimeV2SerDe::from_decimal_batch<DataTypeDecimal256>(
const DataTypeDecimal256::ColumnType& decimal_col, ColumnNullable& target_col) const;
template Status DataTypeDateTimeV2SerDe::from_decimal_strict_mode_batch<DataTypeDecimal32>(
- const DataTypeDecimal32::ColumnType& decimal_col, IColumn& target_col) const;
+ const DataTypeDecimal32::ColumnType& decimal_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
template Status DataTypeDateTimeV2SerDe::from_decimal_strict_mode_batch<DataTypeDecimal64>(
- const DataTypeDecimal64::ColumnType& decimal_col, IColumn& target_col) const;
+ const DataTypeDecimal64::ColumnType& decimal_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
template Status DataTypeDateTimeV2SerDe::from_decimal_strict_mode_batch<DataTypeDecimalV2>(
- const DataTypeDecimalV2::ColumnType& decimal_col, IColumn& target_col) const;
+ const DataTypeDecimalV2::ColumnType& decimal_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
template Status DataTypeDateTimeV2SerDe::from_decimal_strict_mode_batch<DataTypeDecimal128>(
- const DataTypeDecimal128::ColumnType& decimal_col, IColumn& target_col) const;
+ const DataTypeDecimal128::ColumnType& decimal_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
template Status DataTypeDateTimeV2SerDe::from_decimal_strict_mode_batch<DataTypeDecimal256>(
- const DataTypeDecimal256::ColumnType& decimal_col, IColumn& target_col) const;
+ const DataTypeDecimal256::ColumnType& decimal_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
Status DataTypeDateTimeV2SerDe::read_column_from_orc(IColumn& column,
const OrcDecodedColumnView& view) const {
diff --git a/be/src/core/data_type_serde/data_type_datetimev2_serde.h b/be/src/core/data_type_serde/data_type_datetimev2_serde.h
index 02f190aabfc..af25dc2dbf1 100644
--- a/be/src/core/data_type_serde/data_type_datetimev2_serde.h
+++ b/be/src/core/data_type_serde/data_type_datetimev2_serde.h
@@ -53,22 +53,24 @@ public:
template <typename IntDataType>
Status from_int_batch(const IntDataType::ColumnType& int_col, ColumnNullable& target_col) const;
template <typename IntDataType>
- Status from_int_strict_mode_batch(const IntDataType::ColumnType& int_col,
- IColumn& target_col) const;
+ Status from_int_strict_mode_batch(const IntDataType::ColumnType& int_col, IColumn& target_col,
+ const NullMap::value_type* null_map = nullptr) const;
template <typename FloatDataType>
Status from_float_batch(const FloatDataType::ColumnType& float_col,
ColumnNullable& target_col) const;
template <typename FloatDataType>
Status from_float_strict_mode_batch(const FloatDataType::ColumnType& float_col,
- IColumn& target_col) const;
+ IColumn& target_col,
+ const NullMap::value_type* null_map = nullptr) const;
template <typename DecimalDataType>
Status from_decimal_batch(const DecimalDataType::ColumnType& decimal_col,
ColumnNullable& target_col) const;
template <typename DecimalDataType>
Status from_decimal_strict_mode_batch(const DecimalDataType::ColumnType& decimal_col,
- IColumn& target_col) const;
+ IColumn& target_col,
+ const NullMap::value_type* null_map = nullptr) const;
Status serialize_one_cell_to_json(const IColumn& column, int64_t row_num, BufferWritable& bw,
FormatOptions& options) const override;
diff --git a/be/src/core/data_type_serde/data_type_datev2_serde.cpp b/be/src/core/data_type_serde/data_type_datev2_serde.cpp
index 02059045469..b7021af6f01 100644
--- a/be/src/core/data_type_serde/data_type_datev2_serde.cpp
+++ b/be/src/core/data_type_serde/data_type_datev2_serde.cpp
@@ -600,12 +600,16 @@ Status DataTypeDateV2SerDe::from_int_batch(const typename IntDataType::ColumnTyp
template <typename IntDataType>
Status DataTypeDateV2SerDe::from_int_strict_mode_batch(
- const typename IntDataType::ColumnType& int_col, IColumn& target_col) const {
+ const typename IntDataType::ColumnType& int_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const {
auto& col_data = assert_cast<ColumnDateV2&>(target_col);
col_data.resize(int_col.size());
CastParameters params {.status = Status::OK(), .is_strict = true};
for (size_t i = 0; i < int_col.size(); ++i) {
+ if (null_map && null_map[i]) {
+ continue;
+ }
DateV2Value<DateV2ValueType> val;
CastToDateV2::from_integer<DatelikeParseMode::STRICT>(int_col.get_element(i), val, params);
if (!params.status.ok()) [[unlikely]] {
@@ -643,12 +647,16 @@ Status DataTypeDateV2SerDe::from_float_batch(const typename FloatDataType::Colum
template <typename FloatDataType>
Status DataTypeDateV2SerDe::from_float_strict_mode_batch(
- const typename FloatDataType::ColumnType& float_col, IColumn& target_col) const {
+ const typename FloatDataType::ColumnType& float_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const {
auto& col_data = assert_cast<ColumnDateV2&>(target_col);
col_data.resize(float_col.size());
CastParameters params {.status = Status::OK(), .is_strict = true};
for (size_t i = 0; i < float_col.size(); ++i) {
+ if (null_map && null_map[i]) {
+ continue;
+ }
DateV2Value<DateV2ValueType> val;
CastToDateV2::from_float<DatelikeParseMode::STRICT>(float_col.get_data()[i], val, params);
if (!params.status.ok()) [[unlikely]] {
@@ -688,12 +696,16 @@ Status DataTypeDateV2SerDe::from_decimal_batch(
template <typename DecimalDataType>
Status DataTypeDateV2SerDe::from_decimal_strict_mode_batch(
- const typename DecimalDataType::ColumnType& decimal_col, IColumn& target_col) const {
+ const typename DecimalDataType::ColumnType& decimal_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const {
auto& col_data = assert_cast<ColumnDateV2&>(target_col);
col_data.resize(decimal_col.size());
CastParameters params {.status = Status::OK(), .is_strict = true};
for (size_t i = 0; i < decimal_col.size(); ++i) {
+ if (null_map && null_map[i]) {
+ continue;
+ }
DateV2Value<DateV2ValueType> val;
CastToDateV2::from_decimal<DatelikeParseMode::STRICT>(decimal_col.get_intergral_part(i),
decimal_col.get_scale(), val, params);
@@ -732,23 +744,30 @@ template Status DataTypeDateV2SerDe::from_int_batch<DataTypeInt64>(
template Status DataTypeDateV2SerDe::from_int_batch<DataTypeInt128>(
const DataTypeInt128::ColumnType& int_col, ColumnNullable& target_col) const;
template Status DataTypeDateV2SerDe::from_int_strict_mode_batch<DataTypeInt8>(
- const DataTypeInt8::ColumnType& int_col, IColumn& target_col) const;
+ const DataTypeInt8::ColumnType& int_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
template Status DataTypeDateV2SerDe::from_int_strict_mode_batch<DataTypeInt16>(
- const DataTypeInt16::ColumnType& int_col, IColumn& target_col) const;
+ const DataTypeInt16::ColumnType& int_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
template Status DataTypeDateV2SerDe::from_int_strict_mode_batch<DataTypeInt32>(
- const DataTypeInt32::ColumnType& int_col, IColumn& target_col) const;
+ const DataTypeInt32::ColumnType& int_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
template Status DataTypeDateV2SerDe::from_int_strict_mode_batch<DataTypeInt64>(
- const DataTypeInt64::ColumnType& int_col, IColumn& target_col) const;
+ const DataTypeInt64::ColumnType& int_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
template Status DataTypeDateV2SerDe::from_int_strict_mode_batch<DataTypeInt128>(
- const DataTypeInt128::ColumnType& int_col, IColumn& target_col) const;
+ const DataTypeInt128::ColumnType& int_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
template Status DataTypeDateV2SerDe::from_float_batch<DataTypeFloat32>(
const DataTypeFloat32::ColumnType& float_col, ColumnNullable& target_col) const;
template Status DataTypeDateV2SerDe::from_float_batch<DataTypeFloat64>(
const DataTypeFloat64::ColumnType& float_col, ColumnNullable& target_col) const;
template Status DataTypeDateV2SerDe::from_float_strict_mode_batch<DataTypeFloat32>(
- const DataTypeFloat32::ColumnType& float_col, IColumn& target_col) const;
+ const DataTypeFloat32::ColumnType& float_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
template Status DataTypeDateV2SerDe::from_float_strict_mode_batch<DataTypeFloat64>(
- const DataTypeFloat64::ColumnType& float_col, IColumn& target_col) const;
+ const DataTypeFloat64::ColumnType& float_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
template Status DataTypeDateV2SerDe::from_decimal_batch<DataTypeDecimal32>(
const DataTypeDecimal32::ColumnType& decimal_col, ColumnNullable& target_col) const;
template Status DataTypeDateV2SerDe::from_decimal_batch<DataTypeDecimal64>(
@@ -760,15 +779,20 @@ template Status DataTypeDateV2SerDe::from_decimal_batch<DataTypeDecimal128>(
template Status DataTypeDateV2SerDe::from_decimal_batch<DataTypeDecimal256>(
const DataTypeDecimal256::ColumnType& decimal_col, ColumnNullable& target_col) const;
template Status DataTypeDateV2SerDe::from_decimal_strict_mode_batch<DataTypeDecimal32>(
- const DataTypeDecimal32::ColumnType& decimal_col, IColumn& target_col) const;
+ const DataTypeDecimal32::ColumnType& decimal_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
template Status DataTypeDateV2SerDe::from_decimal_strict_mode_batch<DataTypeDecimal64>(
- const DataTypeDecimal64::ColumnType& decimal_col, IColumn& target_col) const;
+ const DataTypeDecimal64::ColumnType& decimal_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
template Status DataTypeDateV2SerDe::from_decimal_strict_mode_batch<DataTypeDecimalV2>(
- const DataTypeDecimalV2::ColumnType& decimal_col, IColumn& target_col) const;
+ const DataTypeDecimalV2::ColumnType& decimal_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
template Status DataTypeDateV2SerDe::from_decimal_strict_mode_batch<DataTypeDecimal128>(
- const DataTypeDecimal128::ColumnType& decimal_col, IColumn& target_col) const;
+ const DataTypeDecimal128::ColumnType& decimal_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
template Status DataTypeDateV2SerDe::from_decimal_strict_mode_batch<DataTypeDecimal256>(
- const DataTypeDecimal256::ColumnType& decimal_col, IColumn& target_col) const;
+ const DataTypeDecimal256::ColumnType& decimal_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
Status DataTypeDateV2SerDe::read_column_from_orc(IColumn& column,
const OrcDecodedColumnView& view) const {
diff --git a/be/src/core/data_type_serde/data_type_datev2_serde.h b/be/src/core/data_type_serde/data_type_datev2_serde.h
index 18230a81351..7223edd25fc 100644
--- a/be/src/core/data_type_serde/data_type_datev2_serde.h
+++ b/be/src/core/data_type_serde/data_type_datev2_serde.h
@@ -52,22 +52,24 @@ public:
template <typename IntDataType>
Status from_int_batch(const IntDataType::ColumnType& int_col, ColumnNullable& target_col) const;
template <typename IntDataType>
- Status from_int_strict_mode_batch(const IntDataType::ColumnType& int_col,
- IColumn& target_col) const;
+ Status from_int_strict_mode_batch(const IntDataType::ColumnType& int_col, IColumn& target_col,
+ const NullMap::value_type* null_map = nullptr) const;
template <typename FloatDataType>
Status from_float_batch(const FloatDataType::ColumnType& float_col,
ColumnNullable& target_col) const;
template <typename FloatDataType>
Status from_float_strict_mode_batch(const FloatDataType::ColumnType& float_col,
- IColumn& target_col) const;
+ IColumn& target_col,
+ const NullMap::value_type* null_map = nullptr) const;
template <typename DecimalDataType>
Status from_decimal_batch(const DecimalDataType::ColumnType& decimal_col,
ColumnNullable& target_col) const;
template <typename DecimalDataType>
Status from_decimal_strict_mode_batch(const DecimalDataType::ColumnType& decimal_col,
- IColumn& target_col) const;
+ IColumn& target_col,
+ const NullMap::value_type* null_map = nullptr) const;
Status serialize_one_cell_to_json(const IColumn& column, int64_t row_num, BufferWritable& bw,
FormatOptions& options) const override;
diff --git a/be/src/core/data_type_serde/data_type_decimal_serde.cpp b/be/src/core/data_type_serde/data_type_decimal_serde.cpp
index 6634008bfdc..9fc8e674d27 100644
--- a/be/src/core/data_type_serde/data_type_decimal_serde.cpp
+++ b/be/src/core/data_type_serde/data_type_decimal_serde.cpp
@@ -1270,8 +1270,9 @@ Status DataTypeDecimalSerDe<T>::serialize_column_to_jsonb(const IColumn& from_co
}
template <PrimitiveType T>
-Status DataTypeDecimalSerDe<T>::serialize_column_to_jsonb_vector(const IColumn& from_column,
- ColumnString& to_column) const {
+Status DataTypeDecimalSerDe<T>::serialize_column_to_jsonb_vector(
+ const IColumn& from_column, ColumnString& to_column,
+ const NullMap::value_type* source_null_map) const {
if constexpr (T == TYPE_DECIMALV2) {
return Status::NotSupported("DECIMALV2 does not support serialize_column_to_jsonb_vector");
} else {
@@ -1279,6 +1280,10 @@ Status DataTypeDecimalSerDe<T>::serialize_column_to_jsonb_vector(const IColumn&
JsonbWriter writer;
const auto& data = assert_cast<const ColumnDecimal<T>&>(from_column).get_data();
for (int i = 0; i < size; i++) {
+ if (source_null_map && source_null_map[i]) {
+ to_column.insert_default();
+ continue;
+ }
writer.reset();
if (!writer.writeDecimal(data[i], precision, scale)) {
return Status::InvalidArgument(
@@ -1309,8 +1314,8 @@ Status DataTypeDecimalSerDe<T>::deserialize_column_from_jsonb(IColumn& column,
template <PrimitiveType T>
Status DataTypeDecimalSerDe<T>::deserialize_column_from_jsonb_vector(
- ColumnNullable& column_to, const ColumnString& col_from_json,
- CastParameters& castParms) const {
+ ColumnNullable& column_to, const ColumnString& col_from_json, CastParameters& castParms,
+ const NullMap::value_type* source_null_map) const {
if constexpr (T == TYPE_DECIMALV2) {
return Status::NotSupported(
"DECIMALV2 does not support deserialize_column_from_jsonb_vector");
@@ -1325,6 +1330,11 @@ Status DataTypeDecimalSerDe<T>::deserialize_column_from_jsonb_vector(
data.resize(size);
for (size_t i = 0; i < size; ++i) {
+ if (source_null_map && source_null_map[i]) {
+ null_map[i] = true;
+ data[i] = {};
+ continue;
+ }
const auto& val = col_from_json.get_data_at(i);
auto* jsonb_value = handle_jsonb_value(val);
if (!jsonb_value) {
diff --git a/be/src/core/data_type_serde/data_type_decimal_serde.h b/be/src/core/data_type_serde/data_type_decimal_serde.h
index 424c837a7ca..04c55a06bf0 100644
--- a/be/src/core/data_type_serde/data_type_decimal_serde.h
+++ b/be/src/core/data_type_serde/data_type_decimal_serde.h
@@ -92,15 +92,16 @@ public:
Status serialize_column_to_jsonb(const IColumn& from_column, int64_t row_num,
JsonbWriter& writer) const override;
- Status serialize_column_to_jsonb_vector(const IColumn& from_column,
- ColumnString& to_column) const override;
+ Status serialize_column_to_jsonb_vector(
+ const IColumn& from_column, ColumnString& to_column,
+ const NullMap::value_type* source_null_map = nullptr) const override;
Status deserialize_column_from_jsonb(IColumn& column, const JsonbValue* jsonb_value,
CastParameters& castParms) const override;
- Status deserialize_column_from_jsonb_vector(ColumnNullable& column_to,
- const ColumnString& from_column,
- CastParameters& castParms) const override;
+ Status deserialize_column_from_jsonb_vector(
+ ColumnNullable& column_to, const ColumnString& from_column, CastParameters& castParms,
+ const NullMap::value_type* source_null_map = nullptr) const override;
Status write_column_to_arrow(const IColumn& column, const NullMap* null_map,
arrow::ArrayBuilder* array_builder, int64_t start, int64_t end,
diff --git a/be/src/core/data_type_serde/data_type_number_serde.cpp b/be/src/core/data_type_serde/data_type_number_serde.cpp
index 077e4fde5b0..5bc68bac04f 100644
--- a/be/src/core/data_type_serde/data_type_number_serde.cpp
+++ b/be/src/core/data_type_serde/data_type_number_serde.cpp
@@ -1201,8 +1201,9 @@ Status DataTypeNumberSerDe<T>::serialize_column_to_jsonb(const IColumn& from_col
}
template <PrimitiveType T>
-Status DataTypeNumberSerDe<T>::serialize_column_to_jsonb_vector(const IColumn& from_column,
- ColumnString& to_column) const {
+Status DataTypeNumberSerDe<T>::serialize_column_to_jsonb_vector(
+ const IColumn& from_column, ColumnString& to_column,
+ const NullMap::value_type* source_null_map) const {
if constexpr (!can_write_to_jsonb_from_number<T>()) {
return Status::NotSupported("{} does not support serialize_column_to_jsonb", get_name());
}
@@ -1211,6 +1212,10 @@ Status DataTypeNumberSerDe<T>::serialize_column_to_jsonb_vector(const IColumn& f
const auto& data = assert_cast<const ColumnType&>(from_column).get_data();
const auto scale = get_scale();
for (int i = 0; i < size; i++) {
+ if (source_null_map && source_null_map[i]) {
+ to_column.insert_default();
+ continue;
+ }
writer.reset();
if (!write_to_jsonb_from_number<T>(data[i], writer, scale)) {
return Status::InvalidArgument(
@@ -1255,8 +1260,8 @@ Status DataTypeNumberSerDe<T>::deserialize_column_from_jsonb(IColumn& column,
template <PrimitiveType T>
Status DataTypeNumberSerDe<T>::deserialize_column_from_jsonb_vector(
- ColumnNullable& column_to, const ColumnString& col_from_json,
- CastParameters& castParms) const {
+ ColumnNullable& column_to, const ColumnString& col_from_json, CastParameters& castParms,
+ const NullMap::value_type* source_null_map) const {
if constexpr (!can_write_to_jsonb_from_number<T>()) {
return Status::NotSupported("{} does not support serialize_column_to_jsonb", get_name());
} else {
@@ -1270,6 +1275,11 @@ Status DataTypeNumberSerDe<T>::deserialize_column_from_jsonb_vector(
data.resize(size);
for (size_t i = 0; i < size; ++i) {
+ if (source_null_map && source_null_map[i]) {
+ null_map[i] = true;
+ data[i] = {};
+ continue;
+ }
const auto& val = col_from_json.get_data_at(i);
auto* jsonb_value = handle_jsonb_value(val);
if (!jsonb_value) {
diff --git a/be/src/core/data_type_serde/data_type_number_serde.h b/be/src/core/data_type_serde/data_type_number_serde.h
index 55cdbca0f78..78d47c499a1 100644
--- a/be/src/core/data_type_serde/data_type_number_serde.h
+++ b/be/src/core/data_type_serde/data_type_number_serde.h
@@ -90,15 +90,16 @@ public:
Status serialize_column_to_jsonb(const IColumn& from_column, int64_t row_num,
JsonbWriter& writer) const override;
- Status serialize_column_to_jsonb_vector(const IColumn& from_column,
- ColumnString& to_column) const override;
+ Status serialize_column_to_jsonb_vector(
+ const IColumn& from_column, ColumnString& to_column,
+ const NullMap::value_type* source_null_map = nullptr) const override;
Status deserialize_column_from_jsonb(IColumn& column, const JsonbValue* jsonb_value,
CastParameters& castParms) const override;
- Status deserialize_column_from_jsonb_vector(ColumnNullable& column_to,
- const ColumnString& from_column,
- CastParameters& castParms) const override;
+ Status deserialize_column_from_jsonb_vector(
+ ColumnNullable& column_to, const ColumnString& from_column, CastParameters& castParms,
+ const NullMap::value_type* source_null_map = nullptr) const override;
void insert_column_last_value_multiple_times(IColumn& column, uint64_t times) const override;
diff --git a/be/src/core/data_type_serde/data_type_serde.cpp b/be/src/core/data_type_serde/data_type_serde.cpp
index 4cdbde805d6..d3844c441de 100644
--- a/be/src/core/data_type_serde/data_type_serde.cpp
+++ b/be/src/core/data_type_serde/data_type_serde.cpp
@@ -144,11 +144,16 @@ Status DataTypeSerDe::default_from_string(StringRef& str, IColumn& column) const
return deserialize_one_cell_from_json(column, slice, options);
}
-Status DataTypeSerDe::serialize_column_to_jsonb_vector(const IColumn& from_column,
- ColumnString& to_column) const {
+Status DataTypeSerDe::serialize_column_to_jsonb_vector(
+ const IColumn& from_column, ColumnString& to_column,
+ const NullMap::value_type* source_null_map) const {
const auto size = from_column.size();
JsonbWriter writer;
for (int i = 0; i < size; i++) {
+ if (source_null_map && source_null_map[i]) {
+ to_column.insert_default();
+ continue;
+ }
writer.reset();
RETURN_IF_ERROR(serialize_column_to_jsonb(from_column, i, writer));
to_column.insert_data(writer.getOutput()->getBuffer(), writer.getOutput()->getSize());
@@ -170,12 +175,16 @@ Status DataTypeSerDe::parse_column_from_jsonb_string(IColumn& column, const Json
return deserialize_one_cell_from_json(column, slice, format_options);
}
-Status DataTypeSerDe::deserialize_column_from_jsonb_vector(ColumnNullable& column_to,
- const ColumnString& col_from_json,
- CastParameters& castParms) const {
+Status DataTypeSerDe::deserialize_column_from_jsonb_vector(
+ ColumnNullable& column_to, const ColumnString& col_from_json, CastParameters& castParms,
+ const NullMap::value_type* source_null_map) const {
const size_t size = col_from_json.size();
const bool is_strict = castParms.is_strict;
for (size_t i = 0; i < size; ++i) {
+ if (source_null_map && source_null_map[i]) {
+ column_to.insert_default();
+ continue;
+ }
const auto& val = col_from_json.get_data_at(i);
const auto* value = handle_jsonb_value(val);
if (!value) {
diff --git a/be/src/core/data_type_serde/data_type_serde.h b/be/src/core/data_type_serde/data_type_serde.h
index 3464ae71c5e..b305f8df79d 100644
--- a/be/src/core/data_type_serde/data_type_serde.h
+++ b/be/src/core/data_type_serde/data_type_serde.h
@@ -445,8 +445,10 @@ public:
return Status::NotSupported("{} does not support serialize_column_to_jsonb", get_name());
}
- virtual Status serialize_column_to_jsonb_vector(const IColumn& from_column,
- ColumnString& to_column) const;
+ // Skip rows in source_null_map and insert a default value for each skipped row.
+ virtual Status serialize_column_to_jsonb_vector(
+ const IColumn& from_column, ColumnString& to_column,
+ const NullMap::value_type* source_null_map = nullptr) const;
virtual Status deserialize_column_from_jsonb(IColumn& column, const JsonbValue* jsonb_value,
CastParameters& castParms) const {
@@ -459,9 +461,10 @@ public:
// else return jsonb_value
static const JsonbValue* handle_jsonb_value(const StringRef& val);
- virtual Status deserialize_column_from_jsonb_vector(ColumnNullable& column_to,
- const ColumnString& from_column,
- CastParameters& castParms) const;
+ // Skip rows in source_null_map and insert NULL for each skipped row.
+ virtual Status deserialize_column_from_jsonb_vector(
+ ColumnNullable& column_to, const ColumnString& from_column, CastParameters& castParms,
+ const NullMap::value_type* source_null_map = nullptr) const;
Status parse_column_from_jsonb_string(IColumn& column, const JsonbValue* jsonb_value,
CastParameters& castParms) const;
diff --git a/be/src/core/data_type_serde/data_type_string_serde.cpp b/be/src/core/data_type_serde/data_type_string_serde.cpp
index 675ba6bcf8e..20286768424 100644
--- a/be/src/core/data_type_serde/data_type_string_serde.cpp
+++ b/be/src/core/data_type_serde/data_type_string_serde.cpp
@@ -813,8 +813,8 @@ Status DataTypeStringSerDeBase<ColumnType>::deserialize_column_from_jsonb(
template <typename ColumnType>
Status DataTypeStringSerDeBase<ColumnType>::deserialize_column_from_jsonb_vector(
- ColumnNullable& column_to, const ColumnString& col_from_json,
- CastParameters& castParms) const {
+ ColumnNullable& column_to, const ColumnString& col_from_json, CastParameters& castParms,
+ const NullMap::value_type* source_null_map) const {
if constexpr (!std::is_same_v<ColumnType, ColumnString>) {
return Status::NotSupported(
"DataTypeStringSerDeBase only supports ColumnString for "
@@ -831,6 +831,11 @@ Status DataTypeStringSerDeBase<ColumnType>::deserialize_column_from_jsonb_vector
col_str.get_chars().reserve(col_from_json.get_chars().size());
for (size_t i = 0; i < size; ++i) {
+ if (source_null_map && source_null_map[i]) {
+ null_map[i] = true;
+ col_str.insert_default();
+ continue;
+ }
const auto& val = col_from_json.get_data_at(i);
auto* jsonb_value = handle_jsonb_value(val);
if (!jsonb_value) {
diff --git a/be/src/core/data_type_serde/data_type_string_serde.h b/be/src/core/data_type_serde/data_type_string_serde.h
index 9343846ad1a..e7c5a9d9c67 100644
--- a/be/src/core/data_type_serde/data_type_string_serde.h
+++ b/be/src/core/data_type_serde/data_type_string_serde.h
@@ -183,9 +183,9 @@ public:
Status deserialize_column_from_jsonb(IColumn& column, const JsonbValue* jsonb_value,
CastParameters& castParms) const override;
- Status deserialize_column_from_jsonb_vector(ColumnNullable& column_to,
- const ColumnString& from_column,
- CastParameters& castParms) const override;
+ Status deserialize_column_from_jsonb_vector(
+ ColumnNullable& column_to, const ColumnString& from_column, CastParameters& castParms,
+ const NullMap::value_type* source_null_map = nullptr) const override;
void write_one_cell_to_jsonb(const IColumn& column, JsonbWriter& result, Arena& mem_pool,
int32_t col_id, int64_t row_num,
diff --git a/be/src/core/data_type_serde/data_type_time_serde.cpp b/be/src/core/data_type_serde/data_type_time_serde.cpp
index 566bd3982b4..2d4f629414c 100644
--- a/be/src/core/data_type_serde/data_type_time_serde.cpp
+++ b/be/src/core/data_type_serde/data_type_time_serde.cpp
@@ -476,12 +476,16 @@ Status DataTypeTimeV2SerDe::from_int_batch(const typename IntDataType::ColumnTyp
template <typename IntDataType>
Status DataTypeTimeV2SerDe::from_int_strict_mode_batch(
- const typename IntDataType::ColumnType& int_col, IColumn& target_col) const {
+ const typename IntDataType::ColumnType& int_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const {
auto& col_data = assert_cast<ColumnTimeV2&>(target_col);
col_data.resize(int_col.size());
CastParameters params {.status = Status::OK(), .is_strict = true};
for (size_t i = 0; i < int_col.size(); ++i) {
+ if (null_map && null_map[i]) {
+ continue;
+ }
TimeValue::TimeType val = 0;
CastToTimeV2::from_integer<DatelikeParseMode::STRICT>(int_col.get_element(i), val, params);
if (!params.status.ok()) [[unlikely]] {
@@ -519,12 +523,16 @@ Status DataTypeTimeV2SerDe::from_float_batch(const typename FloatDataType::Colum
template <typename FloatDataType>
Status DataTypeTimeV2SerDe::from_float_strict_mode_batch(
- const typename FloatDataType::ColumnType& float_col, IColumn& target_col) const {
+ const typename FloatDataType::ColumnType& float_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const {
auto& col_data = assert_cast<ColumnTimeV2&>(target_col);
col_data.resize(float_col.size());
CastParameters params {.status = Status::OK(), .is_strict = true};
for (size_t i = 0; i < float_col.size(); ++i) {
+ if (null_map && null_map[i]) {
+ continue;
+ }
TimeValue::TimeType val = 0;
CastToTimeV2::from_float<DatelikeParseMode::STRICT>(float_col.get_data()[i], val, _scale,
params);
@@ -565,12 +573,16 @@ Status DataTypeTimeV2SerDe::from_decimal_batch(
template <typename DecimalDataType>
Status DataTypeTimeV2SerDe::from_decimal_strict_mode_batch(
- const typename DecimalDataType::ColumnType& decimal_col, IColumn& target_col) const {
+ const typename DecimalDataType::ColumnType& decimal_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const {
auto& col_data = assert_cast<ColumnTimeV2&>(target_col);
col_data.resize(decimal_col.size());
CastParameters params {.status = Status::OK(), .is_strict = true};
for (size_t i = 0; i < decimal_col.size(); ++i) {
+ if (null_map && null_map[i]) {
+ continue;
+ }
TimeValue::TimeType val = 0;
CastToTimeV2::from_decimal<DatelikeParseMode::STRICT>(
decimal_col.get_intergral_part(i), decimal_col.get_fractional_part(i),
@@ -601,23 +613,30 @@ template Status DataTypeTimeV2SerDe::from_int_batch<DataTypeInt64>(
template Status DataTypeTimeV2SerDe::from_int_batch<DataTypeInt128>(
const DataTypeInt128::ColumnType& int_col, ColumnNullable& target_col) const;
template Status DataTypeTimeV2SerDe::from_int_strict_mode_batch<DataTypeInt8>(
- const DataTypeInt8::ColumnType& int_col, IColumn& target_col) const;
+ const DataTypeInt8::ColumnType& int_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
template Status DataTypeTimeV2SerDe::from_int_strict_mode_batch<DataTypeInt16>(
- const DataTypeInt16::ColumnType& int_col, IColumn& target_col) const;
+ const DataTypeInt16::ColumnType& int_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
template Status DataTypeTimeV2SerDe::from_int_strict_mode_batch<DataTypeInt32>(
- const DataTypeInt32::ColumnType& int_col, IColumn& target_col) const;
+ const DataTypeInt32::ColumnType& int_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
template Status DataTypeTimeV2SerDe::from_int_strict_mode_batch<DataTypeInt64>(
- const DataTypeInt64::ColumnType& int_col, IColumn& target_col) const;
+ const DataTypeInt64::ColumnType& int_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
template Status DataTypeTimeV2SerDe::from_int_strict_mode_batch<DataTypeInt128>(
- const DataTypeInt128::ColumnType& int_col, IColumn& target_col) const;
+ const DataTypeInt128::ColumnType& int_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
template Status DataTypeTimeV2SerDe::from_float_batch<DataTypeFloat32>(
const DataTypeFloat32::ColumnType& float_col, ColumnNullable& target_col) const;
template Status DataTypeTimeV2SerDe::from_float_batch<DataTypeFloat64>(
const DataTypeFloat64::ColumnType& float_col, ColumnNullable& target_col) const;
template Status DataTypeTimeV2SerDe::from_float_strict_mode_batch<DataTypeFloat32>(
- const DataTypeFloat32::ColumnType& float_col, IColumn& target_col) const;
+ const DataTypeFloat32::ColumnType& float_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
template Status DataTypeTimeV2SerDe::from_float_strict_mode_batch<DataTypeFloat64>(
- const DataTypeFloat64::ColumnType& float_col, IColumn& target_col) const;
+ const DataTypeFloat64::ColumnType& float_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
template Status DataTypeTimeV2SerDe::from_decimal_batch<DataTypeDecimal32>(
const DataTypeDecimal32::ColumnType& decimal_col, ColumnNullable& target_col) const;
template Status DataTypeTimeV2SerDe::from_decimal_batch<DataTypeDecimal64>(
@@ -629,14 +648,19 @@ template Status DataTypeTimeV2SerDe::from_decimal_batch<DataTypeDecimal128>(
template Status DataTypeTimeV2SerDe::from_decimal_batch<DataTypeDecimal256>(
const DataTypeDecimal256::ColumnType& decimal_col, ColumnNullable& target_col) const;
template Status DataTypeTimeV2SerDe::from_decimal_strict_mode_batch<DataTypeDecimal32>(
- const DataTypeDecimal32::ColumnType& decimal_col, IColumn& target_col) const;
+ const DataTypeDecimal32::ColumnType& decimal_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
template Status DataTypeTimeV2SerDe::from_decimal_strict_mode_batch<DataTypeDecimal64>(
- const DataTypeDecimal64::ColumnType& decimal_col, IColumn& target_col) const;
+ const DataTypeDecimal64::ColumnType& decimal_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
template Status DataTypeTimeV2SerDe::from_decimal_strict_mode_batch<DataTypeDecimalV2>(
- const DataTypeDecimalV2::ColumnType& decimal_col, IColumn& target_col) const;
+ const DataTypeDecimalV2::ColumnType& decimal_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
template Status DataTypeTimeV2SerDe::from_decimal_strict_mode_batch<DataTypeDecimal128>(
- const DataTypeDecimal128::ColumnType& decimal_col, IColumn& target_col) const;
+ const DataTypeDecimal128::ColumnType& decimal_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
template Status DataTypeTimeV2SerDe::from_decimal_strict_mode_batch<DataTypeDecimal256>(
- const DataTypeDecimal256::ColumnType& decimal_col, IColumn& target_col) const;
+ const DataTypeDecimal256::ColumnType& decimal_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
} // namespace doris
diff --git a/be/src/core/data_type_serde/data_type_time_serde.h b/be/src/core/data_type_serde/data_type_time_serde.h
index 39c6bb69d84..3249ba506b0 100644
--- a/be/src/core/data_type_serde/data_type_time_serde.h
+++ b/be/src/core/data_type_serde/data_type_time_serde.h
@@ -55,21 +55,24 @@ public:
ColumnNullable& target_col) const;
template <typename IntDataType>
Status from_int_strict_mode_batch(const typename IntDataType::ColumnType& int_col,
- IColumn& target_col) const;
+ IColumn& target_col,
+ const NullMap::value_type* null_map = nullptr) const;
template <typename FloatDataType>
Status from_float_batch(const typename FloatDataType::ColumnType& float_col,
ColumnNullable& target_col) const;
template <typename FloatDataType>
Status from_float_strict_mode_batch(const typename FloatDataType::ColumnType& float_col,
- IColumn& target_col) const;
+ IColumn& target_col,
+ const NullMap::value_type* null_map = nullptr) const;
template <typename DecimalDataType>
Status from_decimal_batch(const typename DecimalDataType::ColumnType& decimal_col,
ColumnNullable& target_col) const;
template <typename DecimalDataType>
Status from_decimal_strict_mode_batch(const typename DecimalDataType::ColumnType& decimal_col,
- IColumn& target_col) const;
+ IColumn& target_col,
+ const NullMap::value_type* null_map = nullptr) const;
Status read_column_from_decoded_values(IColumn& column,
const DecodedColumnView& view) const override;
Status read_column_from_parquet(IColumn& column, ParquetDecodeSource& source,
diff --git a/be/src/core/data_type_serde/data_type_timestamp_ns_serde.cpp b/be/src/core/data_type_serde/data_type_timestamp_ns_serde.cpp
index a7b47247e14..abe2db517f3 100644
--- a/be/src/core/data_type_serde/data_type_timestamp_ns_serde.cpp
+++ b/be/src/core/data_type_serde/data_type_timestamp_ns_serde.cpp
@@ -169,12 +169,16 @@ Status DataTypeTimeStampNsSerDe::from_int_batch(const typename IntDataType::Colu
template <typename IntDataType>
Status DataTypeTimeStampNsSerDe::from_int_strict_mode_batch(
- const typename IntDataType::ColumnType& int_col, IColumn& target_col) const {
+ const typename IntDataType::ColumnType& int_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const {
auto& col_data = assert_cast<ColumnTimeStampNs&>(target_col);
col_data.resize(int_col.size());
CastParameters params {.status = Status::OK(), .is_strict = true};
for (size_t i = 0; i < int_col.size(); ++i) {
+ if (null_map && null_map[i]) {
+ continue;
+ }
TimeStampNsValue value;
CastToTimestampNs::from_integer<DatelikeParseMode::STRICT>(int_col.get_element(i), value,
params);
@@ -213,12 +217,16 @@ Status DataTypeTimeStampNsSerDe::from_float_batch(
template <typename FloatDataType>
Status DataTypeTimeStampNsSerDe::from_float_strict_mode_batch(
- const typename FloatDataType::ColumnType& float_col, IColumn& target_col) const {
+ const typename FloatDataType::ColumnType& float_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const {
auto& col_data = assert_cast<ColumnTimeStampNs&>(target_col);
col_data.resize(float_col.size());
CastParameters params {.status = Status::OK(), .is_strict = true};
for (size_t i = 0; i < float_col.size(); ++i) {
+ if (null_map && null_map[i]) {
+ continue;
+ }
TimeStampNsValue value;
CastToTimestampNs::from_float<DatelikeParseMode::STRICT>(float_col.get_data()[i], value,
params);
@@ -258,12 +266,16 @@ Status DataTypeTimeStampNsSerDe::from_decimal_batch(
template <typename DecimalDataType>
Status DataTypeTimeStampNsSerDe::from_decimal_strict_mode_batch(
- const typename DecimalDataType::ColumnType& decimal_col, IColumn& target_col) const {
+ const typename DecimalDataType::ColumnType& decimal_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const {
auto& col_data = assert_cast<ColumnTimeStampNs&>(target_col);
col_data.resize(decimal_col.size());
CastParameters params {.status = Status::OK(), .is_strict = true};
for (size_t i = 0; i < decimal_col.size(); ++i) {
+ if (null_map && null_map[i]) {
+ continue;
+ }
TimeStampNsValue value;
CastToTimestampNs::from_decimal<DatelikeParseMode::STRICT>(
decimal_col.get_intergral_part(i), decimal_col.get_fractional_part(i),
@@ -327,9 +339,10 @@ Status DataTypeTimeStampNsSerDe::deserialize_column_from_jsonb(IColumn& column,
}
Status DataTypeTimeStampNsSerDe::deserialize_column_from_jsonb_vector(
- ColumnNullable& column_to, const ColumnString& column_from,
- CastParameters& cast_params) const {
- return DataTypeSerDe::deserialize_column_from_jsonb_vector(column_to, column_from, cast_params);
+ ColumnNullable& column_to, const ColumnString& column_from, CastParameters& cast_params,
+ const NullMap::value_type* source_null_map) const {
+ return DataTypeSerDe::deserialize_column_from_jsonb_vector(column_to, column_from, cast_params,
+ source_null_map);
}
Status DataTypeTimeStampNsSerDe::write_column_to_arrow(const IColumn& column,
@@ -464,23 +477,30 @@ template Status DataTypeTimeStampNsSerDe::from_int_batch<DataTypeInt64>(
template Status DataTypeTimeStampNsSerDe::from_int_batch<DataTypeInt128>(
const DataTypeInt128::ColumnType& int_col, ColumnNullable& target_col) const;
template Status DataTypeTimeStampNsSerDe::from_int_strict_mode_batch<DataTypeInt8>(
- const DataTypeInt8::ColumnType& int_col, IColumn& target_col) const;
+ const DataTypeInt8::ColumnType& int_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
template Status DataTypeTimeStampNsSerDe::from_int_strict_mode_batch<DataTypeInt16>(
- const DataTypeInt16::ColumnType& int_col, IColumn& target_col) const;
+ const DataTypeInt16::ColumnType& int_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
template Status DataTypeTimeStampNsSerDe::from_int_strict_mode_batch<DataTypeInt32>(
- const DataTypeInt32::ColumnType& int_col, IColumn& target_col) const;
+ const DataTypeInt32::ColumnType& int_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
template Status DataTypeTimeStampNsSerDe::from_int_strict_mode_batch<DataTypeInt64>(
- const DataTypeInt64::ColumnType& int_col, IColumn& target_col) const;
+ const DataTypeInt64::ColumnType& int_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
template Status DataTypeTimeStampNsSerDe::from_int_strict_mode_batch<DataTypeInt128>(
- const DataTypeInt128::ColumnType& int_col, IColumn& target_col) const;
+ const DataTypeInt128::ColumnType& int_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
template Status DataTypeTimeStampNsSerDe::from_float_batch<DataTypeFloat32>(
const DataTypeFloat32::ColumnType& float_col, ColumnNullable& target_col) const;
template Status DataTypeTimeStampNsSerDe::from_float_batch<DataTypeFloat64>(
const DataTypeFloat64::ColumnType& float_col, ColumnNullable& target_col) const;
template Status DataTypeTimeStampNsSerDe::from_float_strict_mode_batch<DataTypeFloat32>(
- const DataTypeFloat32::ColumnType& float_col, IColumn& target_col) const;
+ const DataTypeFloat32::ColumnType& float_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
template Status DataTypeTimeStampNsSerDe::from_float_strict_mode_batch<DataTypeFloat64>(
- const DataTypeFloat64::ColumnType& float_col, IColumn& target_col) const;
+ const DataTypeFloat64::ColumnType& float_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
template Status DataTypeTimeStampNsSerDe::from_decimal_batch<DataTypeDecimal32>(
const DataTypeDecimal32::ColumnType& decimal_col, ColumnNullable& target_col) const;
template Status DataTypeTimeStampNsSerDe::from_decimal_batch<DataTypeDecimal64>(
@@ -492,14 +512,19 @@ template Status DataTypeTimeStampNsSerDe::from_decimal_batch<DataTypeDecimal128>
template Status DataTypeTimeStampNsSerDe::from_decimal_batch<DataTypeDecimal256>(
const DataTypeDecimal256::ColumnType& decimal_col, ColumnNullable& target_col) const;
template Status DataTypeTimeStampNsSerDe::from_decimal_strict_mode_batch<DataTypeDecimal32>(
- const DataTypeDecimal32::ColumnType& decimal_col, IColumn& target_col) const;
+ const DataTypeDecimal32::ColumnType& decimal_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
template Status DataTypeTimeStampNsSerDe::from_decimal_strict_mode_batch<DataTypeDecimal64>(
- const DataTypeDecimal64::ColumnType& decimal_col, IColumn& target_col) const;
+ const DataTypeDecimal64::ColumnType& decimal_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
template Status DataTypeTimeStampNsSerDe::from_decimal_strict_mode_batch<DataTypeDecimalV2>(
- const DataTypeDecimalV2::ColumnType& decimal_col, IColumn& target_col) const;
+ const DataTypeDecimalV2::ColumnType& decimal_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
template Status DataTypeTimeStampNsSerDe::from_decimal_strict_mode_batch<DataTypeDecimal128>(
- const DataTypeDecimal128::ColumnType& decimal_col, IColumn& target_col) const;
+ const DataTypeDecimal128::ColumnType& decimal_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
template Status DataTypeTimeStampNsSerDe::from_decimal_strict_mode_batch<DataTypeDecimal256>(
- const DataTypeDecimal256::ColumnType& decimal_col, IColumn& target_col) const;
+ const DataTypeDecimal256::ColumnType& decimal_col, IColumn& target_col,
+ const NullMap::value_type* null_map) const;
} // namespace doris
diff --git a/be/src/core/data_type_serde/data_type_timestamp_ns_serde.h b/be/src/core/data_type_serde/data_type_timestamp_ns_serde.h
index 97eb42cb2e8..8351aa96961 100644
--- a/be/src/core/data_type_serde/data_type_timestamp_ns_serde.h
+++ b/be/src/core/data_type_serde/data_type_timestamp_ns_serde.h
@@ -44,22 +44,24 @@ public:
template <typename IntDataType>
Status from_int_batch(const IntDataType::ColumnType& int_col, ColumnNullable& target_col) const;
template <typename IntDataType>
- Status from_int_strict_mode_batch(const IntDataType::ColumnType& int_col,
- IColumn& target_col) const;
+ Status from_int_strict_mode_batch(const IntDataType::ColumnType& int_col, IColumn& target_col,
+ const NullMap::value_type* null_map = nullptr) const;
template <typename FloatDataType>
Status from_float_batch(const FloatDataType::ColumnType& float_col,
ColumnNullable& target_col) const;
template <typename FloatDataType>
Status from_float_strict_mode_batch(const FloatDataType::ColumnType& float_col,
- IColumn& target_col) const;
+ IColumn& target_col,
+ const NullMap::value_type* null_map = nullptr) const;
template <typename DecimalDataType>
Status from_decimal_batch(const DecimalDataType::ColumnType& decimal_col,
ColumnNullable& target_col) const;
template <typename DecimalDataType>
Status from_decimal_strict_mode_batch(const DecimalDataType::ColumnType& decimal_col,
- IColumn& target_col) const;
+ IColumn& target_col,
+ const NullMap::value_type* null_map = nullptr) const;
Status serialize_one_cell_to_json(const IColumn& column, int64_t row_num, BufferWritable& bw,
FormatOptions& options) const override;
@@ -72,9 +74,9 @@ public:
const FormatOptions& options) const override;
Status deserialize_column_from_jsonb(IColumn& column, const JsonbValue* jsonb_value,
CastParameters& cast_params) const override;
- Status deserialize_column_from_jsonb_vector(ColumnNullable& column_to,
- const ColumnString& column_from,
- CastParameters& cast_params) const override;
+ Status deserialize_column_from_jsonb_vector(
+ ColumnNullable& column_to, const ColumnString& column_from, CastParameters& cast_params,
+ const NullMap::value_type* source_null_map = nullptr) const override;
Status write_column_to_arrow(const IColumn& column, const NullMap* null_map,
arrow::ArrayBuilder* array_builder, int64_t start, int64_t end,
diff --git a/be/src/exprs/function/cast/cast_base.cpp b/be/src/exprs/function/cast/cast_base.cpp
index ecaf5875853..00f29e7a9b2 100644
--- a/be/src/exprs/function/cast/cast_base.cpp
+++ b/be/src/exprs/function/cast/cast_base.cpp
@@ -17,11 +17,62 @@
#include "exprs/function/cast/cast_base.h"
+#include <algorithm>
#include <cstdint>
+#include <utility>
#include "util/jsonb_writer.h"
namespace doris::CastWrapper {
+bool has_masked_row(const NullMap::value_type* null_map, size_t rows) {
+ return std::any_of(null_map, null_map + rows,
+ [](const NullMap::value_type value) { return value != 0; });
+}
+
+ChildNullMask build_child_null_mask(const NullMap::value_type* parent_null_map,
+ const IColumn::Offsets64* offsets, const ColumnPtr& child) {
+ if (parent_null_map == nullptr) {
+ return {.column = child, .null_map = nullptr, .mask_holder = nullptr};
+ }
+ const size_t rows = offsets == nullptr ? child->size() : offsets->size();
+ if (!has_masked_row(parent_null_map, rows)) {
+ return {.column = child, .null_map = nullptr, .mask_holder = nullptr};
+ }
+
+ auto mask = ColumnUInt8::create(child->size(), 0);
+ auto& mask_data = mask->get_data();
+ if (offsets == nullptr) {
+ // The children of a STRUCT share the rows of their parent.
+ std::copy_n(parent_null_map, rows, mask_data.begin());
+ } else {
+ // ARRAY and MAP children are flattened, so the NULL of a row has to be expanded to every
+ // element/entry that belongs to it.
+ for (size_t row = 0; row < rows; ++row) {
+ if (parent_null_map[row] == 0) {
+ continue;
+ }
+ const size_t first_child = row == 0 ? 0 : (*offsets)[row - 1];
+ std::fill(mask_data.begin() + first_child, mask_data.begin() + (*offsets)[row], 1);
+ }
+ }
+
+ const auto* nullable_child = check_and_get_column<ColumnNullable>(child.get());
+ if (nullable_child == nullptr) {
+ return {.column = child, .null_map = mask_data.data(), .mask_holder = std::move(mask)};
+ }
+
+ // The NULL state of a child is the union of its own NULL map and the mask inherited from the
+ // rows of its parent. Merge it into `mask` instead of building a second mask of the same size.
+ const auto& child_null_map = nullable_child->get_null_map_data();
+ for (size_t i = 0; i < mask_data.size(); ++i) {
+ mask_data[i] |= child_null_map[i];
+ }
+ return {.column = ColumnNullable::create(nullable_child->get_nested_column_ptr(),
+ std::move(mask)),
+ .null_map = nullptr,
+ .mask_holder = nullptr};
+}
+
Status cast_from_generic_to_jsonb(FunctionContext* context, Block& block,
const ColumnNumbers& arguments, uint32_t result,
size_t input_rows_count, const NullMap::value_type* null_map) {
@@ -239,4 +290,4 @@ Status cast_from_string_to_complex_type_strict_mode(FunctionContext* context, Bl
return Status::OK();
}
-} // namespace doris::CastWrapper
\ No newline at end of file
+} // namespace doris::CastWrapper
diff --git a/be/src/exprs/function/cast/cast_base.h b/be/src/exprs/function/cast/cast_base.h
index cc58e29d4ac..ce828c77943 100644
--- a/be/src/exprs/function/cast/cast_base.h
+++ b/be/src/exprs/function/cast/cast_base.h
@@ -153,6 +153,32 @@ WrapperType prepare_impl(FunctionContext* context, const DataTypePtr& from_type,
ElementWrappers get_element_wrappers(FunctionContext* context, const DataTypes& from_element_types,
const DataTypes& to_element_types);
+/// A child of an ARRAY/MAP/STRUCT column inherits the NULL state of the row it belongs to: when a
+/// row is NULL, its children are hidden payloads without SQL semantics (IF/NULLIF keep an arbitrary
+/// value there) and a strict cast must not validate them.
+struct ChildNullMask {
+ /// Column to hand to the child cast. A nullable child column carries the merged NULL mask
+ /// itself, because a child cast reads the NULL state of a nullable source from its column.
+ ColumnPtr column;
+ /// Mask to hand to the child cast when the child column cannot carry it (not nullable child),
+ /// so the child cast skips the masked children itself.
+ const NullMap::value_type* null_map = nullptr;
+ /// Owns `null_map`, so it has to outlive the child cast.
+ ColumnPtr mask_holder;
+};
+
+/// Merge the NULL mask of the rows of a complex column into the mask of one of its children.
+/// `offsets` expands the row aligned `parent_null_map` to a flattened child (ARRAY, MAP), pass
+/// nullptr for a child that shares the rows of its parent (STRUCT).
+ChildNullMask build_child_null_mask(const NullMap::value_type* parent_null_map,
+ const IColumn::Offsets64* offsets, const ColumnPtr& child);
+
+/// Whether the mask of `rows` rows marks at least one of them as NULL. A nullable column keeps an
+/// all zero mask when it has no NULL row, and there is nothing to inherit from such a column. Callers
+/// that build masks for several children of the same parent can check this once and pass nullptr to
+/// build_child_null_mask when it is false.
+bool has_masked_row(const NullMap::value_type* null_map, size_t rows);
+
WrapperType create_identity_wrapper(const DataTypePtr&);
} // namespace CastWrapper
diff --git a/be/src/exprs/function/cast/cast_to_array.h b/be/src/exprs/function/cast/cast_to_array.h
index d4eb6886530..b8f92bb78af 100644
--- a/be/src/exprs/function/cast/cast_to_array.h
+++ b/be/src/exprs/function/cast/cast_to_array.h
@@ -71,9 +71,12 @@ WrapperType create_array_wrapper(FunctionContext* context, const DataTypePtr& fr
const auto* from_col_array = check_and_get_column<ColumnArray>(from_column.get());
if (from_col_array) {
+ /// The elements of a row that is NULL are hidden payloads, so the NULL of the row has to
+ /// be inherited by the elements that belong to it before they are cast.
+ auto child_mask = build_child_null_mask(null_map, &from_col_array->get_offsets(),
+ from_col_array->get_data_ptr());
/// create columns for converting nested column containing original and result columns
- ColumnWithTypeAndName from_nested_column {from_col_array->get_data_ptr(),
- from_nested_type, ""};
+ ColumnWithTypeAndName from_nested_column {child_mask.column, from_nested_type, ""};
/// convert nested column
ColumnNumbers new_arguments {block.columns()};
@@ -82,7 +85,7 @@ WrapperType create_array_wrapper(FunctionContext* context, const DataTypePtr& fr
uint32_t nested_result = block.columns();
block.insert({to_nested_type, ""});
RETURN_IF_ERROR(nested_function(context, block, new_arguments, nested_result,
- from_col_array->get_data_ptr()->size(), null_map));
+ child_mask.column->size(), child_mask.null_map));
auto nested_result_column = block.get_by_position(nested_result).column;
/// set converted nested column to result
diff --git a/be/src/exprs/function/cast/cast_to_date.h b/be/src/exprs/function/cast/cast_to_date.h
index dad009409a6..6391bd8444a 100644
--- a/be/src/exprs/function/cast/cast_to_date.h
+++ b/be/src/exprs/function/cast/cast_to_date.h
@@ -99,15 +99,15 @@ public:
// WON'T write nulls to the result column, just raise errors. null_map is only used to skip invalid rows
if constexpr (IsDataTypeInt<FromDataType>) {
RETURN_IF_ERROR(concrete_serde->template from_int_strict_mode_batch<FromDataType>(
- *col_from, *column_to));
+ *col_from, *column_to, null_map));
} else if constexpr (IsDataTypeFloat<FromDataType>) {
RETURN_IF_ERROR(concrete_serde->template from_float_strict_mode_batch<FromDataType>(
- *col_from, *column_to));
+ *col_from, *column_to, null_map));
} else {
static_assert(IsDataTypeDecimal<FromDataType>);
RETURN_IF_ERROR(
concrete_serde->template from_decimal_strict_mode_batch<FromDataType>(
- *col_from, *column_to));
+ *col_from, *column_to, null_map));
}
block.get_by_position(result).column = std::move(column_to);
} else {
diff --git a/be/src/exprs/function/cast/cast_to_decimal.h b/be/src/exprs/function/cast/cast_to_decimal.h
index 1ee8cbee5ab..692d555bab1 100644
--- a/be/src/exprs/function/cast/cast_to_decimal.h
+++ b/be/src/exprs/function/cast/cast_to_decimal.h
@@ -797,6 +797,10 @@ public:
RETURN_IF_ERROR(std::visit(
[&](auto multiply_may_overflow, auto narrow_integral) {
for (size_t i = 0; i < size; i++) {
+ // Skip hidden payload of rows marked as null by the input null map.
+ if (null_map && null_map[i]) {
+ continue;
+ }
if (!CastToDecimal::_from_int<typename FromDataType::FieldType,
typename ToDataType::FieldType,
multiply_may_overflow, narrow_integral>(
@@ -878,6 +882,10 @@ public:
DataTypeDecimal<ToFieldType::PType>::get_max_digits_number(to_precision);
typename ToFieldType::NativeType min_result = -max_result;
for (size_t i = 0; i < size; i++) {
+ // Skip hidden payload of rows marked as null by the input null map.
+ if (null_map && null_map[i]) {
+ continue;
+ }
if (!CastToDecimal::_from_float<typename FromDataType::FieldType,
typename ToDataType::FieldType>(
vec_from_data[i], vec_to_data[i], to_precision, to_scale, scale_multiplier,
@@ -996,6 +1004,10 @@ public:
RETURN_IF_ERROR(std::visit(
[&](auto multiply_may_overflow, auto narrow_integral) {
for (size_t i = 0; i < size; i++) {
+ // Skip hidden payload of rows marked as null by the input null map.
+ if (null_map && null_map[i]) {
+ continue;
+ }
if (!CastToDecimal::_from_decimal<FromFieldType, ToFieldType,
multiply_may_overflow, narrow_integral>(
vec_from_data[i], from_precision, from_scale, vec_to_data[i],
@@ -1103,6 +1115,10 @@ public:
RETURN_IF_ERROR(std::visit(
[&](auto multiply_may_overflow, auto narrow_integral) {
for (size_t i = 0; i < size; i++) {
+ // Skip hidden payload of rows marked as null by the input null map.
+ if (null_map && null_map[i]) {
+ continue;
+ }
if (!CastToDecimal::_from_decimal<FromFieldType, ToFieldType,
multiply_may_overflow, narrow_integral>(
vec_from_data[i], from_precision, from_scale, vec_to_data[i],
diff --git a/be/src/exprs/function/cast/cast_to_int.h b/be/src/exprs/function/cast/cast_to_int.h
index bf7672d3c11..cb1710d1a3c 100644
--- a/be/src/exprs/function/cast/cast_to_int.h
+++ b/be/src/exprs/function/cast/cast_to_int.h
@@ -72,6 +72,11 @@ public:
CastParameters params;
params.is_strict = (CastMode == CastModeType::StrictMode);
for (size_t i = 0; i < input_rows_count; ++i) {
+ // The source value of a row marked as null by the input null map is a hidden
+ // payload and has no SQL semantics, so it must not be checked.
+ if (null_map && null_map[i]) {
+ continue;
+ }
if constexpr (IsDataTypeInt<FromDataType>) {
// although always nullable output, but only set null when overflow in non-strict mode.
// in strict mode, just raise error on overflow.
@@ -178,6 +183,10 @@ public:
params.is_strict = (CastMode == CastModeType::StrictMode);
size_t size = vec_from.size();
for (size_t i = 0; i < size; i++) {
+ // Skip hidden payload of rows marked as null by the input null map.
+ if (null_map && null_map[i]) {
+ continue;
+ }
if (!CastToInt::from_decimal<typename FromDataType::FieldType,
typename ToDataType::FieldType>(
vec_from_data[i], from_precision, from_scale, vec_to_data[i], params)) {
diff --git a/be/src/exprs/function/cast/cast_to_jsonb.h b/be/src/exprs/function/cast/cast_to_jsonb.h
index 6c2cf50420a..60393ef6225 100644
--- a/be/src/exprs/function/cast/cast_to_jsonb.h
+++ b/be/src/exprs/function/cast/cast_to_jsonb.h
@@ -54,6 +54,11 @@ struct ConvertImplGenericFromJsonb {
const bool is_complex = is_complex_type(data_type_to->get_primitive_type());
const bool is_dst_string = is_string_type(data_type_to->get_primitive_type());
for (size_t i = 0; i < size; ++i) {
+ if (null_map && null_map[i]) {
+ (*vec_null_map_to)[i] = 1;
+ col_to->insert_default();
+ continue;
+ }
const auto& val = col_from_string->get_data_at(i);
const JsonbDocument* doc = nullptr;
auto st = JsonbDocument::checkAndCreateDocument(val.data, val.size, &doc);
@@ -132,7 +137,7 @@ WrapperType create_cast_from_jsonb_wrapper(const DataTypeJsonb& from_type,
}
return [](FunctionContext* context, Block& block, const ColumnNumbers& arguments,
- uint32_t result, size_t input_rows_count, const NullMap::value_type*) {
+ uint32_t result, size_t input_rows_count, const NullMap::value_type* null_map) {
CastParameters params;
params.is_strict = context->enable_strict_mode();
@@ -145,8 +150,8 @@ WrapperType create_cast_from_jsonb_wrapper(const DataTypeJsonb& from_type,
auto column_to = make_nullable(data_type_to)->create_column();
auto& column_to_nullable = assert_cast<ColumnNullable&>(*column_to);
- RETURN_IF_ERROR(serde_to->deserialize_column_from_jsonb_vector(column_to_nullable,
- col_from_json, params));
+ RETURN_IF_ERROR(serde_to->deserialize_column_from_jsonb_vector(
+ column_to_nullable, col_from_json, params, null_map));
block.get_by_position(result).column = std::move(column_to);
return Status::OK();
@@ -230,13 +235,13 @@ WrapperType create_cast_to_jsonb_wrapper(const DataTypePtr& from_type, const Dat
}
return [](FunctionContext* context, Block& block, const ColumnNumbers& arguments,
- uint32_t result, size_t input_rows_count, const NullMap::value_type*) {
+ uint32_t result, size_t input_rows_count, const NullMap::value_type* null_map) {
// same as to_json function
auto to_column = ColumnString::create();
auto from_type_serde = block.get_by_position(arguments[0]).type->get_serde();
auto from_column = block.get_by_position(arguments[0]).column;
- RETURN_IF_ERROR(
- from_type_serde->serialize_column_to_jsonb_vector(*from_column, *to_column));
+ RETURN_IF_ERROR(from_type_serde->serialize_column_to_jsonb_vector(*from_column, *to_column,
+ null_map));
block.get_by_position(result).column = std::move(to_column);
return Status::OK();
};
diff --git a/be/src/exprs/function/cast/cast_to_map.h b/be/src/exprs/function/cast/cast_to_map.h
index 44a7576a6cc..4714d22805d 100644
--- a/be/src/exprs/function/cast/cast_to_map.h
+++ b/be/src/exprs/function/cast/cast_to_map.h
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+#include <array>
+
#include "core/column/column_map.h"
#include "core/column/column_nullable.h"
#include "core/data_type/data_type_map.h"
@@ -86,7 +88,11 @@ WrapperType create_map_wrapper(FunctionContext* context, const DataTypePtr& from
to_kv_types.push_back(to_type.get_value_type());
auto kv_wrappers = get_element_wrappers(context, from_kv_types, to_kv_types);
- return [kv_wrappers, from_kv_types, to_kv_types](
+ /// A key or value whose type does not change is passed through, so it needs neither a child mask
+ /// nor a call into its (identity) wrapper.
+ std::array<bool, 2> unchanged_kv = {from_kv_types[0]->equals(*to_kv_types[0]),
+ from_kv_types[1]->equals(*to_kv_types[1])};
+ return [kv_wrappers, from_kv_types, to_kv_types, unchanged_kv](
FunctionContext* context, Block& block, const ColumnNumbers& arguments,
uint32_t result, size_t /*input_rows_count*/,
const NullMap::value_type* null_map = nullptr) -> Status {
@@ -102,13 +108,27 @@ WrapperType create_map_wrapper(FunctionContext* context, const DataTypePtr& from
columnsWithTypeAndName[0] = {from_col_map->get_keys_ptr(), from_kv_types[0], ""};
columnsWithTypeAndName[1] = {from_col_map->get_values_ptr(), from_kv_types[1], ""};
+ const auto& offsets = from_col_map->get_offsets();
+ // Keys and values share the rows of their parent, so the parent mask is scanned once.
+ const NullMap::value_type* inherited_null_map =
+ (null_map != nullptr && has_masked_row(null_map, offsets.size())) ? null_map
+ : nullptr;
for (size_t i = 0; i < 2; ++i) {
+ if (unchanged_kv[i]) {
+ converted_columns[i] = columnsWithTypeAndName[i].column;
+ continue;
+ }
+ /// Keys and values are flattened, so the NULL of a map row has to be inherited by the
+ /// entries that belong to it before they are cast.
+ auto child_mask = build_child_null_mask(inherited_null_map, &offsets,
+ columnsWithTypeAndName[i].column);
+ ColumnWithTypeAndName child_column {child_mask.column, from_kv_types[i], ""};
ColumnNumbers element_arguments {block.columns()};
- block.insert(columnsWithTypeAndName[i]);
+ block.insert(child_column);
auto element_result = block.columns();
block.insert({to_kv_types[i], ""});
RETURN_IF_ERROR(kv_wrappers[i](context, block, element_arguments, element_result,
- columnsWithTypeAndName[i].column->size(), null_map));
+ child_mask.column->size(), child_mask.null_map));
converted_columns[i] = block.get_by_position(element_result).column;
}
@@ -119,4 +139,4 @@ WrapperType create_map_wrapper(FunctionContext* context, const DataTypePtr& from
return Status::OK();
};
}
-} // namespace doris::CastWrapper
\ No newline at end of file
+} // namespace doris::CastWrapper
diff --git a/be/src/exprs/function/cast/cast_to_struct.h b/be/src/exprs/function/cast/cast_to_struct.h
index 83d6b741d9d..c2223f3d60e 100644
--- a/be/src/exprs/function/cast/cast_to_struct.h
+++ b/be/src/exprs/function/cast/cast_to_struct.h
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+#include <vector>
+
#include "core/column/column_struct.h"
#include "core/data_type/data_type_struct.h"
#include "core/types.h"
@@ -54,7 +56,15 @@ WrapperType create_struct_wrapper(FunctionContext* context, const DataTypePtr& f
}
auto element_wrappers = get_element_wrappers(context, from_element_types, to_element_types);
- return [element_wrappers, from_element_types, to_element_types](
+ /// A field whose type does not change is passed through, so it needs neither a child mask nor a
+ /// call into its (identity) wrapper.
+ std::vector<bool> unchanged_fields;
+ unchanged_fields.reserve(from_element_types.size());
+ for (size_t i = 0; i < from_element_types.size(); ++i) {
+ unchanged_fields.push_back(from_element_types[i]->equals(*to_element_types[i]));
+ }
+ return [element_wrappers, from_element_types, to_element_types,
+ unchanged_fields = std::move(unchanged_fields)](
FunctionContext* context, Block& block, const ColumnNumbers& arguments,
uint32_t result, size_t /*input_rows_count*/,
const NullMap::value_type* null_map = nullptr) -> Status {
@@ -67,9 +77,24 @@ WrapperType create_struct_wrapper(FunctionContext* context, const DataTypePtr& f
size_t elements_num = to_element_types.size();
Columns converted_columns(elements_num);
+ // Fields share the rows of their parent, so the parent mask is scanned once for all of them.
+ const NullMap::value_type* inherited_null_map =
+ (null_map != nullptr && has_masked_row(null_map, from_col_struct->size()))
+ ? null_map
+ : nullptr;
for (size_t i = 0; i < elements_num; ++i) {
- ColumnWithTypeAndName from_element_column {from_col_struct->get_column_ptr(i),
- from_element_types[i], ""};
+ if (unchanged_fields[i]) {
+ /// The field is not converted, so it cannot validate the hidden payload of a NULL
+ /// row, and the NULL of the row hides the field anyway.
+ converted_columns[i] = from_col_struct->get_column_ptr(i);
+ continue;
+ }
+ /// A field of a row that is NULL is a hidden payload as well. Fields share the rows of
+ /// their parent, so the mask of the parent can be used as is.
+ auto child_mask = build_child_null_mask(inherited_null_map, nullptr,
+ from_col_struct->get_column_ptr(i));
+ ColumnWithTypeAndName from_element_column {child_mask.column, from_element_types[i],
+ ""};
ColumnNumbers element_arguments {block.columns()};
block.insert(from_element_column);
@@ -77,7 +102,7 @@ WrapperType create_struct_wrapper(FunctionContext* context, const DataTypePtr& f
block.insert({to_element_types[i], ""});
RETURN_IF_ERROR(element_wrappers[i](context, block, element_arguments, element_result,
- from_col_struct->get_column(i).size(), null_map));
+ child_mask.column->size(), child_mask.null_map));
converted_columns[i] = block.get_by_position(element_result).column;
}
@@ -85,4 +110,4 @@ WrapperType create_struct_wrapper(FunctionContext* context, const DataTypePtr& f
return Status::OK();
};
}
-} // namespace doris::CastWrapper
\ No newline at end of file
+} // namespace doris::CastWrapper
diff --git a/be/src/exprs/function/cast/cast_to_timestamp_ns.h b/be/src/exprs/function/cast/cast_to_timestamp_ns.h
index 7e664553967..a3bd3f22507 100644
--- a/be/src/exprs/function/cast/cast_to_timestamp_ns.h
+++ b/be/src/exprs/function/cast/cast_to_timestamp_ns.h
@@ -247,7 +247,9 @@ public:
if constexpr (CastMode == CastModeType::StrictMode) {
MutableColumnPtr column_to = nested_to_type->create_column();
- RETURN_IF_ERROR(from_number_strict(*concrete_serde, *col_from, *column_to));
+ // WON'T write nulls to the result column, just raise errors. null_map is only used to
+ // skip invalid rows
+ RETURN_IF_ERROR(from_number_strict(*concrete_serde, *col_from, *column_to, null_map));
block.get_by_position(result).column = std::move(column_to);
} else {
auto nullable_col_to = create_empty_nullable_column(nested_to_type);
@@ -260,16 +262,17 @@ public:
private:
static Status from_number_strict(const DataTypeTimeStampNsSerDe& serde,
const typename FromDataType::ColumnType& column_from,
- IColumn& column_to) {
+ IColumn& column_to, const NullMap::value_type* null_map) {
if constexpr (IsDataTypeDecimal<FromDataType>) {
return serde.template from_decimal_strict_mode_batch<FromDataType>(column_from,
- column_to);
+ column_to, null_map);
} else if constexpr (IsDataTypeInt<FromDataType>) {
- return serde.template from_int_strict_mode_batch<FromDataType>(column_from, column_to);
+ return serde.template from_int_strict_mode_batch<FromDataType>(column_from, column_to,
+ null_map);
} else {
static_assert(IsDataTypeFloat<FromDataType>);
- return serde.template from_float_strict_mode_batch<FromDataType>(column_from,
- column_to);
+ return serde.template from_float_strict_mode_batch<FromDataType>(column_from, column_to,
+ null_map);
}
}
diff --git a/be/src/exprs/function/cast/function_cast.cpp b/be/src/exprs/function/cast/function_cast.cpp
index 84d01dc3b3d..7defb971c79 100644
--- a/be/src/exprs/function/cast/function_cast.cpp
+++ b/be/src/exprs/function/cast/function_cast.cpp
@@ -211,8 +211,11 @@ WrapperType prepare_remove_nullable(FunctionContext* context, const DataTypePtr&
block.insert(block.get_by_position(arguments[0]));
}
+ /// get_nullable_column_info() already scanned the NULL map of the source, so the mask is
+ /// handed to the cast only when it really contains a NULL. An all zero mask is the common
+ /// case, and passing it would add a load and a branch to every row of the cast kernels.
const NullMap::value_type* arg_null_map = nullptr;
- if (source_info.is_nullable) {
+ if (source_info.has_null) {
arg_null_map = block.get_by_position(arguments[0])
.get_nullable_null_map_column()
->get_data()
diff --git a/be/test/core/data_type_serde/data_type_serde_datelike_batch_test.cpp b/be/test/core/data_type_serde/data_type_serde_datelike_batch_test.cpp
index d8ec668af18..316e4b496d6 100644
--- a/be/test/core/data_type_serde/data_type_serde_datelike_batch_test.cpp
+++ b/be/test/core/data_type_serde/data_type_serde_datelike_batch_test.cpp
@@ -527,4 +527,102 @@ TEST_F(DatelikeSerDeBatchTest, datetimev1_from_decimal_batch_mixed) {
EXPECT_TRUE(target->is_null_at(3));
}
+// ============================================================================
+// A row that the caller supplied null map marks as NULL may still carry an
+// arbitrary hidden payload in the source column (for example the branch of an
+// IF() that was not taken). The strict mode batches must skip such rows instead
+// of validating their payload, while rows that are not marked as NULL must still
+// report errors.
+// ============================================================================
+TEST_F(DatelikeSerDeBatchTest, datev1_from_int_strict_mode_batch_skips_masked_rows) {
+ DataTypeDateSerDe<TYPE_DATE> serde;
+ // Row 0 is masked as NULL and keeps the invalid hidden payload -1.
+ auto from_col = build_int64_column({-1, 20230115});
+ NullMap null_map {1, 0};
+
+ auto target = ColumnDate::create();
+ auto st = serde.from_int_strict_mode_batch<DataTypeInt64>(*from_col, *target, null_map.data());
+ ASSERT_TRUE(st.ok()) << st.to_string();
+
+ auto expected = ColumnDate::create();
+ auto visible_col = build_int64_column({20230115});
+ ASSERT_TRUE(serde.from_int_strict_mode_batch<DataTypeInt64>(*visible_col, *expected).ok());
+ EXPECT_TRUE(target->get_data()[1] == expected->get_data()[0]);
+
+ // The hidden payload is visible to the cast when no null map is passed.
+ EXPECT_FALSE(serde.from_int_strict_mode_batch<DataTypeInt64>(*from_col, *target).ok());
+}
+
+TEST_F(DatelikeSerDeBatchTest, datev2_from_int_strict_mode_batch_skips_masked_rows) {
+ DataTypeDateV2SerDe serde;
+ auto from_col = build_int64_column({-1, 20230115});
+ NullMap null_map {1, 0};
+
+ auto target = ColumnDateV2::create();
+ auto st = serde.from_int_strict_mode_batch<DataTypeInt64>(*from_col, *target, null_map.data());
+ ASSERT_TRUE(st.ok()) << st.to_string();
+
+ auto expected = ColumnDateV2::create();
+ auto visible_col = build_int64_column({20230115});
+ ASSERT_TRUE(serde.from_int_strict_mode_batch<DataTypeInt64>(*visible_col, *expected).ok());
+ EXPECT_EQ(target->get_data()[1], expected->get_data()[0]);
+
+ EXPECT_FALSE(serde.from_int_strict_mode_batch<DataTypeInt64>(*from_col, *target).ok());
+}
+
+TEST_F(DatelikeSerDeBatchTest, datetimev2_from_decimal_strict_mode_batch_skips_masked_rows) {
+ DataTypeDateTimeV2SerDe serde(/*scale=*/0);
+ auto from_col = build_decimal64_column({-1, 20230115143059}, /*scale=*/0);
+ NullMap null_map {1, 0};
+
+ auto target = ColumnDateTimeV2::create();
+ auto st = serde.from_decimal_strict_mode_batch<DataTypeDecimal64>(*from_col, *target,
+ null_map.data());
+ ASSERT_TRUE(st.ok()) << st.to_string();
+
+ auto expected = ColumnDateTimeV2::create();
+ auto visible_col = build_decimal64_column({20230115143059}, /*scale=*/0);
+ ASSERT_TRUE(
+ serde.from_decimal_strict_mode_batch<DataTypeDecimal64>(*visible_col, *expected).ok());
+ EXPECT_EQ(target->get_data()[1], expected->get_data()[0]);
+
+ EXPECT_FALSE(serde.from_decimal_strict_mode_batch<DataTypeDecimal64>(*from_col, *target).ok());
+}
+
+TEST_F(DatelikeSerDeBatchTest, timestamp_ns_from_int_strict_mode_batch_skips_masked_rows) {
+ DataTypeTimeStampNsSerDe serde;
+ // Row 0 is masked as NULL and keeps 16770921001243, which is outside the supported range.
+ auto from_col = build_int64_column({16770921001243, 20230115143059});
+ NullMap null_map {1, 0};
+
+ auto target = ColumnTimeStampNs::create();
+ auto st = serde.from_int_strict_mode_batch<DataTypeInt64>(*from_col, *target, null_map.data());
+ ASSERT_TRUE(st.ok()) << st.to_string();
+
+ auto expected = ColumnTimeStampNs::create();
+ auto visible_col = build_int64_column({20230115143059});
+ ASSERT_TRUE(serde.from_int_strict_mode_batch<DataTypeInt64>(*visible_col, *expected).ok());
+ EXPECT_EQ(target->get_data()[1], expected->get_data()[0]);
+
+ EXPECT_FALSE(serde.from_int_strict_mode_batch<DataTypeInt64>(*from_col, *target).ok());
+}
+
+TEST_F(DatelikeSerDeBatchTest, timev2_from_float_strict_mode_batch_skips_masked_rows) {
+ DataTypeTimeV2SerDe serde(/*scale=*/0);
+ auto from_col = build_float64_column({99999999.0, 123.0});
+ NullMap null_map {1, 0};
+
+ auto target = ColumnTimeV2::create();
+ auto st = serde.from_float_strict_mode_batch<DataTypeFloat64>(*from_col, *target,
+ null_map.data());
+ ASSERT_TRUE(st.ok()) << st.to_string();
+
+ auto expected = ColumnTimeV2::create();
+ auto visible_col = build_float64_column({123.0});
+ ASSERT_TRUE(serde.from_float_strict_mode_batch<DataTypeFloat64>(*visible_col, *expected).ok());
+ EXPECT_EQ(target->get_data()[1], expected->get_data()[0]);
+
+ EXPECT_FALSE(serde.from_float_strict_mode_batch<DataTypeFloat64>(*from_col, *target).ok());
+}
+
} // namespace doris
diff --git a/be/test/exprs/function/cast/cast_jsonb_null_mask_test.cpp b/be/test/exprs/function/cast/cast_jsonb_null_mask_test.cpp
new file mode 100644
index 00000000000..3ad2500ed5c
--- /dev/null
+++ b/be/test/exprs/function/cast/cast_jsonb_null_mask_test.cpp
@@ -0,0 +1,212 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "core/column/column_array.h"
+#include "core/column/column_struct.h"
+#include "core/data_type/data_type_array.h"
+#include "core/data_type/data_type_decimal.h"
+#include "core/data_type/data_type_jsonb.h"
+#include "core/data_type/data_type_struct.h"
+#include "core/value/jsonb_value.h"
+#include "exprs/function/cast/cast_test.h"
+#include "runtime/runtime_state.h"
+
+namespace doris {
+
+static ColumnPtr make_nullable_jsonb_column(const std::vector<std::string>& values,
+ const std::vector<UInt8>& null_map) {
+ auto nested = ColumnString::create();
+ JsonBinaryValue jsonb;
+ for (const auto& value : values) {
+ EXPECT_TRUE(jsonb.from_json_string(value).ok());
+ nested->insert_data(jsonb.value(), jsonb.size());
+ }
+ return ColumnNullable::create(std::move(nested),
+ ColumnHelper::create_column<DataTypeUInt8>(null_map));
+}
+
+TEST_F(FunctionCastTest, jsonb_source_skips_null_rows) {
+ const auto from_type = make_nullable(std::make_shared<DataTypeJsonb>());
+ const DataTypes targets = {
+ std::make_shared<DataTypeInt8>(), std::make_shared<DataTypeDecimal32>(9, 0),
+ std::make_shared<DataTypeArray>(make_nullable(std::make_shared<DataTypeInt8>()))};
+ for (const auto& target : targets) {
+ const auto to_type = make_nullable(target);
+ const auto valid = target->get_primitive_type() == TYPE_ARRAY ? "[1]" : "1";
+ for (bool strict : {false, true}) {
+ SCOPED_TRACE(target->get_name() + (strict ? " strict" : " non-strict"));
+ auto ctx = create_context(strict);
+ auto fn = get_cast_wrapper(ctx.get(), from_type, to_type);
+ // The invalid JSONB strings are hidden. The valid rows must keep their positions.
+ Block block {{make_nullable_jsonb_column({R"("bad")", valid, R"("bad")", valid},
+ {1, 0, 1, 0}),
+ from_type, "from"},
+ {nullptr, to_type, "to"}};
+ ASSERT_TRUE(fn(ctx.get(), block, {0}, 1, block.rows(), nullptr).ok());
+ const auto& result =
+ assert_cast<const ColumnNullable&>(*block.get_by_position(1).column);
+ ASSERT_EQ(result.size(), 4);
+ for (size_t i = 0; i < 4; ++i) {
+ EXPECT_EQ(result.is_null_at(i), i % 2 == 0);
+ }
+ EXPECT_EQ(target->to_string(result.get_nested_column(), 1), valid);
+ EXPECT_EQ(target->to_string(result.get_nested_column(), 3), valid);
+ auto default_value = target->create_column();
+ default_value->insert_default();
+ EXPECT_EQ(result.get_nested_column().compare_at(0, 0, *default_value, 1), 0);
+ EXPECT_EQ(result.get_nested_column().compare_at(2, 0, *default_value, 1), 0);
+
+ // Reveal one invalid row. Strict mode must fail. Non-strict mode must return NULL.
+ Block invalid {{make_nullable_jsonb_column({R"("bad")", valid, R"("bad")", valid},
+ {1, 0, 0, 0}),
+ from_type, "from"},
+ {nullptr, to_type, "to"}};
+ auto st = fn(ctx.get(), invalid, {0}, 1, invalid.rows(), nullptr);
+ if (strict) {
+ EXPECT_FALSE(st.ok());
+ } else {
+ ASSERT_TRUE(st.ok());
+ const auto& invalid_result =
+ assert_cast<const ColumnNullable&>(*invalid.get_by_position(1).column);
+ ASSERT_EQ(invalid_result.size(), 4);
+ EXPECT_TRUE(invalid_result.is_null_at(2));
+ EXPECT_EQ(target->to_string(invalid_result.get_nested_column(), 3), valid);
+ }
+ }
+ }
+}
+
+TEST_F(FunctionCastTest, jsonb_source_string_skips_null_rows) {
+ const auto from_type = make_nullable(std::make_shared<DataTypeJsonb>());
+ const auto to_type = make_nullable(std::make_shared<DataTypeString>());
+ RuntimeState state;
+ for (bool string_as_string : {false, true}) {
+ auto ctx = FunctionContext::create_context(&state, to_type, {from_type});
+ ctx->set_jsonb_string_as_string(string_as_string);
+ auto fn = get_cast_wrapper(ctx.get(), from_type, to_type);
+ Block block {
+ {make_nullable_jsonb_column({R"("bad")", R"("ok")"}, {1, 0}), from_type, "from"},
+ {nullptr, to_type, "to"}};
+ ASSERT_TRUE(fn(ctx.get(), block, {0}, 1, block.rows(), nullptr).ok());
+ const auto& result = assert_cast<const ColumnNullable&>(*block.get_by_position(1).column);
+ ASSERT_EQ(result.size(), 2);
+ EXPECT_TRUE(result.is_null_at(0));
+ EXPECT_FALSE(result.is_null_at(1));
+ EXPECT_EQ(result.get_nested_column().get_data_at(0).size, 0);
+ EXPECT_EQ(result.get_nested_column().get_data_at(1).to_string(), "ok");
+ }
+}
+
+TEST_F(FunctionCastTest, jsonb_destination_skips_null_struct_rows) {
+ const auto struct_type = std::make_shared<DataTypeStruct>(
+ DataTypes {std::make_shared<DataTypeInt32>()}, Strings {std::string(256, 'k')});
+ const auto from_type = make_nullable(struct_type);
+ const auto to_type = make_nullable(std::make_shared<DataTypeJsonb>());
+ for (bool strict : {false, true}) {
+ auto ctx = create_context(strict);
+ auto fn = get_cast_wrapper(ctx.get(), from_type, to_type);
+ const auto build_source = [](const std::vector<UInt8>& null_map) {
+ auto nested = ColumnStruct::create(
+ Columns {ColumnHelper::create_column<DataTypeInt32>({1, 2})});
+ return ColumnNullable::create(std::move(nested),
+ ColumnHelper::create_column<DataTypeUInt8>(null_map));
+ };
+ Block block {{build_source({1, 1}), from_type, "from"}, {nullptr, to_type, "to"}};
+ ASSERT_TRUE(fn(ctx.get(), block, {0}, 1, block.rows(), nullptr).ok());
+ const auto& result = assert_cast<const ColumnNullable&>(*block.get_by_position(1).column);
+ ASSERT_EQ(result.size(), 2);
+ EXPECT_TRUE(result.is_null_at(0));
+ EXPECT_TRUE(result.is_null_at(1));
+ EXPECT_EQ(result.get_nested_column().get_data_at(0).size, 0);
+ EXPECT_EQ(result.get_nested_column().get_data_at(1).size, 0);
+
+ // The same field name must cause an error when its row is visible, in either mode.
+ for (const auto& mask : {std::vector<UInt8> {1, 0}, std::vector<UInt8> {0, 0}}) {
+ Block invalid {{build_source(mask), from_type, "from"}, {nullptr, to_type, "to"}};
+ auto st = fn(ctx.get(), invalid, {0}, 1, invalid.rows(), nullptr);
+ EXPECT_FALSE(st.ok());
+ EXPECT_NE(st.to_string().find("key size exceeds max limit"), std::string::npos);
+ }
+ }
+}
+
+TEST_F(FunctionCastTest, jsonb_destination_skips_hidden_struct_and_keeps_visible_array) {
+ const auto struct_type = std::make_shared<DataTypeStruct>(
+ DataTypes {std::make_shared<DataTypeInt32>()}, Strings {std::string(256, 'k')});
+ const auto from_type =
+ make_nullable(std::make_shared<DataTypeArray>(make_nullable(struct_type)));
+ const auto to_type = make_nullable(std::make_shared<DataTypeJsonb>());
+ for (bool strict : {false, true}) {
+ auto ctx = create_context(strict);
+ auto fn = get_cast_wrapper(ctx.get(), from_type, to_type);
+ const auto build_source = [](const std::vector<UInt8>& null_map) {
+ auto structs = ColumnStruct::create(
+ Columns {ColumnHelper::create_column<DataTypeInt32>({1, 2})});
+ auto elements = ColumnNullable::create(std::move(structs), ColumnUInt8::create(2, 0));
+ auto arrays = ColumnArray::create(
+ std::move(elements),
+ ColumnHelper::create_column_offsets<TYPE_UINT64>({1, 1, 2, 2}));
+ return ColumnNullable::create(std::move(arrays),
+ ColumnHelper::create_column<DataTypeUInt8>(null_map));
+ };
+ Block block {{build_source({1, 0, 1, 0}), from_type, "from"}, {nullptr, to_type, "to"}};
+ ASSERT_TRUE(fn(ctx.get(), block, {0}, 1, block.rows(), nullptr).ok());
+ const auto& result = assert_cast<const ColumnNullable&>(*block.get_by_position(1).column);
+ ASSERT_EQ(result.size(), 4);
+ for (size_t i = 0; i < 4; ++i) {
+ EXPECT_EQ(result.is_null_at(i), i % 2 == 0);
+ }
+ EXPECT_EQ(to_type->to_string(result, 1), "[]");
+ EXPECT_EQ(to_type->to_string(result, 3), "[]");
+ EXPECT_EQ(result.get_nested_column().get_data_at(0).size, 0);
+ EXPECT_EQ(result.get_nested_column().get_data_at(2).size, 0);
+
+ Block invalid {{build_source({1, 0, 0, 0}), from_type, "from"}, {nullptr, to_type, "to"}};
+ auto st = fn(ctx.get(), invalid, {0}, 1, invalid.rows(), nullptr);
+ EXPECT_FALSE(st.ok());
+ EXPECT_NE(st.to_string().find("key size exceeds max limit"), std::string::npos);
+ }
+}
+
+TEST_F(FunctionCastTest, jsonb_destination_numeric_skips_null_rows) {
+ const DataTypes targets = {std::make_shared<DataTypeInt32>(),
+ std::make_shared<DataTypeDecimal32>(9, 0)};
+ for (const auto& nested_type : targets) {
+ auto nested = nested_type->create_column();
+ nested->insert_default();
+ nested->insert_default();
+ const auto from_type = make_nullable(nested_type);
+ const auto to_type = make_nullable(std::make_shared<DataTypeJsonb>());
+ ColumnPtr source = ColumnNullable::create(
+ std::move(nested), ColumnHelper::create_column<DataTypeUInt8>({1, 0}));
+ for (bool strict : {false, true}) {
+ auto ctx = create_context(strict);
+ auto fn = get_cast_wrapper(ctx.get(), from_type, to_type);
+ Block block {{source, from_type, "from"}, {nullptr, to_type, "to"}};
+ ASSERT_TRUE(fn(ctx.get(), block, {0}, 1, block.rows(), nullptr).ok());
+ const auto& result =
+ assert_cast<const ColumnNullable&>(*block.get_by_position(1).column);
+ ASSERT_EQ(result.size(), 2);
+ EXPECT_TRUE(result.is_null_at(0));
+ EXPECT_FALSE(result.is_null_at(1));
+ EXPECT_EQ(result.get_nested_column().get_data_at(0).size, 0);
+ EXPECT_EQ(to_type->to_string(result, 1), "0");
+ }
+ }
+}
+
+} // namespace doris
diff --git a/be/test/exprs/function/cast/cast_timestamp_ns_test.cpp b/be/test/exprs/function/cast/cast_timestamp_ns_test.cpp
index 633aa40f9a9..4ec635ce317 100644
--- a/be/test/exprs/function/cast/cast_timestamp_ns_test.cpp
+++ b/be/test/exprs/function/cast/cast_timestamp_ns_test.cpp
@@ -289,4 +289,44 @@ TEST_F(FunctionCastTest, string_to_timestamp_ns_strict_rejects_non_strict_format
"Invalid TIMESTAMP_NS value");
}
+// A row that the input null map marks as NULL may still carry an arbitrary hidden payload in the
+// nested column. Strict cast parses the source value with a strict-mode serde batch, so the hidden
+// payload of a NULL row must not be validated.
+TEST_F(FunctionCastTest, numeric_to_timestamp_ns_skips_null_covered_payload) {
+ auto ctx = create_context(true);
+ auto from_type = std::make_shared<DataTypeNullable>(std::make_shared<DataTypeInt64>());
+ auto to_type = std::make_shared<DataTypeNullable>(std::make_shared<DataTypeTimeStampNs>());
+
+ // Row 0 is NULL with hidden payload 16770921001243, which is outside the signed
+ // epoch-nanosecond range; row 1 is a valid timestamp.
+ ColumnPtr from_column = ColumnHelper::create_nullable_column<DataTypeInt64>(
+ {16770921001243, 19700101000000}, {1, 0});
+
+ auto fn = get_cast_wrapper(ctx.get(), from_type, to_type);
+ ASSERT_TRUE(fn != nullptr);
+
+ Block block = {
+ {std::move(from_column), from_type, "from"},
+ {nullptr, to_type, "to"},
+ };
+ ASSERT_TRUE(fn(ctx.get(), block, {0}, 1, block.rows(), nullptr));
+
+ const auto& result = assert_cast<const ColumnNullable&>(*block.get_by_position(1).column);
+ EXPECT_EQ(result.get_null_map_data()[0], 1);
+ EXPECT_EQ(result.get_null_map_data()[1], 0);
+ EXPECT_EQ(to_type->to_string(*block.get_by_position(1).column, 1),
+ "1970-01-01 00:00:00.000000000");
+
+ // An out-of-range value in a visible (non NULL) row must still fail in strict mode.
+ {
+ ColumnPtr overflow_column = ColumnHelper::create_nullable_column<DataTypeInt64>(
+ {16770921001243, 22620412000000}, {1, 0});
+ Block overflow_block = {
+ {std::move(overflow_column), from_type, "from"},
+ {nullptr, to_type, "to"},
+ };
+ EXPECT_FALSE(fn(ctx.get(), overflow_block, {0}, 1, overflow_block.rows(), nullptr).ok());
+ }
+}
+
} // namespace doris
diff --git a/be/test/exprs/function/cast/cast_to_array_test.cpp b/be/test/exprs/function/cast/cast_to_array_test.cpp
index bf3b8a95b16..823589e7009 100644
--- a/be/test/exprs/function/cast/cast_to_array_test.cpp
+++ b/be/test/exprs/function/cast/cast_to_array_test.cpp
@@ -287,4 +287,144 @@ TEST_F(FunctionCastTest, test_from_non_null_array_to_nested_array_is_rejected) {
EXPECT_FALSE(fn(ctx.get(), block, {0}, 1, block.rows(), nullptr));
}
+// Build a Nullable(Array(Nullable(ElementDataType))) column. A row marked as NULL by `null_map`
+// keeps the elements that its offsets describe as hidden payload.
+template <typename ElementDataType>
+static ColumnPtr build_nullable_array_column(
+ const std::vector<typename ElementDataType::FieldType>& elements,
+ const std::vector<NullMap::value_type>& element_null_map,
+ const std::vector<uint64_t>& offsets, const std::vector<NullMap::value_type>& null_map) {
+ auto nested = ColumnHelper::create_nullable_column<ElementDataType>(elements, element_null_map);
+ auto array = ColumnArray::create(std::move(nested),
+ ColumnHelper::create_column_offsets<TYPE_UINT64>(offsets));
+ return ColumnNullable::create(std::move(array),
+ ColumnHelper::create_column<DataTypeUInt8>(null_map));
+}
+
+// A row that the input null map of an ARRAY marks as NULL may still keep a hidden payload in the
+// elements that belong to it (for example the branch of an IF() that was not taken). Those elements
+// are NULL by SQL semantics, so the strict cast must not validate them, and the mask of the row has
+// to be expanded to its elements instead of being applied to the element indexes directly.
+TEST_F(FunctionCastTest, test_cast_array_null_row_skips_hidden_payload) {
+ auto from_type = std::make_shared<DataTypeNullable>(
+ std::make_shared<DataTypeArray>(std::make_shared<DataTypeInt32>()));
+ auto to_type = std::make_shared<DataTypeNullable>(
+ std::make_shared<DataTypeArray>(std::make_shared<DataTypeInt8>()));
+
+ auto ctx = create_context(true);
+ auto fn = get_cast_wrapper(ctx.get(), from_type, to_type);
+ ASSERT_TRUE(fn != nullptr);
+
+ // Row 0 is NULL and keeps the hidden payload 128, which does not fit into TINYINT.
+ Block block = {
+ {build_nullable_array_column<DataTypeInt32>({128, 1, 2}, {0, 0, 0}, {1, 3}, {1, 0}),
+ from_type, "from"},
+ {nullptr, to_type, "to"},
+ };
+ ASSERT_TRUE(fn(ctx.get(), block, {0}, 1, block.rows(), nullptr));
+
+ const auto& result = assert_cast<const ColumnNullable&>(*block.get_by_position(1).column);
+ EXPECT_TRUE(result.is_null_at(0));
+ EXPECT_EQ(to_type->to_string(*block.get_by_position(1).column, 1), "[1, 2]");
+}
+
+// The mask of a NULL row has to be inherited through every level of nesting.
+TEST_F(FunctionCastTest, test_cast_nested_array_null_row_skips_hidden_payload) {
+ auto from_type = std::make_shared<DataTypeNullable>(std::make_shared<DataTypeArray>(
+ std::make_shared<DataTypeArray>(std::make_shared<DataTypeInt32>())));
+ auto to_type = std::make_shared<DataTypeNullable>(std::make_shared<DataTypeArray>(
+ std::make_shared<DataTypeArray>(std::make_shared<DataTypeInt8>())));
+
+ // Row 0 is NULL and keeps the hidden inner array [128], row 1 is [[1, 2]].
+ auto inner_array =
+ build_nullable_array_column<DataTypeInt32>({128, 1, 2}, {0, 0, 0}, {1, 3}, {0, 0});
+ auto outer_array = ColumnArray::create(
+ inner_array, ColumnHelper::create_column_offsets<TYPE_UINT64>({1, 2}));
+ ColumnPtr from_column = ColumnNullable::create(
+ std::move(outer_array), ColumnHelper::create_column<DataTypeUInt8>({1, 0}));
+
+ auto ctx = create_context(true);
+ auto fn = get_cast_wrapper(ctx.get(), from_type, to_type);
+ ASSERT_TRUE(fn != nullptr);
+
+ Block block = {
+ {std::move(from_column), from_type, "from"},
+ {nullptr, to_type, "to"},
+ };
+ ASSERT_TRUE(fn(ctx.get(), block, {0}, 1, block.rows(), nullptr));
+
+ const auto& result = assert_cast<const ColumnNullable&>(*block.get_by_position(1).column);
+ EXPECT_TRUE(result.is_null_at(0));
+ EXPECT_EQ(to_type->to_string(*block.get_by_position(1).column, 1), "[[1, 2]]");
+}
+
+// The same masking gap exists when the element cast cannot replace the hidden payload with a
+// default, so the mask itself has to reach the element cast.
+TEST_F(FunctionCastTest, test_cast_array_null_row_skips_hidden_string_payload) {
+ auto from_type = std::make_shared<DataTypeNullable>(
+ std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>()));
+ auto to_type = std::make_shared<DataTypeNullable>(
+ std::make_shared<DataTypeArray>(std::make_shared<DataTypeInt8>()));
+
+ auto ctx = create_context(true);
+ auto fn = get_cast_wrapper(ctx.get(), from_type, to_type);
+ ASSERT_TRUE(fn != nullptr);
+
+ Block block = {
+ {build_nullable_array_column<DataTypeString>({"abc", "1", "2"}, {0, 0, 0}, {1, 3},
+ {1, 0}),
+ from_type, "from"},
+ {nullptr, to_type, "to"},
+ };
+ ASSERT_TRUE(fn(ctx.get(), block, {0}, 1, block.rows(), nullptr));
+
+ const auto& result = assert_cast<const ColumnNullable&>(*block.get_by_position(1).column);
+ EXPECT_TRUE(result.is_null_at(0));
+ EXPECT_EQ(to_type->to_string(*block.get_by_position(1).column, 1), "[1, 2]");
+}
+
+// Only the rows that are NULL are skipped: an out of range element of a visible row still fails.
+TEST_F(FunctionCastTest, test_cast_array_visible_element_still_fails) {
+ auto from_type = std::make_shared<DataTypeNullable>(
+ std::make_shared<DataTypeArray>(std::make_shared<DataTypeInt32>()));
+ auto to_type = std::make_shared<DataTypeNullable>(
+ std::make_shared<DataTypeArray>(std::make_shared<DataTypeInt8>()));
+
+ auto ctx = create_context(true);
+ auto fn = get_cast_wrapper(ctx.get(), from_type, to_type);
+ ASSERT_TRUE(fn != nullptr);
+
+ // Row 0 is NULL with a hidden payload, row 1 keeps the element 300 that overflows TINYINT.
+ Block block = {
+ {build_nullable_array_column<DataTypeInt32>({128, 1, 300}, {0, 0, 0}, {1, 3}, {1, 0}),
+ from_type, "from"},
+ {nullptr, to_type, "to"},
+ };
+ EXPECT_FALSE(fn(ctx.get(), block, {0}, 1, block.rows(), nullptr).ok());
+}
+
+// Without strict mode the out of range element of a visible row becomes NULL, while the NULL row
+// and its hidden payload are untouched.
+TEST_F(FunctionCastTest, test_cast_array_non_strict_null_row) {
+ auto from_type = std::make_shared<DataTypeNullable>(
+ std::make_shared<DataTypeArray>(std::make_shared<DataTypeInt32>()));
+ auto to_type = std::make_shared<DataTypeNullable>(
+ std::make_shared<DataTypeArray>(std::make_shared<DataTypeInt8>()));
+
+ auto ctx = create_context(false);
+ auto fn = get_cast_wrapper(ctx.get(), from_type, to_type);
+ ASSERT_TRUE(fn != nullptr);
+
+ Block block = {
+ {build_nullable_array_column<DataTypeInt32>({128, 1, 300}, {0, 0, 0}, {1, 3}, {1, 0}),
+ from_type, "from"},
+ {nullptr, to_type, "to"},
+ };
+ ASSERT_TRUE(fn(ctx.get(), block, {0}, 1, block.rows(), nullptr));
+
+ const auto& result = assert_cast<const ColumnNullable&>(*block.get_by_position(1).column);
+ EXPECT_TRUE(result.is_null_at(0));
+ EXPECT_EQ(to_type->to_string(*block.get_by_position(1).column, 1), "[1, null]");
+}
+
} // namespace doris
diff --git a/be/test/exprs/function/cast/cast_to_date_test.cpp b/be/test/exprs/function/cast/cast_to_date_test.cpp
index d88bc66d74e..7e00bf5f303 100644
--- a/be/test/exprs/function/cast/cast_to_date_test.cpp
+++ b/be/test/exprs/function/cast/cast_to_date_test.cpp
@@ -398,4 +398,56 @@ TEST_F(FunctionCastTest, test_from_time_to_date) {
}
}
+// A row that the input null map marks as NULL may still carry an arbitrary hidden payload in the
+// nested column. Strict cast parses the source value with a strict-mode serde batch, so the hidden
+// payload of a NULL row must not be validated.
+TEST_F(FunctionCastTest, numeric_to_date_skips_null_covered_payload) {
+ auto ctx = create_context(true);
+ auto to_type = std::make_shared<DataTypeNullable>(std::make_shared<DataTypeDateV2>());
+
+ auto run_and_check = [&](const DataTypePtr& from_type, ColumnPtr from_column,
+ const std::string& expected_non_null) {
+ auto fn = get_cast_wrapper(ctx.get(), from_type, to_type);
+ ASSERT_TRUE(fn != nullptr);
+
+ Block block = {
+ {std::move(from_column), from_type, "from"},
+ {nullptr, to_type, "to"},
+ };
+ ASSERT_TRUE(fn(ctx.get(), block, {0}, 1, block.rows(), nullptr));
+
+ const auto& result = assert_cast<const ColumnNullable&>(*block.get_by_position(1).column);
+ // The row with the hidden payload stays NULL.
+ EXPECT_EQ(result.get_null_map_data()[0], 1);
+ EXPECT_EQ(result.get_null_map_data()[1], 0);
+ EXPECT_EQ(to_type->to_string(*block.get_by_position(1).column, 1), expected_non_null);
+ };
+
+ // Nullable(BIGINT) -> Nullable(DATEV2), row 0 is NULL with hidden payload 1000 (invalid date).
+ run_and_check(
+ std::make_shared<DataTypeNullable>(std::make_shared<DataTypeInt64>()),
+ ColumnHelper::create_nullable_column<DataTypeInt64>({1000, 20150102030405}, {1, 0}),
+ "2015-01-02");
+
+ // Nullable(DOUBLE) -> Nullable(DATEV2), row 0 is NULL with hidden payload 7777777.0.
+ run_and_check(std::make_shared<DataTypeNullable>(std::make_shared<DataTypeFloat64>()),
+ ColumnHelper::create_nullable_column<DataTypeFloat64>(
+ {7777777.0, 20150102030405.0}, {1, 0}),
+ "2015-01-02");
+
+ // An invalid value in a visible (non NULL) row must still fail in strict mode.
+ {
+ auto from_type = std::make_shared<DataTypeNullable>(std::make_shared<DataTypeInt64>());
+ auto from_column =
+ ColumnHelper::create_nullable_column<DataTypeInt64>({1000, 7777777}, {1, 0});
+ auto fn = get_cast_wrapper(ctx.get(), from_type, to_type);
+ ASSERT_TRUE(fn != nullptr);
+ Block block = {
+ {std::move(from_column), from_type, "from"},
+ {nullptr, to_type, "to"},
+ };
+ EXPECT_FALSE(fn(ctx.get(), block, {0}, 1, block.rows(), nullptr).ok());
+ }
+}
+
} // namespace doris
diff --git a/be/test/exprs/function/cast/cast_to_decimal.cpp b/be/test/exprs/function/cast/cast_to_decimal.cpp
index 980d1569863..b4745e9b4fc 100644
--- a/be/test/exprs/function/cast/cast_to_decimal.cpp
+++ b/be/test/exprs/function/cast/cast_to_decimal.cpp
@@ -269,4 +269,45 @@ TEST_F(FunctionCastToDecimalTest, test_from_bool_overflow) {
from_bool_overflow_test_func<Decimal128V3>();
from_bool_overflow_test_func<Decimal256>();
}
+
+// A row that the input null map marks as NULL may still carry an arbitrary hidden payload in the
+// nested column. The strict integer-to-decimal kernel must skip such rows instead of reporting the
+// hidden value as out of range. check_overflow_for_decimal is disabled here so that the CAST
+// wrapper does not replace the hidden payload beforehand and the kernel itself has to skip it.
+TEST_F(FunctionCastToDecimalTest, int_to_decimal_skips_null_covered_payload) {
+ auto ctx = create_context(true);
+ ctx->set_check_overflow_for_decimal(false);
+
+ auto from_type = std::make_shared<DataTypeNullable>(std::make_shared<DataTypeInt64>());
+ auto to_type = std::make_shared<DataTypeNullable>(std::make_shared<DataTypeDecimal32>(9, 0));
+
+ // Row 0 is NULL with hidden payload 999999999999999999, which overflows DECIMAL(9, 0).
+ ColumnPtr from_column =
+ ColumnHelper::create_nullable_column<DataTypeInt64>({999999999999999999LL, 1}, {1, 0});
+
+ auto fn = get_cast_wrapper(ctx.get(), from_type, to_type);
+ ASSERT_TRUE(fn != nullptr);
+
+ Block block = {
+ {std::move(from_column), from_type, "from"},
+ {nullptr, to_type, "to"},
+ };
+ ASSERT_TRUE(fn(ctx.get(), block, {0}, 1, block.rows(), nullptr));
+
+ const auto& result = assert_cast<const ColumnNullable&>(*block.get_by_position(1).column);
+ EXPECT_EQ(result.get_null_map_data()[0], 1);
+ EXPECT_EQ(result.get_null_map_data()[1], 0);
+ EXPECT_EQ(to_type->to_string(*block.get_by_position(1).column, 1), "1");
+
+ // An out-of-range value in a visible (non NULL) row must still fail in strict mode.
+ {
+ ColumnPtr overflow_column = ColumnHelper::create_nullable_column<DataTypeInt64>(
+ {999999999999999999LL, 999999999999999999LL}, {1, 0});
+ Block overflow_block = {
+ {std::move(overflow_column), from_type, "from"},
+ {nullptr, to_type, "to"},
+ };
+ EXPECT_FALSE(fn(ctx.get(), overflow_block, {0}, 1, overflow_block.rows(), nullptr).ok());
+ }
+}
} // namespace doris
diff --git a/be/test/exprs/function/cast/cast_to_integer.cpp b/be/test/exprs/function/cast/cast_to_integer.cpp
index ca13022ce3c..8778d39280e 100644
--- a/be/test/exprs/function/cast/cast_to_integer.cpp
+++ b/be/test/exprs/function/cast/cast_to_integer.cpp
@@ -2316,4 +2316,64 @@ TEST_F(FunctionCastToIntTest, test_from_time) {
from_time_test_func<TYPE_BIGINT>();
from_time_test_func<TYPE_LARGEINT>();
}
-} // namespace doris
\ No newline at end of file
+
+// A row that the input null map marks as NULL may still carry an arbitrary hidden payload in the
+// nested column of a Nullable input. That payload has no SQL semantics, so strict cast must skip
+// such rows instead of reporting the hidden value as out of range.
+TEST_F(FunctionCastToIntTest, test_strict_cast_skips_null_covered_payload) {
+ auto ctx = create_context(true);
+ auto to_type = std::make_shared<DataTypeNullable>(std::make_shared<DataTypeInt8>());
+
+ auto run_and_check = [&](const DataTypePtr& from_type, ColumnPtr from_column,
+ int8_t non_null_expected) {
+ auto fn = get_cast_wrapper(ctx.get(), from_type, to_type);
+ ASSERT_TRUE(fn != nullptr);
+
+ Block block = {
+ {std::move(from_column), from_type, "from"},
+ {nullptr, to_type, "to"},
+ };
+ ASSERT_TRUE(fn(ctx.get(), block, {0}, 1, block.rows(), nullptr));
+
+ const auto& result = assert_cast<const ColumnNullable&>(*block.get_by_position(1).column);
+ const auto& null_map = result.get_null_map_data();
+ // The row with the hidden out-of-range payload stays NULL.
+ EXPECT_EQ(null_map[0], 1);
+ EXPECT_EQ(null_map[1], 0);
+ EXPECT_EQ(assert_cast<const ColumnInt8&>(result.get_nested_column()).get_data()[1],
+ non_null_expected);
+ };
+
+ // Nullable(INT) -> Nullable(TINYINT), row 0 is NULL with hidden payload 128.
+ run_and_check(std::make_shared<DataTypeNullable>(std::make_shared<DataTypeInt32>()),
+ ColumnHelper::create_nullable_column<DataTypeInt32>({128, 1}, {1, 0}), 1);
+
+ // Nullable(DOUBLE) -> Nullable(TINYINT), row 0 is NULL with hidden payload 128.0.
+ run_and_check(std::make_shared<DataTypeNullable>(std::make_shared<DataTypeFloat64>()),
+ ColumnHelper::create_nullable_column<DataTypeFloat64>({128.0, 1.0}, {1, 0}), 1);
+}
+
+// A nullable source whose NULL map is empty has nothing to skip, so the cast is a plain row by row
+// conversion: every value is converted and none of them is treated as masked.
+TEST_F(FunctionCastToIntTest, test_strict_cast_nullable_without_null) {
+ auto ctx = create_context(true);
+ auto from_type = std::make_shared<DataTypeNullable>(std::make_shared<DataTypeInt32>());
+ auto to_type = std::make_shared<DataTypeNullable>(std::make_shared<DataTypeInt8>());
+
+ auto fn = get_cast_wrapper(ctx.get(), from_type, to_type);
+ ASSERT_TRUE(fn != nullptr);
+
+ Block block = {
+ {ColumnHelper::create_nullable_column<DataTypeInt32>({1, 2}, {0, 0}), from_type,
+ "from"},
+ {nullptr, to_type, "to"},
+ };
+ ASSERT_TRUE(fn(ctx.get(), block, {0}, 1, block.rows(), nullptr));
+
+ const auto& result = assert_cast<const ColumnNullable&>(*block.get_by_position(1).column);
+ EXPECT_EQ(result.get_null_map_data()[0], 0);
+ EXPECT_EQ(result.get_null_map_data()[1], 0);
+ EXPECT_EQ(to_type->to_string(*block.get_by_position(1).column, 0), "1");
+ EXPECT_EQ(to_type->to_string(*block.get_by_position(1).column, 1), "2");
+}
+} // namespace doris
diff --git a/be/test/exprs/function/cast/cast_to_map_test.cpp b/be/test/exprs/function/cast/cast_to_map_test.cpp
index f3020d94f69..77d0a9f15f0 100644
--- a/be/test/exprs/function/cast/cast_to_map_test.cpp
+++ b/be/test/exprs/function/cast/cast_to_map_test.cpp
@@ -179,4 +179,101 @@ TEST_F(FunctionCastTest, test_from_string_to_map_string_string) {
builder.build(), false);
}
+// A row that the input null map of a MAP marks as NULL may still keep a hidden payload in the
+// entries that belong to it (for example the branch of an IF() that was not taken). Keys and values
+// are stored flattened, so the mask of the row has to be expanded to its entries: applying the row
+// aligned mask to the entry indexes would check the wrong entries and read behind the mask.
+TEST_F(FunctionCastTest, test_cast_map_null_row_skips_hidden_entry_payload) {
+ auto from_key_column =
+ ColumnHelper::create_nullable_column<DataTypeInt32>({1, 2, 128}, {0, 0, 0});
+ auto from_value_column =
+ ColumnHelper::create_nullable_column<DataTypeInt32>({10, 20, 300}, {0, 0, 0});
+ auto from_map = ColumnMap::create(from_key_column, from_value_column,
+ ColumnHelper::create_column_offsets<TYPE_UINT64>({2, 3}));
+ ColumnPtr from_column = ColumnNullable::create(
+ std::move(from_map), ColumnHelper::create_column<DataTypeUInt8>({0, 1}));
+
+ auto from_type = std::make_shared<DataTypeNullable>(std::make_shared<DataTypeMap>(
+ std::make_shared<DataTypeNullable>(std::make_shared<DataTypeInt32>()),
+ std::make_shared<DataTypeNullable>(std::make_shared<DataTypeInt32>())));
+ auto to_type = std::make_shared<DataTypeNullable>(std::make_shared<DataTypeMap>(
+ std::make_shared<DataTypeNullable>(std::make_shared<DataTypeInt8>()),
+ std::make_shared<DataTypeNullable>(std::make_shared<DataTypeInt8>())));
+
+ auto ctx = create_context(true);
+ auto fn = get_cast_wrapper(ctx.get(), from_type, to_type);
+ ASSERT_TRUE(fn != nullptr);
+
+ // Row 0 is {1: 10, 2: 20}, row 1 is NULL and keeps the hidden entry {128: 300}.
+ Block block = {
+ {std::move(from_column), from_type, "from"},
+ {nullptr, to_type, "to"},
+ };
+ ASSERT_TRUE(fn(ctx.get(), block, {0}, 1, block.rows(), nullptr));
+
+ const auto& result = assert_cast<const ColumnNullable&>(*block.get_by_position(1).column);
+ EXPECT_FALSE(result.is_null_at(0));
+ EXPECT_EQ(to_type->to_string(*block.get_by_position(1).column, 0), "{1:10, 2:20}");
+ EXPECT_TRUE(result.is_null_at(1));
+}
+
+// The hidden entry of a visible row is still validated.
+TEST_F(FunctionCastTest, test_cast_map_visible_entry_still_fails) {
+ auto from_key_column = ColumnHelper::create_nullable_column<DataTypeInt32>({1, 128}, {0, 0});
+ auto from_value_column = ColumnHelper::create_nullable_column<DataTypeInt32>({10, 300}, {0, 0});
+ auto from_map = ColumnMap::create(from_key_column, from_value_column,
+ ColumnHelper::create_column_offsets<TYPE_UINT64>({1, 2}));
+ ColumnPtr from_column = ColumnNullable::create(
+ std::move(from_map), ColumnHelper::create_column<DataTypeUInt8>({0, 0}));
+
+ auto from_type = std::make_shared<DataTypeNullable>(std::make_shared<DataTypeMap>(
+ std::make_shared<DataTypeNullable>(std::make_shared<DataTypeInt32>()),
+ std::make_shared<DataTypeNullable>(std::make_shared<DataTypeInt32>())));
+ auto to_type = std::make_shared<DataTypeNullable>(std::make_shared<DataTypeMap>(
+ std::make_shared<DataTypeNullable>(std::make_shared<DataTypeInt8>()),
+ std::make_shared<DataTypeNullable>(std::make_shared<DataTypeInt8>())));
+
+ auto ctx = create_context(true);
+ auto fn = get_cast_wrapper(ctx.get(), from_type, to_type);
+ ASSERT_TRUE(fn != nullptr);
+
+ Block block = {
+ {std::move(from_column), from_type, "from"},
+ {nullptr, to_type, "to"},
+ };
+ EXPECT_FALSE(fn(ctx.get(), block, {0}, 1, block.rows(), nullptr).ok());
+}
+
+// A key whose type does not change is passed through, while the value that does change still
+// inherits the NULL of its row, so the hidden payload of the value is not validated.
+TEST_F(FunctionCastTest, test_cast_map_unchanged_key_with_null_row) {
+ auto from_key_column = ColumnHelper::create_nullable_column<DataTypeInt32>({5, 1}, {0, 0});
+ auto from_value_column = ColumnHelper::create_nullable_column<DataTypeInt32>({128, 2}, {0, 0});
+ auto from_map = ColumnMap::create(from_key_column, from_value_column,
+ ColumnHelper::create_column_offsets<TYPE_UINT64>({1, 2}));
+ ColumnPtr from_column = ColumnNullable::create(
+ std::move(from_map), ColumnHelper::create_column<DataTypeUInt8>({1, 0}));
+
+ auto from_type = std::make_shared<DataTypeNullable>(std::make_shared<DataTypeMap>(
+ std::make_shared<DataTypeNullable>(std::make_shared<DataTypeInt32>()),
+ std::make_shared<DataTypeNullable>(std::make_shared<DataTypeInt32>())));
+ auto to_type = std::make_shared<DataTypeNullable>(std::make_shared<DataTypeMap>(
+ std::make_shared<DataTypeNullable>(std::make_shared<DataTypeInt32>()),
+ std::make_shared<DataTypeNullable>(std::make_shared<DataTypeInt8>())));
+
+ auto ctx = create_context(true);
+ auto fn = get_cast_wrapper(ctx.get(), from_type, to_type);
+ ASSERT_TRUE(fn != nullptr);
+
+ Block block = {
+ {std::move(from_column), from_type, "from"},
+ {nullptr, to_type, "to"},
+ };
+ ASSERT_TRUE(fn(ctx.get(), block, {0}, 1, block.rows(), nullptr));
+
+ const auto& result = assert_cast<const ColumnNullable&>(*block.get_by_position(1).column);
+ EXPECT_TRUE(result.is_null_at(0));
+ EXPECT_EQ(to_type->to_string(*block.get_by_position(1).column, 1), "{1:2}");
+}
+
} // namespace doris
diff --git a/be/test/exprs/function/cast/cast_to_struct_test.cpp b/be/test/exprs/function/cast/cast_to_struct_test.cpp
index a1066704f66..6cd7c2705c9 100644
--- a/be/test/exprs/function/cast/cast_to_struct_test.cpp
+++ b/be/test/exprs/function/cast/cast_to_struct_test.cpp
@@ -128,4 +128,91 @@ TEST_F(FunctionCastTest, test_from_string_to_struct2) {
std::make_shared<DataTypeString>(), "from"),
builder.build({"a", "b", "c", "d"}), false);
}
+
+// A field of a row that the input null map of a STRUCT marks as NULL may still keep a hidden payload
+// (for example the branch of an IF() that was not taken), and the child cast of a nullable field
+// reads its NULL state from the field column, so the mask of the row has to be inherited by it.
+TEST_F(FunctionCastTest, test_cast_struct_null_row_skips_hidden_field_payload) {
+ auto from_field_column = ColumnHelper::create_nullable_column<DataTypeInt32>({128, 1}, {0, 0});
+ auto from_struct = ColumnStruct::create(Columns {from_field_column});
+ ColumnPtr from_column = ColumnNullable::create(
+ std::move(from_struct), ColumnHelper::create_column<DataTypeUInt8>({1, 0}));
+
+ auto from_type = std::make_shared<DataTypeNullable>(std::make_shared<DataTypeStruct>(
+ DataTypes {std::make_shared<DataTypeNullable>(std::make_shared<DataTypeInt32>())}));
+ auto to_type = std::make_shared<DataTypeNullable>(std::make_shared<DataTypeStruct>(
+ DataTypes {std::make_shared<DataTypeNullable>(std::make_shared<DataTypeInt8>())}));
+
+ auto ctx = create_context(true);
+ auto fn = get_cast_wrapper(ctx.get(), from_type, to_type);
+ ASSERT_TRUE(fn != nullptr);
+
+ // Row 0 is NULL and keeps the hidden payload 128, which does not fit into TINYINT.
+ Block block = {
+ {std::move(from_column), from_type, "from"},
+ {nullptr, to_type, "to"},
+ };
+ ASSERT_TRUE(fn(ctx.get(), block, {0}, 1, block.rows(), nullptr));
+
+ const auto& result = assert_cast<const ColumnNullable&>(*block.get_by_position(1).column);
+ EXPECT_TRUE(result.is_null_at(0));
+ EXPECT_EQ(to_type->to_string(*block.get_by_position(1).column, 1), R"({"1":1})");
+}
+
+// An out of range field of a visible row still fails.
+TEST_F(FunctionCastTest, test_cast_struct_visible_field_still_fails) {
+ auto from_field_column =
+ ColumnHelper::create_nullable_column<DataTypeInt32>({128, 300}, {0, 0});
+ auto from_struct = ColumnStruct::create(Columns {from_field_column});
+ ColumnPtr from_column = ColumnNullable::create(
+ std::move(from_struct), ColumnHelper::create_column<DataTypeUInt8>({1, 0}));
+
+ auto from_type = std::make_shared<DataTypeNullable>(std::make_shared<DataTypeStruct>(
+ DataTypes {std::make_shared<DataTypeNullable>(std::make_shared<DataTypeInt32>())}));
+ auto to_type = std::make_shared<DataTypeNullable>(std::make_shared<DataTypeStruct>(
+ DataTypes {std::make_shared<DataTypeNullable>(std::make_shared<DataTypeInt8>())}));
+
+ auto ctx = create_context(true);
+ auto fn = get_cast_wrapper(ctx.get(), from_type, to_type);
+ ASSERT_TRUE(fn != nullptr);
+
+ Block block = {
+ {std::move(from_column), from_type, "from"},
+ {nullptr, to_type, "to"},
+ };
+ EXPECT_FALSE(fn(ctx.get(), block, {0}, 1, block.rows(), nullptr).ok());
+}
+
+// A field whose type does not change is passed through, while the fields that do change still
+// inherit the NULL of their row, so the hidden payload of the changed field is not validated.
+TEST_F(FunctionCastTest, test_cast_struct_unchanged_field_with_null_row) {
+ auto first_column = ColumnHelper::create_nullable_column<DataTypeInt32>({5, 1}, {0, 0});
+ auto second_column = ColumnHelper::create_nullable_column<DataTypeInt32>({128, 2}, {0, 0});
+ auto from_struct = ColumnStruct::create(Columns {first_column, second_column});
+ ColumnPtr from_column = ColumnNullable::create(
+ std::move(from_struct), ColumnHelper::create_column<DataTypeUInt8>({1, 0}));
+
+ auto from_type = std::make_shared<DataTypeNullable>(std::make_shared<DataTypeStruct>(
+ DataTypes {std::make_shared<DataTypeNullable>(std::make_shared<DataTypeInt32>()),
+ std::make_shared<DataTypeNullable>(std::make_shared<DataTypeInt32>())},
+ Strings {"f1", "f2"}));
+ auto to_type = std::make_shared<DataTypeNullable>(std::make_shared<DataTypeStruct>(
+ DataTypes {std::make_shared<DataTypeNullable>(std::make_shared<DataTypeInt32>()),
+ std::make_shared<DataTypeNullable>(std::make_shared<DataTypeInt8>())},
+ Strings {"f1", "f2"}));
+
+ auto ctx = create_context(true);
+ auto fn = get_cast_wrapper(ctx.get(), from_type, to_type);
+ ASSERT_TRUE(fn != nullptr);
+
+ Block block = {
+ {std::move(from_column), from_type, "from"},
+ {nullptr, to_type, "to"},
+ };
+ ASSERT_TRUE(fn(ctx.get(), block, {0}, 1, block.rows(), nullptr));
+
+ const auto& result = assert_cast<const ColumnNullable&>(*block.get_by_position(1).column);
+ EXPECT_TRUE(result.is_null_at(0));
+ EXPECT_EQ(to_type->to_string(*block.get_by_position(1).column, 1), R"({"f1":1, "f2":2})");
+}
} // namespace doris
diff --git a/regression-test/data/function_p0/cast/to_complex/test_cast_complex_null_covered_payload.out b/regression-test/data/function_p0/cast/to_complex/test_cast_complex_null_covered_payload.out
new file mode 100644
index 00000000000..c3259c8c171
--- /dev/null
+++ b/regression-test/data/function_p0/cast/to_complex/test_cast_complex_null_covered_payload.out
@@ -0,0 +1,97 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !array_strict_sc_false --
+1 \N
+2 [1, 2]
+
+-- !array_non_strict_sc_false --
+1 \N
+2 [1, 2]
+
+-- !nullif_array_strict_sc_false --
+1 \N
+2 [1, 2]
+
+-- !nullif_array_non_strict_sc_false --
+1 \N
+2 [1, 2]
+
+-- !array_function_strict_sc_false --
+1 \N
+2 [7]
+
+-- !array_function_non_strict_sc_false --
+1 \N
+2 [7]
+
+-- !nested_array_strict_sc_false --
+1 \N
+2 [[1, 2]]
+
+-- !nested_array_non_strict_sc_false --
+1 \N
+2 [[1, 2]]
+
+-- !map_value_strict_sc_false --
+1 \N
+2 {1:10, 2:20}
+
+-- !map_value_non_strict_sc_false --
+1 \N
+2 {1:10, 2:20}
+
+-- !map_key_value_strict_sc_false --
+1 \N
+2 {1:10, 2:20}
+
+-- !map_key_value_non_strict_sc_false --
+1 \N
+2 {1:10, 2:20}
+
+-- !map_function_strict_sc_false --
+1 \N
+2 {7:7}
+
+-- !map_function_non_strict_sc_false --
+1 \N
+2 {7:7}
+
+-- !struct_strict_sc_false --
+1 \N
+2 {"f1":7}
+
+-- !struct_non_strict_sc_false --
+1 \N
+2 {"f1":7}
+
+-- !string_array_strict_sc_false --
+1 \N
+2 [1, 2]
+
+-- !string_array_non_strict_sc_false --
+1 \N
+2 [1, 2]
+
+-- !string_array_nullif_strict_sc_false --
+1 \N
+2 [1, 2]
+
+-- !array_strict_sc_true --
+1 \N
+2 [1, 2]
+
+-- !array_non_strict_sc_true --
+1 \N
+2 [1, 2]
+
+-- !nested_array_strict_sc_true --
+1 \N
+2 [[1, 2]]
+
+-- !map_key_value_strict_sc_true --
+1 \N
+2 {1:10, 2:20}
+
+-- !struct_strict_sc_true --
+1 \N
+2 {"f1":7}
+
diff --git a/regression-test/data/function_p0/cast/to_int/from_int/test_cast_to_tinyint_from_int_nullable_null_covered_payload.out b/regression-test/data/function_p0/cast/to_int/from_int/test_cast_to_tinyint_from_int_nullable_null_covered_payload.out
new file mode 100644
index 00000000000..ba076596465
--- /dev/null
+++ b/regression-test/data/function_p0/cast/to_int/from_int/test_cast_to_tinyint_from_int_nullable_null_covered_payload.out
@@ -0,0 +1,49 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !if_strict_sc_false --
+1 \N
+2 1
+
+-- !if_non_strict_sc_false --
+1 \N
+2 1
+
+-- !nullif_strict_sc_false --
+1 \N
+2 1
+
+-- !nullif_non_strict_sc_false --
+1 \N
+2 1
+
+-- !if_strict_sc_true --
+1 \N
+2 1
+
+-- !if_non_strict_sc_true --
+1 \N
+2 1
+
+-- !nullif_strict_sc_true --
+1 \N
+2 1
+
+-- !nullif_non_strict_sc_true --
+1 \N
+2 1
+
+-- !date_strict --
+1 \N
+2 2015-01-02
+
+-- !date_non_strict --
+1 \N
+2 2015-01-02
+
+-- !timestamp_ns_strict --
+1 \N
+2 2015-01-02 03:04:05.000000000
+
+-- !timestamp_ns_non_strict --
+1 \N
+2 2015-01-02 03:04:05.000000000
+
diff --git a/regression-test/suites/function_p0/cast/to_complex/test_cast_complex_null_covered_payload.groovy b/regression-test/suites/function_p0/cast/to_complex/test_cast_complex_null_covered_payload.groovy
new file mode 100644
index 00000000000..1d765497c44
--- /dev/null
+++ b/regression-test/suites/function_p0/cast/to_complex/test_cast_complex_null_covered_payload.groovy
@@ -0,0 +1,248 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_cast_complex_null_covered_payload") {
+
+ // Reproducer: IF/NULLIF keep the value of the branch that was not taken as a hidden payload of
+ // the rows they mark as NULL. Every child of such a row is NULL by SQL semantics, so a strict
+ // cast must not validate it, and the NULL mask of the row has to be expanded to the children
+ // that belong to it: ARRAY elements, nested ARRAY elements and MAP keys/values are stored
+ // flattened, while STRUCT fields keep the rows of their parent.
+ sql "drop table if exists test_cast_complex_null_covered_payload;"
+ sql """
+ create table test_cast_complex_null_covered_payload (
+ id int,
+ p boolean not null,
+ x int not null,
+ arr array<int> not null,
+ nested array<array<int>> not null,
+ m map<int, int> not null,
+ s struct<f1:int> not null
+ )
+ duplicate key(id)
+ distributed by hash(id) buckets 1
+ properties("replication_num"="1");
+ """
+ sql """
+ insert into test_cast_complex_null_covered_payload values
+ (1, false, 128, [128], [[128]], {1: 128}, {128}),
+ (2, true, 7, [1, 2], [[1, 2]], {1: 10, 2: 20}, {7});
+ """
+
+ // Used to verify that a genuinely non NULL out of range child is still rejected.
+ sql "drop table if exists test_cast_complex_null_covered_payload_overflow;"
+ sql """
+ create table test_cast_complex_null_covered_payload_overflow (
+ id int,
+ arr array<int> not null,
+ m map<int, int> not null,
+ s struct<f1:int> not null
+ )
+ duplicate key(id)
+ distributed by hash(id) buckets 1
+ properties("replication_num"="1");
+ """
+ sql """
+ insert into test_cast_complex_null_covered_payload_overflow values
+ (1, [300], {1: 300}, {300});
+ """
+
+ // The hidden payload of a string element cannot be replaced with a default value, so the mask of
+ // the NULL row has to reach the element cast itself.
+ sql "drop table if exists test_cast_complex_null_covered_payload_str;"
+ sql """
+ create table test_cast_complex_null_covered_payload_str (
+ id int,
+ p boolean not null,
+ arrs array<string> not null
+ )
+ duplicate key(id)
+ distributed by hash(id) buckets 1
+ properties("replication_num"="1");
+ """
+ sql """
+ insert into test_cast_complex_null_covered_payload_str values
+ (1, false, ["abc"]),
+ (2, true, ["1", "2"]);
+ """
+
+ // ---------- short_circuit_evaluation=false ----------
+ sql "set short_circuit_evaluation=false;"
+
+ sql "set enable_strict_cast=true;"
+ order_qt_array_strict_sc_false """
+ select id, cast(if(p, arr, null) as array<tinyint>) as r
+ from test_cast_complex_null_covered_payload order by id;
+ """
+ sql "set enable_strict_cast=false;"
+ order_qt_array_non_strict_sc_false """
+ select id, cast(if(p, arr, null) as array<tinyint>) as r
+ from test_cast_complex_null_covered_payload order by id;
+ """
+
+ sql "set enable_strict_cast=true;"
+ order_qt_nullif_array_strict_sc_false """
+ select id, cast(nullif(arr, [128]) as array<tinyint>) as r
+ from test_cast_complex_null_covered_payload order by id;
+ """
+ sql "set enable_strict_cast=false;"
+ order_qt_nullif_array_non_strict_sc_false """
+ select id, cast(nullif(arr, [128]) as array<tinyint>) as r
+ from test_cast_complex_null_covered_payload order by id;
+ """
+
+ sql "set enable_strict_cast=true;"
+ order_qt_array_function_strict_sc_false """
+ select id, cast(if(p, array(x), null) as array<tinyint>) as r
+ from test_cast_complex_null_covered_payload order by id;
+ """
+ sql "set enable_strict_cast=false;"
+ order_qt_array_function_non_strict_sc_false """
+ select id, cast(if(p, array(x), null) as array<tinyint>) as r
+ from test_cast_complex_null_covered_payload order by id;
+ """
+
+ sql "set enable_strict_cast=true;"
+ order_qt_nested_array_strict_sc_false """
+ select id, cast(if(p, nested, null) as array<array<tinyint>>) as r
+ from test_cast_complex_null_covered_payload order by id;
+ """
+ sql "set enable_strict_cast=false;"
+ order_qt_nested_array_non_strict_sc_false """
+ select id, cast(if(p, nested, null) as array<array<tinyint>>) as r
+ from test_cast_complex_null_covered_payload order by id;
+ """
+
+ sql "set enable_strict_cast=true;"
+ order_qt_map_value_strict_sc_false """
+ select id, cast(if(p, m, null) as map<int, tinyint>) as r
+ from test_cast_complex_null_covered_payload order by id;
+ """
+ sql "set enable_strict_cast=false;"
+ order_qt_map_value_non_strict_sc_false """
+ select id, cast(if(p, m, null) as map<int, tinyint>) as r
+ from test_cast_complex_null_covered_payload order by id;
+ """
+
+ // The keys of a map are stored flattened as well, so the NULL of a map row must not be applied
+ // to the entries of another row.
+ sql "set enable_strict_cast=true;"
+ order_qt_map_key_value_strict_sc_false """
+ select id, cast(if(p, m, null) as map<tinyint, tinyint>) as r
+ from test_cast_complex_null_covered_payload order by id;
+ """
+ sql "set enable_strict_cast=false;"
+ order_qt_map_key_value_non_strict_sc_false """
+ select id, cast(if(p, m, null) as map<tinyint, tinyint>) as r
+ from test_cast_complex_null_covered_payload order by id;
+ """
+
+ sql "set enable_strict_cast=true;"
+ order_qt_map_function_strict_sc_false """
+ select id, cast(if(p, map(x, x), null) as map<tinyint, tinyint>) as r
+ from test_cast_complex_null_covered_payload order by id;
+ """
+ sql "set enable_strict_cast=false;"
+ order_qt_map_function_non_strict_sc_false """
+ select id, cast(if(p, map(x, x), null) as map<tinyint, tinyint>) as r
+ from test_cast_complex_null_covered_payload order by id;
+ """
+
+ sql "set enable_strict_cast=true;"
+ order_qt_struct_strict_sc_false """
+ select id, cast(if(p, s, null) as struct<f1:tinyint>) as r
+ from test_cast_complex_null_covered_payload order by id;
+ """
+ sql "set enable_strict_cast=false;"
+ order_qt_struct_non_strict_sc_false """
+ select id, cast(if(p, s, null) as struct<f1:tinyint>) as r
+ from test_cast_complex_null_covered_payload order by id;
+ """
+
+ sql "set enable_strict_cast=true;"
+ order_qt_string_array_strict_sc_false """
+ select id, cast(if(p, arrs, null) as array<tinyint>) as r
+ from test_cast_complex_null_covered_payload_str order by id;
+ """
+ sql "set enable_strict_cast=false;"
+ order_qt_string_array_non_strict_sc_false """
+ select id, cast(if(p, arrs, null) as array<tinyint>) as r
+ from test_cast_complex_null_covered_payload_str order by id;
+ """
+ sql "set enable_strict_cast=true;"
+ order_qt_string_array_nullif_strict_sc_false """
+ select id, cast(nullif(arrs, ["abc"]) as array<tinyint>) as r
+ from test_cast_complex_null_covered_payload_str order by id;
+ """
+
+ // ---------- short_circuit_evaluation=true ----------
+ sql "set short_circuit_evaluation=true;"
+
+ sql "set enable_strict_cast=true;"
+ order_qt_array_strict_sc_true """
+ select id, cast(if(p, arr, null) as array<tinyint>) as r
+ from test_cast_complex_null_covered_payload order by id;
+ """
+ sql "set enable_strict_cast=false;"
+ order_qt_array_non_strict_sc_true """
+ select id, cast(if(p, arr, null) as array<tinyint>) as r
+ from test_cast_complex_null_covered_payload order by id;
+ """
+
+ sql "set enable_strict_cast=true;"
+ order_qt_nested_array_strict_sc_true """
+ select id, cast(if(p, nested, null) as array<array<tinyint>>) as r
+ from test_cast_complex_null_covered_payload order by id;
+ """
+
+ sql "set enable_strict_cast=true;"
+ order_qt_map_key_value_strict_sc_true """
+ select id, cast(if(p, m, null) as map<tinyint, tinyint>) as r
+ from test_cast_complex_null_covered_payload order by id;
+ """
+
+ sql "set enable_strict_cast=true;"
+ order_qt_struct_strict_sc_true """
+ select id, cast(if(p, s, null) as struct<f1:tinyint>) as r
+ from test_cast_complex_null_covered_payload order by id;
+ """
+
+ // A genuinely non NULL out of range child must still fail in strict mode.
+ sql "set short_circuit_evaluation=false;"
+ sql "set enable_strict_cast=true;"
+ test {
+ sql """
+ select cast(arr as array<tinyint>) as r
+ from test_cast_complex_null_covered_payload_overflow where id = 1;
+ """
+ exception "Value 300 out of range for type tinyint"
+ }
+ test {
+ sql """
+ select cast(m as map<int, tinyint>) as r
+ from test_cast_complex_null_covered_payload_overflow where id = 1;
+ """
+ exception "Value 300 out of range for type tinyint"
+ }
+ test {
+ sql """
+ select cast(s as struct<f1:tinyint>) as r
+ from test_cast_complex_null_covered_payload_overflow where id = 1;
+ """
+ exception "Value 300 out of range for type tinyint"
+ }
+}
diff --git a/regression-test/suites/function_p0/cast/to_int/from_int/test_cast_to_tinyint_from_int_nullable_null_covered_payload.groovy b/regression-test/suites/function_p0/cast/to_int/from_int/test_cast_to_tinyint_from_int_nullable_null_covered_payload.groovy
new file mode 100644
index 00000000000..4adb607cbae
--- /dev/null
+++ b/regression-test/suites/function_p0/cast/to_int/from_int/test_cast_to_tinyint_from_int_nullable_null_covered_payload.groovy
@@ -0,0 +1,153 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_cast_to_tinyint_from_int_nullable_null_covered_payload") {
+
+ // Reproducer: row 1 keeps 128 as the hidden nested payload of a Nullable(INT) produced by
+ // IF/NULLIF, while the null map marks it as NULL. Strict cast must skip such NULL rows.
+ sql "drop table if exists test_cast_null_covered_payload;"
+ sql """
+ create table test_cast_null_covered_payload (
+ id int,
+ p boolean not null,
+ x int not null,
+ y int not null
+ )
+ duplicate key(id)
+ distributed by hash(id) buckets 1
+ properties("replication_num"="1");
+ """
+ sql """
+ insert into test_cast_null_covered_payload values
+ (1, false, 128, 128),
+ (2, true, 1, 2);
+ """
+
+ // Used to verify that a genuinely non NULL out-of-range value is still rejected.
+ sql "drop table if exists test_cast_null_covered_payload_overflow;"
+ sql """
+ create table test_cast_null_covered_payload_overflow (
+ id int,
+ x int not null
+ )
+ duplicate key(id)
+ distributed by hash(id) buckets 1
+ properties("replication_num"="1");
+ """
+ sql "insert into test_cast_null_covered_payload_overflow values (1, 128);"
+
+ // ---------- short_circuit_evaluation=false ----------
+ sql "set short_circuit_evaluation=false;"
+
+ sql "set enable_strict_cast=true;"
+ order_qt_if_strict_sc_false """
+ select id, cast(if(p, x, null) as tinyint) as r
+ from test_cast_null_covered_payload order by id;
+ """
+ sql "set enable_strict_cast=false;"
+ order_qt_if_non_strict_sc_false """
+ select id, cast(if(p, x, null) as tinyint) as r
+ from test_cast_null_covered_payload order by id;
+ """
+
+ sql "set enable_strict_cast=true;"
+ order_qt_nullif_strict_sc_false """
+ select id, cast(nullif(x, y) as tinyint) as r
+ from test_cast_null_covered_payload order by id;
+ """
+ sql "set enable_strict_cast=false;"
+ order_qt_nullif_non_strict_sc_false """
+ select id, cast(nullif(x, y) as tinyint) as r
+ from test_cast_null_covered_payload order by id;
+ """
+
+ // ---------- short_circuit_evaluation=true ----------
+ sql "set short_circuit_evaluation=true;"
+
+ sql "set enable_strict_cast=true;"
+ order_qt_if_strict_sc_true """
+ select id, cast(if(p, x, null) as tinyint) as r
+ from test_cast_null_covered_payload order by id;
+ """
+ sql "set enable_strict_cast=false;"
+ order_qt_if_non_strict_sc_true """
+ select id, cast(if(p, x, null) as tinyint) as r
+ from test_cast_null_covered_payload order by id;
+ """
+
+ sql "set enable_strict_cast=true;"
+ order_qt_nullif_strict_sc_true """
+ select id, cast(nullif(x, y) as tinyint) as r
+ from test_cast_null_covered_payload order by id;
+ """
+ sql "set enable_strict_cast=false;"
+ order_qt_nullif_non_strict_sc_true """
+ select id, cast(nullif(x, y) as tinyint) as r
+ from test_cast_null_covered_payload order by id;
+ """
+
+ // Number to date/timestamp_ns casts parse the source value with a strict-mode serde batch that
+ // cannot skip rows, so the hidden payload of a NULL row must not fail the query either.
+ sql "drop table if exists test_cast_null_covered_payload_datetime;"
+ sql """
+ create table test_cast_null_covered_payload_datetime (
+ id int,
+ p boolean not null,
+ big bigint not null
+ )
+ duplicate key(id)
+ distributed by hash(id) buckets 1
+ properties("replication_num"="1");
+ """
+ sql """
+ insert into test_cast_null_covered_payload_datetime values
+ (1, false, 1000),
+ (2, true, 20150102030405);
+ """
+
+ sql "set enable_strict_cast=true;"
+ order_qt_date_strict """
+ select id, cast(if(p, big, null) as date) as r
+ from test_cast_null_covered_payload_datetime order by id;
+ """
+ sql "set enable_strict_cast=false;"
+ order_qt_date_non_strict """
+ select id, cast(if(p, big, null) as date) as r
+ from test_cast_null_covered_payload_datetime order by id;
+ """
+ sql "set enable_strict_cast=true;"
+ order_qt_timestamp_ns_strict """
+ select id, cast(if(p, big, null) as timestamp_ns) as r
+ from test_cast_null_covered_payload_datetime order by id;
+ """
+ sql "set enable_strict_cast=false;"
+ order_qt_timestamp_ns_non_strict """
+ select id, cast(if(p, big, null) as timestamp_ns) as r
+ from test_cast_null_covered_payload_datetime order by id;
+ """
+
+ // A genuinely non NULL out-of-range value must still fail in strict mode.
+ sql "set short_circuit_evaluation=false;"
+ sql "set enable_strict_cast=true;"
+ test {
+ sql """
+ select cast(x as tinyint)
+ from test_cast_null_covered_payload_overflow where id = 1;
+ """
+ exception "Value 128 out of range for type tinyint"
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org