You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by GitBox <gi...@apache.org> on 2022/05/09 08:20:59 UTC
[GitHub] [incubator-doris] adonis0147 opened a new pull request, #9466: [feature-wip][array-type] Support more sub types.
adonis0147 opened a new pull request, #9466:
URL: https://github.com/apache/incubator-doris/pull/9466
# Proposed changes
Issue Number: close #9465
## Problem Summary:
Please refer to #9465 .
## Checklist(Required)
1. Does it affect the original behavior: No
2. Has unit tests been added: Yes
3. Has document been added or modified: No
4. Does it need to update dependencies: No
5. Are there any changes that cannot be rolled back: No
## Further comments
If this is a relatively large or complex change, kick off the discussion at [dev@doris.apache.org](mailto:dev@doris.apache.org) by explaining why you chose the solution you did and what alternatives you considered, etc...
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] cambyzju commented on a diff in pull request #9466: [feature-wip][array-type] Support more sub types.
Posted by GitBox <gi...@apache.org>.
cambyzju commented on code in PR #9466:
URL: https://github.com/apache/incubator-doris/pull/9466#discussion_r871918283
##########
be/src/util/array_parser.h:
##########
@@ -175,6 +202,37 @@ class ArrayParser {
memory_copy(string_val->ptr, iterator->GetString(), iterator->GetStringLength());
break;
}
+ case TYPE_DATE:
+ case TYPE_DATETIME: {
+ DateTimeValue value;
+ value.from_date_str(iterator->GetString(), iterator->GetStringLength());
+ *val = reinterpret_cast<AnyVal*>(context->allocate(sizeof(DateTimeVal)));
+ new (*val) DateTimeVal();
+ value.to_datetime_val(static_cast<DateTimeVal*>(*val));
+ break;
+ }
+ case TYPE_DECIMALV2: {
+ *val = reinterpret_cast<AnyVal*>(context->allocate(sizeof(DecimalV2Val)));
+ new (*val) DecimalV2Val();
+
+ if (iterator->IsNumber()) {
+ if (iterator->IsInt() || iterator->IsUint() || iterator->IsInt64()) {
Review Comment:
does IsInt32 better than IsInt, and IsUint32 better than IsUint? At the first look, IsInt == IsInt32 || IsInt64.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] adonis0147 commented on a diff in pull request #9466: [feature-wip][array-type] Support more sub types.
Posted by GitBox <gi...@apache.org>.
adonis0147 commented on code in PR #9466:
URL: https://github.com/apache/incubator-doris/pull/9466#discussion_r873543666
##########
be/src/vec/olap/olap_data_convertor.cpp:
##########
@@ -700,4 +621,55 @@ Status OlapBlockDataConvertor::OlapColumnDataConvertorDecimal::convert_to_olap()
return Status::OK();
}
-} // namespace doris::vectorized
\ No newline at end of file
+Status OlapBlockDataConvertor::OlapColumnDataConvertorArray::convert_to_olap() {
+ const ColumnArray* column_array = nullptr;
+ const DataTypeArray* data_type_ptr_array = nullptr;
+ if (_nullmap) {
+ const auto* nullable_column =
+ assert_cast<const ColumnNullable*>(_typed_column.column.get());
+ column_array =
+ assert_cast<const ColumnArray*>(nullable_column->get_nested_column_ptr().get());
+ data_type_ptr_array = assert_cast<const DataTypeArray*>(
+ (assert_cast<const DataTypeNullable*>(_typed_column.type.get())->get_nested_type())
+ .get());
+ } else {
+ column_array = assert_cast<const ColumnArray*>(_typed_column.column.get());
+ data_type_ptr_array = assert_cast<const DataTypeArray*>(_typed_column.type.get());
+ }
+ assert(column_array);
+ assert(data_type_ptr_array);
+
+ CollectionValue* collection_value = _values.data();
+ for (int i = 0; i < _num_rows; ++i, ++collection_value) {
+ int64_t cur_pos = _row_pos + i;
+ int64_t prev_pos = cur_pos - 1;
+ if (_nullmap && _nullmap[cur_pos]) {
+ continue;
+ }
+ const auto& offsets = column_array->get_offsets();
Review Comment:
Refined.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] eldenmoon commented on a diff in pull request #9466: [feature-wip][array-type] Support more sub types.
Posted by GitBox <gi...@apache.org>.
eldenmoon commented on code in PR #9466:
URL: https://github.com/apache/incubator-doris/pull/9466#discussion_r894357886
##########
be/src/vec/olap/olap_data_convertor.cpp:
##########
@@ -700,4 +621,55 @@ Status OlapBlockDataConvertor::OlapColumnDataConvertorDecimal::convert_to_olap()
return Status::OK();
}
-} // namespace doris::vectorized
\ No newline at end of file
+Status OlapBlockDataConvertor::OlapColumnDataConvertorArray::convert_to_olap() {
+ const ColumnArray* column_array = nullptr;
+ const DataTypeArray* data_type_ptr_array = nullptr;
+ if (_nullmap) {
+ const auto* nullable_column =
+ assert_cast<const ColumnNullable*>(_typed_column.column.get());
+ column_array =
+ assert_cast<const ColumnArray*>(nullable_column->get_nested_column_ptr().get());
+ data_type_ptr_array = assert_cast<const DataTypeArray*>(
+ (assert_cast<const DataTypeNullable*>(_typed_column.type.get())->get_nested_type())
+ .get());
+ } else {
+ column_array = assert_cast<const ColumnArray*>(_typed_column.column.get());
+ data_type_ptr_array = assert_cast<const DataTypeArray*>(_typed_column.type.get());
+ }
+ assert(column_array);
+ assert(data_type_ptr_array);
+
+ const auto& offsets = column_array->get_offsets();
+ CollectionValue* collection_value = _values.data();
+ for (int i = 0; i < _num_rows; ++i, ++collection_value) {
+ int64_t cur_pos = _row_pos + i;
+ int64_t prev_pos = cur_pos - 1;
+ if (_nullmap && _nullmap[cur_pos]) {
+ continue;
+ }
+ auto offset = offsets[prev_pos];
+ auto size = offsets[cur_pos] - offsets[prev_pos];
+ new (collection_value) CollectionValue(size);
+
+ if (size == 0) {
+ continue;
+ }
+
+ if (column_array->get_data().is_nullable()) {
+ const auto& data_nullable_column =
+ assert_cast<const ColumnNullable&>(column_array->get_data());
+ const auto* data_null_map = data_nullable_column.get_null_map_data().data();
+ collection_value->set_has_null(true);
+ collection_value->set_null_signs(
+ const_cast<bool*>(reinterpret_cast<const bool*>(data_null_map + offset)));
+ }
+ ColumnWithTypeAndName item_typed_column = {column_array->get_data_ptr(),
+ data_type_ptr_array->get_nested_type(), ""};
+ _item_convertor->set_source_column(item_typed_column, offset, size);
Review Comment:
the same as array, hll or bitmap
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] adonis0147 commented on a diff in pull request #9466: [feature-wip][array-type] Support more sub types.
Posted by GitBox <gi...@apache.org>.
adonis0147 commented on code in PR #9466:
URL: https://github.com/apache/incubator-doris/pull/9466#discussion_r873479667
##########
be/src/util/array_parser.h:
##########
@@ -175,6 +202,37 @@ class ArrayParser {
memory_copy(string_val->ptr, iterator->GetString(), iterator->GetStringLength());
break;
}
+ case TYPE_DATE:
+ case TYPE_DATETIME: {
+ DateTimeValue value;
+ value.from_date_str(iterator->GetString(), iterator->GetStringLength());
+ *val = reinterpret_cast<AnyVal*>(context->allocate(sizeof(DateTimeVal)));
+ new (*val) DateTimeVal();
+ value.to_datetime_val(static_cast<DateTimeVal*>(*val));
+ break;
+ }
+ case TYPE_DECIMALV2: {
+ *val = reinterpret_cast<AnyVal*>(context->allocate(sizeof(DecimalV2Val)));
+ new (*val) DecimalV2Val();
+
+ if (iterator->IsNumber()) {
+ if (iterator->IsInt() || iterator->IsUint() || iterator->IsInt64()) {
Review Comment:
After refinement, it is sufficient to check whether the iterator is `Uint64` here.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] github-actions[bot] commented on pull request #9466: [feature-wip][array-type] Support more sub types.
Posted by GitBox <gi...@apache.org>.
github-actions[bot] commented on PR #9466:
URL: https://github.com/apache/incubator-doris/pull/9466#issuecomment-1135750458
PR approved by anyone and no changes requested.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] adonis0147 commented on a diff in pull request #9466: [feature-wip][array-type] Support more sub types.
Posted by GitBox <gi...@apache.org>.
adonis0147 commented on code in PR #9466:
URL: https://github.com/apache/incubator-doris/pull/9466#discussion_r873471858
##########
be/src/runtime/collection_value.h:
##########
@@ -137,45 +141,110 @@ struct CollectionValue {
friend ArrayIterator;
};
-/**
- * Array's Iterator, support read array by special type
- */
class ArrayIterator {
-private:
- ArrayIterator(PrimitiveType children_type, const CollectionValue* data);
-
public:
- bool seek(uint32_t n) {
- if (n >= _data->size()) {
+ int type_size() const { return _type_size; }
+ bool is_type_fixed_width() const { return _is_type_fixed_width; }
+
+ bool has_next() const { return _offset < _collection_value->size(); }
+ bool next() const {
+ if (has_next()) {
+ ++_offset;
+ return true;
+ }
+ return false;
+ }
+ bool seek(int n) const {
+ if (n >= _collection_value->size()) {
Review Comment:
Fixed.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] eldenmoon commented on a diff in pull request #9466: [feature-wip][array-type] Support more sub types.
Posted by GitBox <gi...@apache.org>.
eldenmoon commented on code in PR #9466:
URL: https://github.com/apache/incubator-doris/pull/9466#discussion_r894355137
##########
be/src/vec/olap/olap_data_convertor.cpp:
##########
@@ -700,4 +621,55 @@ Status OlapBlockDataConvertor::OlapColumnDataConvertorDecimal::convert_to_olap()
return Status::OK();
}
-} // namespace doris::vectorized
\ No newline at end of file
+Status OlapBlockDataConvertor::OlapColumnDataConvertorArray::convert_to_olap() {
+ const ColumnArray* column_array = nullptr;
+ const DataTypeArray* data_type_ptr_array = nullptr;
+ if (_nullmap) {
+ const auto* nullable_column =
+ assert_cast<const ColumnNullable*>(_typed_column.column.get());
+ column_array =
+ assert_cast<const ColumnArray*>(nullable_column->get_nested_column_ptr().get());
+ data_type_ptr_array = assert_cast<const DataTypeArray*>(
+ (assert_cast<const DataTypeNullable*>(_typed_column.type.get())->get_nested_type())
+ .get());
+ } else {
+ column_array = assert_cast<const ColumnArray*>(_typed_column.column.get());
+ data_type_ptr_array = assert_cast<const DataTypeArray*>(_typed_column.type.get());
+ }
+ assert(column_array);
+ assert(data_type_ptr_array);
+
+ const auto& offsets = column_array->get_offsets();
+ CollectionValue* collection_value = _values.data();
+ for (int i = 0; i < _num_rows; ++i, ++collection_value) {
+ int64_t cur_pos = _row_pos + i;
+ int64_t prev_pos = cur_pos - 1;
+ if (_nullmap && _nullmap[cur_pos]) {
+ continue;
+ }
+ auto offset = offsets[prev_pos];
+ auto size = offsets[cur_pos] - offsets[prev_pos];
+ new (collection_value) CollectionValue(size);
+
+ if (size == 0) {
+ continue;
+ }
+
+ if (column_array->get_data().is_nullable()) {
+ const auto& data_nullable_column =
+ assert_cast<const ColumnNullable&>(column_array->get_data());
+ const auto* data_null_map = data_nullable_column.get_null_map_data().data();
+ collection_value->set_has_null(true);
+ collection_value->set_null_signs(
+ const_cast<bool*>(reinterpret_cast<const bool*>(data_null_map + offset)));
+ }
+ ColumnWithTypeAndName item_typed_column = {column_array->get_data_ptr(),
+ data_type_ptr_array->get_nested_type(), ""};
+ _item_convertor->set_source_column(item_typed_column, offset, size);
Review Comment:
if item_type is or varchar or string, `set_source_column` will `resize()` the `_item_convertor._slice` and the slices from the previous `convert_to_olap()` will be freed, and then lead to heap use after free in the subsequent use of the collection_value
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] adonis0147 commented on a diff in pull request #9466: [feature-wip][array-type] Support more sub types.
Posted by GitBox <gi...@apache.org>.
adonis0147 commented on code in PR #9466:
URL: https://github.com/apache/incubator-doris/pull/9466#discussion_r871535628
##########
be/src/vec/core/block.cpp:
##########
@@ -768,20 +768,38 @@ void Block::deep_copy_slot(void* dst, MemPool* pool, const doris::TypeDescriptor
}
auto item_column = array_column->get_data_ptr().get();
auto offset = array_column->get_offsets()[row - 1];
+ auto iterator = collection_value->iterator(item_type_desc.type);
for (int i = 0; i < collection_value->length(); ++i) {
- char* item_dst = reinterpret_cast<char*>(collection_value->mutable_data()) +
- i * item_type_desc.get_slot_size();
if (array[i].is_null()) {
const auto& null_value = doris_udf::AnyVal(true);
- collection_value->set(i, item_type_desc.type, &null_value);
+ iterator.set(&null_value);
} else {
auto item_offset = offset + i;
const auto& data_ref = item_type_desc.type != TYPE_ARRAY
? item_column->get_data_at(item_offset)
: StringRef();
- deep_copy_slot(item_dst, pool, item_type_desc, data_ref, item_column, item_offset,
- padding_char);
+ if (!item_type_desc.is_date_type() && !item_type_desc.is_decimal_type()) {
+ deep_copy_slot(iterator.get(), pool, item_type_desc, data_ref, item_column,
+ item_offset, padding_char);
+ } else if (item_type_desc.is_date_type()) {
+ // In CollectionValue, date type data is stored as either uint24_t or uint64_t.
+ DateTimeValue datetime_value;
+ deep_copy_slot(&datetime_value, pool, item_type_desc, data_ref, item_column,
+ item_offset, padding_char);
+ DateTimeVal datetime_val;
+ datetime_value.to_datetime_val(&datetime_val);
+ iterator.set(&datetime_val);
+ } else {
Review Comment:
Refined.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] cambyzju commented on a diff in pull request #9466: [feature-wip][array-type] Support more sub types.
Posted by GitBox <gi...@apache.org>.
cambyzju commented on code in PR #9466:
URL: https://github.com/apache/incubator-doris/pull/9466#discussion_r871922956
##########
be/src/runtime/collection_value.h:
##########
@@ -137,45 +141,110 @@ struct CollectionValue {
friend ArrayIterator;
};
-/**
- * Array's Iterator, support read array by special type
- */
class ArrayIterator {
-private:
- ArrayIterator(PrimitiveType children_type, const CollectionValue* data);
-
public:
- bool seek(uint32_t n) {
- if (n >= _data->size()) {
+ int type_size() const { return _type_size; }
+ bool is_type_fixed_width() const { return _is_type_fixed_width; }
+
+ bool has_next() const { return _offset < _collection_value->size(); }
+ bool next() const {
+ if (has_next()) {
+ ++_offset;
+ return true;
+ }
+ return false;
+ }
+ bool seek(int n) const {
+ if (n >= _collection_value->size()) {
Review Comment:
check n < 0?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] cambyzju commented on a diff in pull request #9466: [feature-wip][array-type] Support more sub types.
Posted by GitBox <gi...@apache.org>.
cambyzju commented on code in PR #9466:
URL: https://github.com/apache/incubator-doris/pull/9466#discussion_r871919417
##########
be/src/util/array_parser.h:
##########
@@ -175,6 +202,37 @@ class ArrayParser {
memory_copy(string_val->ptr, iterator->GetString(), iterator->GetStringLength());
break;
}
+ case TYPE_DATE:
+ case TYPE_DATETIME: {
+ DateTimeValue value;
+ value.from_date_str(iterator->GetString(), iterator->GetStringLength());
+ *val = reinterpret_cast<AnyVal*>(context->allocate(sizeof(DateTimeVal)));
+ new (*val) DateTimeVal();
+ value.to_datetime_val(static_cast<DateTimeVal*>(*val));
+ break;
+ }
+ case TYPE_DECIMALV2: {
+ *val = reinterpret_cast<AnyVal*>(context->allocate(sizeof(DecimalV2Val)));
+ new (*val) DecimalV2Val();
+
+ if (iterator->IsNumber()) {
+ if (iterator->IsInt() || iterator->IsUint() || iterator->IsInt64()) {
+ DecimalV2Value(iterator->GetInt64(), 0)
+ .to_decimal_val(static_cast<DecimalV2Val*>(*val));
+ } else if (iterator->IsUint64()) {
+ DecimalV2Value(iterator->GetUint64(), 0)
Review Comment:
Why not use GetUint64 for IsUint?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] jackwener commented on pull request #9466: [feature-wip][array-type] Support more sub types.
Posted by GitBox <gi...@apache.org>.
jackwener commented on PR #9466:
URL: https://github.com/apache/incubator-doris/pull/9466#issuecomment-1132527815
cc @platoneko
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] adonis0147 commented on a diff in pull request #9466: [feature-wip][array-type] Support more sub types.
Posted by GitBox <gi...@apache.org>.
adonis0147 commented on code in PR #9466:
URL: https://github.com/apache/incubator-doris/pull/9466#discussion_r873472738
##########
be/src/util/array_parser.h:
##########
@@ -175,6 +202,37 @@ class ArrayParser {
memory_copy(string_val->ptr, iterator->GetString(), iterator->GetStringLength());
break;
}
+ case TYPE_DATE:
+ case TYPE_DATETIME: {
+ DateTimeValue value;
+ value.from_date_str(iterator->GetString(), iterator->GetStringLength());
+ *val = reinterpret_cast<AnyVal*>(context->allocate(sizeof(DateTimeVal)));
+ new (*val) DateTimeVal();
+ value.to_datetime_val(static_cast<DateTimeVal*>(*val));
+ break;
+ }
+ case TYPE_DECIMALV2: {
+ *val = reinterpret_cast<AnyVal*>(context->allocate(sizeof(DecimalV2Val)));
+ new (*val) DecimalV2Val();
+
+ if (iterator->IsNumber()) {
+ if (iterator->IsInt() || iterator->IsUint() || iterator->IsInt64()) {
+ DecimalV2Value(iterator->GetInt64(), 0)
+ .to_decimal_val(static_cast<DecimalV2Val*>(*val));
+ } else if (iterator->IsUint64()) {
+ DecimalV2Value(iterator->GetUint64(), 0)
Review Comment:
Refined.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] cambyzju commented on a diff in pull request #9466: [feature-wip][array-type] Support more sub types.
Posted by GitBox <gi...@apache.org>.
cambyzju commented on code in PR #9466:
URL: https://github.com/apache/incubator-doris/pull/9466#discussion_r873502452
##########
be/src/vec/olap/olap_data_convertor.cpp:
##########
@@ -700,4 +621,55 @@ Status OlapBlockDataConvertor::OlapColumnDataConvertorDecimal::convert_to_olap()
return Status::OK();
}
-} // namespace doris::vectorized
\ No newline at end of file
+Status OlapBlockDataConvertor::OlapColumnDataConvertorArray::convert_to_olap() {
+ const ColumnArray* column_array = nullptr;
+ const DataTypeArray* data_type_ptr_array = nullptr;
+ if (_nullmap) {
+ const auto* nullable_column =
+ assert_cast<const ColumnNullable*>(_typed_column.column.get());
+ column_array =
+ assert_cast<const ColumnArray*>(nullable_column->get_nested_column_ptr().get());
+ data_type_ptr_array = assert_cast<const DataTypeArray*>(
+ (assert_cast<const DataTypeNullable*>(_typed_column.type.get())->get_nested_type())
+ .get());
+ } else {
+ column_array = assert_cast<const ColumnArray*>(_typed_column.column.get());
+ data_type_ptr_array = assert_cast<const DataTypeArray*>(_typed_column.type.get());
+ }
+ assert(column_array);
+ assert(data_type_ptr_array);
+
+ CollectionValue* collection_value = _values.data();
+ for (int i = 0; i < _num_rows; ++i, ++collection_value) {
+ int64_t cur_pos = _row_pos + i;
+ int64_t prev_pos = cur_pos - 1;
+ if (_nullmap && _nullmap[cur_pos]) {
+ continue;
+ }
+ const auto& offsets = column_array->get_offsets();
Review Comment:
could put this line outside the for loop.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] cambyzju commented on pull request #9466: [feature-wip][array-type] Support more sub types.
Posted by GitBox <gi...@apache.org>.
cambyzju commented on PR #9466:
URL: https://github.com/apache/incubator-doris/pull/9466#issuecomment-1127427750
LGTM
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] github-actions[bot] commented on pull request #9466: [feature-wip][array-type] Support more sub types.
Posted by GitBox <gi...@apache.org>.
github-actions[bot] commented on PR #9466:
URL: https://github.com/apache/incubator-doris/pull/9466#issuecomment-1135750384
PR approved by at least one committer and no changes requested.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] morningman merged pull request #9466: [feature-wip][array-type] Support more sub types.
Posted by GitBox <gi...@apache.org>.
morningman merged PR #9466:
URL: https://github.com/apache/incubator-doris/pull/9466
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] cambyzju commented on a diff in pull request #9466: [feature-wip][array-type] Support more sub types.
Posted by GitBox <gi...@apache.org>.
cambyzju commented on code in PR #9466:
URL: https://github.com/apache/incubator-doris/pull/9466#discussion_r870911684
##########
be/src/vec/core/block.cpp:
##########
@@ -768,20 +768,38 @@ void Block::deep_copy_slot(void* dst, MemPool* pool, const doris::TypeDescriptor
}
auto item_column = array_column->get_data_ptr().get();
auto offset = array_column->get_offsets()[row - 1];
+ auto iterator = collection_value->iterator(item_type_desc.type);
for (int i = 0; i < collection_value->length(); ++i) {
- char* item_dst = reinterpret_cast<char*>(collection_value->mutable_data()) +
- i * item_type_desc.get_slot_size();
if (array[i].is_null()) {
const auto& null_value = doris_udf::AnyVal(true);
- collection_value->set(i, item_type_desc.type, &null_value);
+ iterator.set(&null_value);
} else {
auto item_offset = offset + i;
const auto& data_ref = item_type_desc.type != TYPE_ARRAY
? item_column->get_data_at(item_offset)
: StringRef();
- deep_copy_slot(item_dst, pool, item_type_desc, data_ref, item_column, item_offset,
- padding_char);
+ if (!item_type_desc.is_date_type() && !item_type_desc.is_decimal_type()) {
+ deep_copy_slot(iterator.get(), pool, item_type_desc, data_ref, item_column,
+ item_offset, padding_char);
+ } else if (item_type_desc.is_date_type()) {
+ // In CollectionValue, date type data is stored as either uint24_t or uint64_t.
+ DateTimeValue datetime_value;
+ deep_copy_slot(&datetime_value, pool, item_type_desc, data_ref, item_column,
+ item_offset, padding_char);
+ DateTimeVal datetime_val;
+ datetime_value.to_datetime_val(&datetime_val);
+ iterator.set(&datetime_val);
+ } else {
Review Comment:
it is better use: if (is_data_type); else if (is_decimal_type); else (others);
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [incubator-doris] adonis0147 commented on a diff in pull request #9466: [feature-wip][array-type] Support more sub types.
Posted by GitBox <gi...@apache.org>.
adonis0147 commented on code in PR #9466:
URL: https://github.com/apache/incubator-doris/pull/9466#discussion_r896631651
##########
be/src/vec/olap/olap_data_convertor.cpp:
##########
@@ -700,4 +621,55 @@ Status OlapBlockDataConvertor::OlapColumnDataConvertorDecimal::convert_to_olap()
return Status::OK();
}
-} // namespace doris::vectorized
\ No newline at end of file
+Status OlapBlockDataConvertor::OlapColumnDataConvertorArray::convert_to_olap() {
+ const ColumnArray* column_array = nullptr;
+ const DataTypeArray* data_type_ptr_array = nullptr;
+ if (_nullmap) {
+ const auto* nullable_column =
+ assert_cast<const ColumnNullable*>(_typed_column.column.get());
+ column_array =
+ assert_cast<const ColumnArray*>(nullable_column->get_nested_column_ptr().get());
+ data_type_ptr_array = assert_cast<const DataTypeArray*>(
+ (assert_cast<const DataTypeNullable*>(_typed_column.type.get())->get_nested_type())
+ .get());
+ } else {
+ column_array = assert_cast<const ColumnArray*>(_typed_column.column.get());
+ data_type_ptr_array = assert_cast<const DataTypeArray*>(_typed_column.type.get());
+ }
+ assert(column_array);
+ assert(data_type_ptr_array);
+
+ const auto& offsets = column_array->get_offsets();
+ CollectionValue* collection_value = _values.data();
+ for (int i = 0; i < _num_rows; ++i, ++collection_value) {
+ int64_t cur_pos = _row_pos + i;
+ int64_t prev_pos = cur_pos - 1;
+ if (_nullmap && _nullmap[cur_pos]) {
+ continue;
+ }
+ auto offset = offsets[prev_pos];
+ auto size = offsets[cur_pos] - offsets[prev_pos];
+ new (collection_value) CollectionValue(size);
+
+ if (size == 0) {
+ continue;
+ }
+
+ if (column_array->get_data().is_nullable()) {
+ const auto& data_nullable_column =
+ assert_cast<const ColumnNullable&>(column_array->get_data());
+ const auto* data_null_map = data_nullable_column.get_null_map_data().data();
+ collection_value->set_has_null(true);
+ collection_value->set_null_signs(
+ const_cast<bool*>(reinterpret_cast<const bool*>(data_null_map + offset)));
+ }
+ ColumnWithTypeAndName item_typed_column = {column_array->get_data_ptr(),
+ data_type_ptr_array->get_nested_type(), ""};
+ _item_convertor->set_source_column(item_typed_column, offset, size);
Review Comment:
Hi @eldenmoon , you are right.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org