You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by da...@apache.org on 2022/07/16 06:18:45 UTC
[doris] branch master updated: [Bug] handle null map right in vectorized load (#10883)
This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 75ca21dafa [Bug] handle null map right in vectorized load (#10883)
75ca21dafa is described below
commit 75ca21dafaeb197f31990a28e427135f6a9eb2c5
Author: Gabriel <ga...@gmail.com>
AuthorDate: Sat Jul 16 14:18:38 2022 +0800
[Bug] handle null map right in vectorized load (#10883)
---
be/src/vec/olap/olap_data_convertor.cpp | 44 ++++++++++++++++-----------------
be/src/vec/olap/olap_data_convertor.h | 3 +--
2 files changed, 23 insertions(+), 24 deletions(-)
diff --git a/be/src/vec/olap/olap_data_convertor.cpp b/be/src/vec/olap/olap_data_convertor.cpp
index 2bf619dece..a7238c64f3 100644
--- a/be/src/vec/olap/olap_data_convertor.cpp
+++ b/be/src/vec/olap/olap_data_convertor.cpp
@@ -148,7 +148,6 @@ void OlapBlockDataConvertor::OlapColumnDataConvertorBase::set_source_column(
auto nullable_column =
assert_cast<const vectorized::ColumnNullable*>(_typed_column.column.get());
_nullmap = nullable_column->get_null_map_data().data();
- _nullmap += row_pos;
}
}
@@ -157,9 +156,11 @@ void OlapBlockDataConvertor::OlapColumnDataConvertorBase::clear_source_column()
_typed_column.column = nullptr;
}
+// This should be called only in SegmentWriter. If you want to access nullmap in Convertor,
+// use `_nullmap` directly.
const UInt8* OlapBlockDataConvertor::OlapColumnDataConvertorBase::get_nullmap() const {
assert(_typed_column.column);
- return _nullmap;
+ return _nullmap + _row_pos;
}
// class OlapBlockDataConvertor::OlapColumnDataConvertorObject
@@ -177,7 +178,6 @@ const void* OlapBlockDataConvertor::OlapColumnDataConvertorObject::get_data() co
const void* OlapBlockDataConvertor::OlapColumnDataConvertorObject::get_data_at(
size_t offset) const {
- assert(offset < _num_rows && _num_rows == _slice.size());
UInt8 null_flag = 0;
if (_nullmap) {
null_flag = _nullmap[offset];
@@ -205,7 +205,7 @@ Status OlapBlockDataConvertor::OlapColumnDataConvertorBitMap::convert_to_olap()
size_t total_size = 0;
if (_nullmap) {
- const UInt8* nullmap_cur = _nullmap;
+ const UInt8* nullmap_cur = _nullmap + _row_pos;
while (bitmap_value_cur != bitmap_value_end) {
if (!*nullmap_cur) {
total_size += bitmap_value_cur->getSizeInBytes();
@@ -226,7 +226,7 @@ Status OlapBlockDataConvertor::OlapColumnDataConvertorBitMap::convert_to_olap()
char* raw_data = _raw_data.data();
Slice* slice = _slice.data();
if (_nullmap) {
- const UInt8* nullmap_cur = _nullmap;
+ const UInt8* nullmap_cur = _nullmap + _row_pos;
while (bitmap_value_cur != bitmap_value_end) {
if (!*nullmap_cur) {
slice_size = bitmap_value_cur->getSizeInBytes();
@@ -244,7 +244,7 @@ Status OlapBlockDataConvertor::OlapColumnDataConvertorBitMap::convert_to_olap()
++nullmap_cur;
++bitmap_value_cur;
}
- assert(nullmap_cur == _nullmap + _num_rows && slice == _slice.get_end_ptr());
+ assert(nullmap_cur == _nullmap + _row_pos + _num_rows && slice == _slice.get_end_ptr());
} else {
while (bitmap_value_cur != bitmap_value_end) {
slice_size = bitmap_value_cur->getSizeInBytes();
@@ -281,7 +281,7 @@ Status OlapBlockDataConvertor::OlapColumnDataConvertorHLL::convert_to_olap() {
size_t total_size = 0;
if (_nullmap) {
- const UInt8* nullmap_cur = _nullmap;
+ const UInt8* nullmap_cur = _nullmap + _row_pos;
while (hll_value_cur != hll_value_end) {
if (!*nullmap_cur) {
total_size += hll_value_cur->max_serialized_size();
@@ -303,7 +303,7 @@ Status OlapBlockDataConvertor::OlapColumnDataConvertorHLL::convert_to_olap() {
hll_value_cur = hll_value;
if (_nullmap) {
- const UInt8* nullmap_cur = _nullmap;
+ const UInt8* nullmap_cur = _nullmap + _row_pos;
while (hll_value_cur != hll_value_end) {
if (!*nullmap_cur) {
slice_size = hll_value_cur->serialize((uint8_t*)raw_data);
@@ -320,7 +320,7 @@ Status OlapBlockDataConvertor::OlapColumnDataConvertorHLL::convert_to_olap() {
++nullmap_cur;
++hll_value_cur;
}
- assert(nullmap_cur == _nullmap + _num_rows && slice == _slice.get_end_ptr());
+ assert(nullmap_cur == _nullmap + _row_pos + _num_rows && slice == _slice.get_end_ptr());
} else {
while (hll_value_cur != hll_value_end) {
slice_size = hll_value_cur->serialize((uint8_t*)raw_data);
@@ -355,7 +355,6 @@ const void* OlapBlockDataConvertor::OlapColumnDataConvertorChar::get_data() cons
}
const void* OlapBlockDataConvertor::OlapColumnDataConvertorChar::get_data_at(size_t offset) const {
- assert(offset < _num_rows && _num_rows == _slice.size());
UInt8 null_flag = 0;
if (_nullmap) {
null_flag = _nullmap[offset];
@@ -382,7 +381,7 @@ Status OlapBlockDataConvertor::OlapColumnDataConvertorChar::convert_to_olap() {
}
for (size_t i = 0; i < _num_rows; i++) {
- if (!_nullmap || !_nullmap[i]) {
+ if (!_nullmap || !_nullmap[i + _row_pos]) {
_slice[i] = column_string->get_data_at(i + _row_pos).to_slice();
DCHECK(_slice[i].size == _length)
<< "char type data length not equal to schema, schema=" << _length
@@ -411,7 +410,6 @@ const void* OlapBlockDataConvertor::OlapColumnDataConvertorVarChar::get_data() c
const void* OlapBlockDataConvertor::OlapColumnDataConvertorVarChar::get_data_at(
size_t offset) const {
- assert(offset < _num_rows && _num_rows == _slice.size());
UInt8 null_flag = 0;
if (_nullmap) {
null_flag = _nullmap[offset];
@@ -440,7 +438,7 @@ Status OlapBlockDataConvertor::OlapColumnDataConvertorVarChar::convert_to_olap()
Slice* slice = _slice.data();
size_t string_offset = *(offset_cur - 1);
if (_nullmap) {
- const UInt8* nullmap_cur = _nullmap;
+ const UInt8* nullmap_cur = _nullmap + _row_pos;
while (offset_cur != offset_end) {
if (!*nullmap_cur) {
slice->data = const_cast<char*>(char_data + string_offset);
@@ -461,7 +459,7 @@ Status OlapBlockDataConvertor::OlapColumnDataConvertorVarChar::convert_to_olap()
++slice;
++offset_cur;
}
- assert(nullmap_cur == _nullmap + _num_rows && slice == _slice.get_end_ptr());
+ assert(nullmap_cur == _nullmap + _row_pos + _num_rows && slice == _slice.get_end_ptr());
} else {
while (offset_cur != offset_end) {
slice->data = const_cast<char*>(char_data + string_offset);
@@ -513,7 +511,7 @@ Status OlapBlockDataConvertor::OlapColumnDataConvertorDate::convert_to_olap() {
const DateV2Value* datetime_end = datetime_cur + _num_rows;
uint24_t* value = _values.data();
if (_nullmap) {
- const UInt8* nullmap_cur = _nullmap;
+ const UInt8* nullmap_cur = _nullmap + _row_pos;
while (datetime_cur != datetime_end) {
if (!*nullmap_cur) {
*value = datetime_cur->to_olap_date();
@@ -524,7 +522,8 @@ Status OlapBlockDataConvertor::OlapColumnDataConvertorDate::convert_to_olap() {
++datetime_cur;
++nullmap_cur;
}
- assert(nullmap_cur == _nullmap + _num_rows && value == _values.get_end_ptr());
+ assert(nullmap_cur == _nullmap + _row_pos + _num_rows &&
+ value == _values.get_end_ptr());
} else {
while (datetime_cur != datetime_end) {
*value = datetime_cur->to_olap_date();
@@ -553,7 +552,7 @@ Status OlapBlockDataConvertor::OlapColumnDataConvertorDate::convert_to_olap() {
const VecDateTimeValue* datetime_end = datetime_cur + _num_rows;
uint24_t* value = _values.data();
if (_nullmap) {
- const UInt8* nullmap_cur = _nullmap;
+ const UInt8* nullmap_cur = _nullmap + _row_pos;
while (datetime_cur != datetime_end) {
if (!*nullmap_cur) {
*value = datetime_cur->to_olap_date();
@@ -564,7 +563,8 @@ Status OlapBlockDataConvertor::OlapColumnDataConvertorDate::convert_to_olap() {
++datetime_cur;
++nullmap_cur;
}
- assert(nullmap_cur == _nullmap + _num_rows && value == _values.get_end_ptr());
+ assert(nullmap_cur == _nullmap + _row_pos + _num_rows &&
+ value == _values.get_end_ptr());
} else {
while (datetime_cur != datetime_end) {
*value = datetime_cur->to_olap_date();
@@ -597,7 +597,7 @@ Status OlapBlockDataConvertor::OlapColumnDataConvertorDateTime::convert_to_olap(
const VecDateTimeValue* datetime_end = datetime_cur + _num_rows;
uint64_t* value = _values.data();
if (_nullmap) {
- const UInt8* nullmap_cur = _nullmap;
+ const UInt8* nullmap_cur = _nullmap + _row_pos;
while (datetime_cur != datetime_end) {
if (!*nullmap_cur) {
*value = datetime_cur->to_olap_datetime();
@@ -608,7 +608,7 @@ Status OlapBlockDataConvertor::OlapColumnDataConvertorDateTime::convert_to_olap(
++datetime_cur;
++nullmap_cur;
}
- assert(nullmap_cur == _nullmap + _num_rows && value == _values.get_end_ptr());
+ assert(nullmap_cur == _nullmap + _row_pos + _num_rows && value == _values.get_end_ptr());
} else {
while (datetime_cur != datetime_end) {
*value = datetime_cur->to_olap_datetime();
@@ -652,7 +652,7 @@ Status OlapBlockDataConvertor::OlapColumnDataConvertorDecimal::convert_to_olap()
++decimal_cur;
++nullmap_cur;
}
- assert(nullmap_cur == _nullmap + _num_rows && value == _values.get_end_ptr());
+ assert(nullmap_cur == _nullmap + _row_pos + _num_rows && value == _values.get_end_ptr());
} else {
while (decimal_cur != decimal_end) {
value->integer = decimal_cur->int_value();
@@ -713,7 +713,7 @@ Status OlapBlockDataConvertor::OlapColumnDataConvertorArray::convert_to_olap(
for (size_t i = 0; i < _num_rows; ++i, ++collection_value) {
int64_t cur_pos = _row_pos + i;
int64_t prev_pos = cur_pos - 1;
- if (_nullmap && _nullmap[cur_pos - _row_pos]) {
+ if (_nullmap && _nullmap[cur_pos]) {
continue;
}
auto offset = offsets[prev_pos];
diff --git a/be/src/vec/olap/olap_data_convertor.h b/be/src/vec/olap/olap_data_convertor.h
index 37bd4b1ec3..7b844d04f6 100644
--- a/be/src/vec/olap/olap_data_convertor.h
+++ b/be/src/vec/olap/olap_data_convertor.h
@@ -178,7 +178,6 @@ private:
}
const void* get_data() const override { return _values.data(); }
const void* get_data_at(size_t offset) const override {
- assert(offset < _num_rows && _num_rows == _values.size());
UInt8 null_flag = 0;
if (_nullmap) {
null_flag = _nullmap[offset];
@@ -298,7 +297,7 @@ private:
const VecDateTimeValue* datetime_end = datetime_cur + _num_rows;
uint32_t* value = const_cast<uint32_t*>(values_);
if (_nullmap) {
- const UInt8* nullmap_cur = _nullmap;
+ const UInt8* nullmap_cur = _nullmap + _row_pos;
while (datetime_cur != datetime_end) {
if (!*nullmap_cur) {
*value = datetime_cur->to_date_v2();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org