You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2023/01/03 01:22:06 UTC
[doris] branch master updated: [refactor](remove rowblockv2) remove rowblock v2 structure (#15540)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 14eaf41029 [refactor](remove rowblockv2) remove rowblock v2 structure (#15540)
14eaf41029 is described below
commit 14eaf41029c9b6d0af9932c08a1cea4328ebf3b7
Author: yiguolei <67...@qq.com>
AuthorDate: Tue Jan 3 09:21:57 2023 +0800
[refactor](remove rowblockv2) remove rowblock v2 structure (#15540)
* [refactor](remove rowblockv2) remove rowblock v2 structure
* fix bugs
Co-authored-by: yiguolei <yi...@gmail.com>
---
be/src/olap/CMakeLists.txt | 1 -
be/src/olap/block_column_predicate.cpp | 1 -
be/src/olap/row_block2.cpp | 682 ---------------------
be/src/olap/row_block2.h | 162 -----
be/src/olap/rowset/beta_rowset_reader.cpp | 1 -
be/src/olap/rowset/beta_rowset_reader.h | 1 -
.../rowset/segment_v2/empty_segment_iterator.cpp | 2 -
be/src/olap/rowset/segment_v2/segment_iterator.cpp | 13 -
be/src/olap/rowset/segment_v2/segment_iterator.h | 4 -
be/src/olap/schema.cpp | 1 -
be/src/util/arrow/row_block.cpp | 155 -----
be/src/util/arrow/row_block.h | 11 -
be/src/vec/olap/vgeneric_iterators.h | 1 -
be/test/olap/block_column_predicate_test.cpp | 1 -
be/test/runtime/array_test.cpp | 52 --
be/test/tools/benchmark_tool.cpp | 7 +-
be/test/vec/exec/vgeneric_iterators_test.cpp | 1 -
17 files changed, 6 insertions(+), 1090 deletions(-)
diff --git a/be/src/olap/CMakeLists.txt b/be/src/olap/CMakeLists.txt
index 25100a090f..9af9e86754 100644
--- a/be/src/olap/CMakeLists.txt
+++ b/be/src/olap/CMakeLists.txt
@@ -54,7 +54,6 @@ add_library(Olap STATIC
page_cache.cpp
push_handler.cpp
reader.cpp
- row_block2.cpp
row_cursor.cpp
version_graph.cpp
schema.cpp
diff --git a/be/src/olap/block_column_predicate.cpp b/be/src/olap/block_column_predicate.cpp
index 670c776353..7f6c524328 100644
--- a/be/src/olap/block_column_predicate.cpp
+++ b/be/src/olap/block_column_predicate.cpp
@@ -17,7 +17,6 @@
#include "block_column_predicate.h"
-#include "olap/row_block2.h"
#include "olap/rowset/segment_v2/bloom_filter.h"
namespace doris {
diff --git a/be/src/olap/row_block2.cpp b/be/src/olap/row_block2.cpp
deleted file mode 100644
index 07343eeca6..0000000000
--- a/be/src/olap/row_block2.cpp
+++ /dev/null
@@ -1,682 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "olap/row_block2.h"
-
-#include <algorithm>
-#include <sstream>
-#include <utility>
-
-#include "gutil/strings/substitute.h"
-#include "olap/row_cursor.h"
-#include "util/bitmap.h"
-#include "vec/columns/column_array.h"
-#include "vec/columns/column_complex.h"
-#include "vec/columns/column_vector.h"
-#include "vec/core/block.h"
-#include "vec/core/types.h"
-#include "vec/runtime/vdatetime_value.h"
-
-using strings::Substitute;
-namespace doris {
-
-RowBlockV2::RowBlockV2(const Schema& schema, uint16_t capacity)
- : _schema(schema),
- _capacity(capacity),
- _column_vector_batches(_schema.num_columns()),
- _pool(new MemPool()),
- _selection_vector(nullptr) {
- for (auto cid : _schema.column_ids()) {
- Status status = ColumnVectorBatch::create(
- _capacity, _schema.column(cid)->is_nullable(), _schema.column(cid)->type_info(),
- const_cast<Field*>(_schema.column(cid)), &_column_vector_batches[cid]);
- if (!status.ok()) {
- LOG(ERROR) << "failed to create ColumnVectorBatch for type: "
- << _schema.column(cid)->type();
- return;
- }
- }
- _selection_vector = new uint16_t[_capacity];
- clear();
-}
-
-RowBlockV2::~RowBlockV2() {
- delete[] _selection_vector;
-}
-
-Status RowBlockV2::_copy_data_to_column(int cid,
- doris::vectorized::MutableColumnPtr& origin_column) {
- auto* column = origin_column.get();
- bool nullable_mark_array[_selected_size];
-
- bool column_nullable = origin_column->is_nullable();
- bool origin_nullable = _schema.column(cid)->is_nullable();
- if (column_nullable) {
- auto nullable_column = assert_cast<vectorized::ColumnNullable*>(origin_column.get());
- auto& null_map = nullable_column->get_null_map_data();
- column = nullable_column->get_nested_column_ptr().get();
-
- if (origin_nullable) {
- for (uint16_t i = 0; i < _selected_size; ++i) {
- uint16_t row_idx = _selection_vector[i];
- null_map.push_back(_column_vector_batches[cid]->is_null_at(row_idx));
- nullable_mark_array[i] = null_map.back();
- }
- } else {
- null_map.resize_fill(null_map.size() + _selected_size, 0);
- memset(nullable_mark_array, false, _selected_size * sizeof(bool));
- }
- } else {
- memset(nullable_mark_array, false, _selected_size * sizeof(bool));
- }
-
- auto insert_data_directly = [this, &nullable_mark_array](int cid, auto& column) {
- for (uint16_t j = 0; j < _selected_size; ++j) {
- if (!nullable_mark_array[j]) {
- uint16_t row_idx = _selection_vector[j];
- column->insert_data(
- reinterpret_cast<const char*>(column_block(cid).cell_ptr(row_idx)), 0);
- } else {
- column->insert_default();
- }
- }
- };
-
- switch (_schema.column(cid)->type()) {
- case OLAP_FIELD_TYPE_OBJECT: {
- auto column_bitmap = assert_cast<vectorized::ColumnBitmap*>(column);
- for (uint16_t j = 0; j < _selected_size; ++j) {
- column_bitmap->insert_default();
- if (!nullable_mark_array[j]) {
- uint16_t row_idx = _selection_vector[j];
- auto slice = reinterpret_cast<const Slice*>(column_block(cid).cell_ptr(row_idx));
-
- BitmapValue* pvalue = &column_bitmap->get_element(column_bitmap->size() - 1);
-
- if (slice->size != 0) {
- BitmapValue value;
- value.deserialize(slice->data);
- *pvalue = std::move(value);
- } else {
- *pvalue = std::move(*reinterpret_cast<BitmapValue*>(slice->data));
- }
- }
- }
- break;
- }
- case OLAP_FIELD_TYPE_HLL: {
- auto column_hll = assert_cast<vectorized::ColumnHLL*>(column);
- for (uint16_t j = 0; j < _selected_size; ++j) {
- column_hll->insert_default();
- if (!nullable_mark_array[j]) {
- uint16_t row_idx = _selection_vector[j];
- auto slice = reinterpret_cast<const Slice*>(column_block(cid).cell_ptr(row_idx));
-
- HyperLogLog* pvalue = &column_hll->get_element(column_hll->size() - 1);
-
- if (slice->size != 0) {
- HyperLogLog value;
- value.deserialize(*slice);
- *pvalue = std::move(value);
- } else {
- *pvalue = std::move(*reinterpret_cast<HyperLogLog*>(slice->data));
- }
- }
- }
- break;
- }
- case OLAP_FIELD_TYPE_MAP:
- case OLAP_FIELD_TYPE_VARCHAR: {
- auto column_string = assert_cast<vectorized::ColumnString*>(column);
-
- for (uint16_t j = 0; j < _selected_size; ++j) {
- if (!nullable_mark_array[j]) {
- uint16_t row_idx = _selection_vector[j];
- auto slice = reinterpret_cast<const Slice*>(column_block(cid).cell_ptr(row_idx));
- column_string->insert_data(slice->data, slice->size);
- } else {
- column_string->insert_default();
- }
- }
- break;
- }
- case OLAP_FIELD_TYPE_STRING: {
- auto column_string = assert_cast<vectorized::ColumnString*>(column);
- size_t limit = config::string_type_length_soft_limit_bytes;
- for (uint16_t j = 0; j < _selected_size; ++j) {
- if (!nullable_mark_array[j]) {
- uint16_t row_idx = _selection_vector[j];
- auto slice = reinterpret_cast<const Slice*>(column_block(cid).cell_ptr(row_idx));
- if (LIKELY(slice->size <= limit)) {
- column_string->insert_data(slice->data, slice->size);
- } else {
- return Status::NotSupported(
- "Not support string len over than {} in vec engine.", limit);
- }
- } else {
- column_string->insert_default();
- }
- }
- break;
- }
- case OLAP_FIELD_TYPE_CHAR: {
- auto column_string = assert_cast<vectorized::ColumnString*>(column);
-
- for (uint16_t j = 0; j < _selected_size; ++j) {
- if (!nullable_mark_array[j]) {
- uint16_t row_idx = _selection_vector[j];
- auto slice = reinterpret_cast<const Slice*>(column_block(cid).cell_ptr(row_idx));
- column_string->insert_data(slice->data, strnlen(slice->data, slice->size));
- } else {
- column_string->insert_default();
- }
- }
- break;
- }
- case OLAP_FIELD_TYPE_DATE: {
- auto column_int = assert_cast<vectorized::ColumnVector<vectorized::Int64>*>(column);
-
- for (uint16_t j = 0; j < _selected_size; ++j) {
- if (!nullable_mark_array[j]) {
- uint16_t row_idx = _selection_vector[j];
- auto ptr = reinterpret_cast<const char*>(column_block(cid).cell_ptr(row_idx));
-
- uint64_t value = 0;
- value = *(unsigned char*)(ptr + 2);
- value <<= 8;
- value |= *(unsigned char*)(ptr + 1);
- value <<= 8;
- value |= *(unsigned char*)(ptr);
- vectorized::VecDateTimeValue date =
- vectorized::VecDateTimeValue::create_from_olap_date(value);
- (column_int)->insert_data(reinterpret_cast<char*>(&date), 0);
- } else {
- column_int->insert_default();
- }
- }
- break;
- }
- case OLAP_FIELD_TYPE_DATEV2: {
- auto column_int = assert_cast<vectorized::ColumnVector<vectorized::UInt32>*>(column);
- insert_data_directly(cid, column_int);
- break;
- }
- case OLAP_FIELD_TYPE_DATETIMEV2: {
- auto column_int = assert_cast<vectorized::ColumnVector<vectorized::UInt64>*>(column);
- insert_data_directly(cid, column_int);
- break;
- }
- case OLAP_FIELD_TYPE_DATETIME: {
- auto column_int = assert_cast<vectorized::ColumnVector<vectorized::Int64>*>(column);
-
- for (uint16_t j = 0; j < _selected_size; ++j) {
- if (!nullable_mark_array[j]) {
- uint16_t row_idx = _selection_vector[j];
- auto ptr = reinterpret_cast<const char*>(column_block(cid).cell_ptr(row_idx));
-
- uint64_t value = *reinterpret_cast<const uint64_t*>(ptr);
- vectorized::VecDateTimeValue datetime =
- vectorized::VecDateTimeValue::create_from_olap_datetime(value);
- (column_int)->insert_data(reinterpret_cast<char*>(&datetime), 0);
- } else {
- column_int->insert_default();
- }
- }
- break;
- }
- case OLAP_FIELD_TYPE_DECIMAL: {
- auto column_decimal =
- assert_cast<vectorized::ColumnDecimal<vectorized::Decimal128>*>(column);
-
- for (uint16_t j = 0; j < _selected_size; ++j) {
- if (!nullable_mark_array[j]) {
- uint16_t row_idx = _selection_vector[j];
- auto ptr = reinterpret_cast<const char*>(column_block(cid).cell_ptr(row_idx));
-
- int64_t int_value = *(int64_t*)(ptr);
- int32_t frac_value = *(int32_t*)(ptr + sizeof(int64_t));
- DecimalV2Value data(int_value, frac_value);
- column_decimal->insert_data(reinterpret_cast<char*>(&data), 0);
- } else {
- column_decimal->insert_default();
- }
- }
- break;
- }
- case OLAP_FIELD_TYPE_DECIMAL32: {
- auto column_decimal =
- assert_cast<vectorized::ColumnDecimal<vectorized::Decimal32>*>(column);
- insert_data_directly(cid, column_decimal);
- break;
- }
- case OLAP_FIELD_TYPE_DECIMAL64: {
- auto column_decimal =
- assert_cast<vectorized::ColumnDecimal<vectorized::Decimal64>*>(column);
- insert_data_directly(cid, column_decimal);
- break;
- }
- case OLAP_FIELD_TYPE_DECIMAL128I: {
- auto column_decimal =
- assert_cast<vectorized::ColumnDecimal<vectorized::Decimal128I>*>(column);
- insert_data_directly(cid, column_decimal);
- break;
- }
- case OLAP_FIELD_TYPE_JSONB: {
- auto json_string = assert_cast<vectorized::ColumnString*>(column);
- size_t limit = config::jsonb_type_length_soft_limit_bytes;
- for (uint16_t j = 0; j < _selected_size; ++j) {
- if (!nullable_mark_array[j]) {
- uint16_t row_idx = _selection_vector[j];
- auto slice = reinterpret_cast<const Slice*>(column_block(cid).cell_ptr(row_idx));
- if (LIKELY(slice->size <= limit)) {
- json_string->insert_data(slice->data, slice->size);
- } else {
- return Status::NotSupported(
- fmt::format("Not support json len over than {} in vec engine.", limit));
- }
- } else {
- // TODO
- json_string->insert_default();
- }
- }
- break;
- }
- case OLAP_FIELD_TYPE_ARRAY: {
- auto column_array = assert_cast<vectorized::ColumnArray*>(column);
- auto nested_col = (*column_array->get_data_ptr()).assume_mutable();
- auto src_col = static_cast<ArrayColumnVectorBatch*>(_column_vector_batches[cid].get());
-
- auto& offsets_col = column_array->get_offsets();
- offsets_col.reserve(_selected_size);
- uint64_t offset = offsets_col.back();
- for (uint16_t j = 0; j < _selected_size; ++j) {
- uint16_t row_idx = _selection_vector[j];
- auto cv = reinterpret_cast<const CollectionValue*>(column_block(cid).cell_ptr(row_idx));
- if (!nullable_mark_array[j]) {
- offset += cv->length();
- _append_data_to_column(src_col->elements(), src_col->item_offset(row_idx),
- cv->length(), nested_col);
- offsets_col.push_back(offset);
- } else {
- column_array->insert_default();
- }
- }
- break;
- }
- case OLAP_FIELD_TYPE_INT: {
- auto column_int = assert_cast<vectorized::ColumnVector<vectorized::Int32>*>(column);
- insert_data_directly(cid, column_int);
- break;
- }
- case OLAP_FIELD_TYPE_BOOL: {
- auto column_int = assert_cast<vectorized::ColumnVector<vectorized::UInt8>*>(column);
- insert_data_directly(cid, column_int);
- break;
- }
- case OLAP_FIELD_TYPE_TINYINT: {
- auto column_int = assert_cast<vectorized::ColumnVector<vectorized::Int8>*>(column);
- insert_data_directly(cid, column_int);
- break;
- }
- case OLAP_FIELD_TYPE_SMALLINT: {
- auto column_int = assert_cast<vectorized::ColumnVector<vectorized::Int16>*>(column);
- insert_data_directly(cid, column_int);
- break;
- }
- case OLAP_FIELD_TYPE_BIGINT: {
- auto column_int = assert_cast<vectorized::ColumnVector<vectorized::Int64>*>(column);
- insert_data_directly(cid, column_int);
- break;
- }
- case OLAP_FIELD_TYPE_LARGEINT: {
- auto column_int = assert_cast<vectorized::ColumnVector<vectorized::Int128>*>(column);
- insert_data_directly(cid, column_int);
- break;
- }
- case OLAP_FIELD_TYPE_FLOAT: {
- auto column_float = assert_cast<vectorized::ColumnVector<vectorized::Float32>*>(column);
- insert_data_directly(cid, column_float);
- break;
- }
- case OLAP_FIELD_TYPE_DOUBLE: {
- auto column_float = assert_cast<vectorized::ColumnVector<vectorized::Float64>*>(column);
- insert_data_directly(cid, column_float);
- break;
- }
- default: {
- DCHECK(false) << "Invalid type in RowBlockV2:" << _schema.column(cid)->type();
- }
- }
-
- return Status::OK();
-}
-
-Status RowBlockV2::_append_data_to_column(const ColumnVectorBatch* batch, size_t start,
- uint32_t len,
- doris::vectorized::MutableColumnPtr& origin_column) {
- auto* column = origin_column.get();
- uint32_t selected_size = len;
- bool nullable_mark_array[selected_size];
-
- bool column_nullable = origin_column->is_nullable();
- bool origin_nullable = batch->is_nullable();
- if (column_nullable) {
- auto nullable_column = assert_cast<vectorized::ColumnNullable*>(origin_column.get());
- auto& null_map = nullable_column->get_null_map_data();
- column = nullable_column->get_nested_column_ptr().get();
-
- if (origin_nullable) {
- for (uint32_t i = 0; i < selected_size; ++i) {
- uint32_t row_idx = i + start;
- null_map.push_back(batch->is_null_at(row_idx));
- nullable_mark_array[i] = null_map.back();
- }
- } else {
- null_map.resize_fill(null_map.size() + selected_size, 0);
- memset(nullable_mark_array, false, selected_size * sizeof(bool));
- }
- } else {
- memset(nullable_mark_array, false, selected_size * sizeof(bool));
- }
-
- auto insert_data_directly = [&nullable_mark_array](auto& batch, auto& column, auto& start,
- auto& length) {
- for (uint32_t j = 0; j < length; ++j) {
- if (!nullable_mark_array[j]) {
- uint32_t row_idx = j + start;
- column->insert_data(reinterpret_cast<const char*>(batch->cell_ptr(row_idx)), 0);
- } else {
- column->insert_default();
- }
- }
- };
-
- switch (batch->type_info()->type()) {
- case OLAP_FIELD_TYPE_OBJECT: {
- auto column_bitmap = assert_cast<vectorized::ColumnBitmap*>(column);
- for (uint32_t j = 0; j < selected_size; ++j) {
- column_bitmap->insert_default();
- if (!nullable_mark_array[j]) {
- uint32_t row_idx = j + start;
- auto slice = reinterpret_cast<const Slice*>(batch->cell_ptr(row_idx));
-
- BitmapValue* pvalue = &column_bitmap->get_element(column_bitmap->size() - 1);
-
- if (slice->size != 0) {
- BitmapValue value;
- value.deserialize(slice->data);
- *pvalue = std::move(value);
- } else {
- *pvalue = std::move(*reinterpret_cast<BitmapValue*>(slice->data));
- }
- }
- }
- break;
- }
- case OLAP_FIELD_TYPE_HLL: {
- auto column_hll = assert_cast<vectorized::ColumnHLL*>(column);
- for (uint32_t j = 0; j < selected_size; ++j) {
- column_hll->insert_default();
- if (!nullable_mark_array[j]) {
- uint32_t row_idx = j + start;
- auto slice = reinterpret_cast<const Slice*>(batch->cell_ptr(row_idx));
-
- HyperLogLog* pvalue = &column_hll->get_element(column_hll->size() - 1);
-
- if (slice->size != 0) {
- HyperLogLog value;
- value.deserialize(*slice);
- *pvalue = std::move(value);
- } else {
- *pvalue = std::move(*reinterpret_cast<HyperLogLog*>(slice->data));
- }
- }
- }
- break;
- }
- case OLAP_FIELD_TYPE_MAP:
- case OLAP_FIELD_TYPE_VARCHAR: {
- auto column_string = assert_cast<vectorized::ColumnString*>(column);
-
- for (uint32_t j = 0; j < selected_size; ++j) {
- if (!nullable_mark_array[j]) {
- uint32_t row_idx = j + start;
- auto slice = reinterpret_cast<const Slice*>(batch->cell_ptr(row_idx));
- column_string->insert_data(slice->data, slice->size);
- } else {
- column_string->insert_default();
- }
- }
- break;
- }
- case OLAP_FIELD_TYPE_STRING: {
- auto column_string = assert_cast<vectorized::ColumnString*>(column);
-
- for (uint32_t j = 0; j < selected_size; ++j) {
- if (!nullable_mark_array[j]) {
- uint32_t row_idx = j + start;
- auto slice = reinterpret_cast<const Slice*>(batch->cell_ptr(row_idx));
- if (LIKELY(slice->size <= config::string_type_length_soft_limit_bytes)) {
- column_string->insert_data(slice->data, slice->size);
- } else {
- return Status::NotSupported(
- "Not support string len over than "
- "`string_type_length_soft_limit_bytes` in vec engine.");
- }
- } else {
- column_string->insert_default();
- }
- }
- break;
- }
- case OLAP_FIELD_TYPE_CHAR: {
- auto column_string = assert_cast<vectorized::ColumnString*>(column);
-
- for (uint32_t j = 0; j < selected_size; ++j) {
- if (!nullable_mark_array[j]) {
- uint32_t row_idx = j + start;
- auto slice = reinterpret_cast<const Slice*>(batch->cell_ptr(row_idx));
- column_string->insert_data(slice->data, strnlen(slice->data, slice->size));
- } else {
- column_string->insert_default();
- }
- }
- break;
- }
- case OLAP_FIELD_TYPE_DATE: {
- auto column_int = assert_cast<vectorized::ColumnVector<vectorized::Int64>*>(column);
-
- for (uint32_t j = 0; j < selected_size; ++j) {
- if (!nullable_mark_array[j]) {
- uint32_t row_idx = j + start;
- auto ptr = reinterpret_cast<const char*>(batch->cell_ptr(row_idx));
-
- uint64_t value = 0;
- value = *(unsigned char*)(ptr + 2);
- value <<= 8;
- value |= *(unsigned char*)(ptr + 1);
- value <<= 8;
- value |= *(unsigned char*)(ptr);
- vectorized::VecDateTimeValue date =
- vectorized::VecDateTimeValue::create_from_olap_date(value);
- (column_int)->insert_data(reinterpret_cast<char*>(&date), 0);
- } else {
- column_int->insert_default();
- }
- }
- break;
- }
- case OLAP_FIELD_TYPE_DATEV2: {
- auto column_int = assert_cast<vectorized::ColumnVector<vectorized::UInt32>*>(column);
- insert_data_directly(batch, column_int, start, len);
- break;
- }
- case OLAP_FIELD_TYPE_DATETIMEV2: {
- auto column_int = assert_cast<vectorized::ColumnVector<vectorized::UInt64>*>(column);
- insert_data_directly(batch, column_int, start, len);
- break;
- }
- case OLAP_FIELD_TYPE_DATETIME: {
- auto column_int = assert_cast<vectorized::ColumnVector<vectorized::Int64>*>(column);
-
- for (uint32_t j = 0; j < selected_size; ++j) {
- if (!nullable_mark_array[j]) {
- uint32_t row_idx = j + start;
- auto ptr = reinterpret_cast<const char*>(batch->cell_ptr(row_idx));
-
- uint64_t value = *reinterpret_cast<const uint64_t*>(ptr);
- vectorized::VecDateTimeValue datetime =
- vectorized::VecDateTimeValue::create_from_olap_datetime(value);
- (column_int)->insert_data(reinterpret_cast<char*>(&datetime), 0);
- } else {
- column_int->insert_default();
- }
- }
- break;
- }
- case OLAP_FIELD_TYPE_DECIMAL: {
- auto column_decimal =
- assert_cast<vectorized::ColumnDecimal<vectorized::Decimal128I>*>(column);
-
- for (uint32_t j = 0; j < selected_size; ++j) {
- if (!nullable_mark_array[j]) {
- uint32_t row_idx = j + start;
- auto ptr = reinterpret_cast<const char*>(batch->cell_ptr(row_idx));
-
- int64_t int_value = *(int64_t*)(ptr);
- int32_t frac_value = *(int32_t*)(ptr + sizeof(int64_t));
- DecimalV2Value data(int_value, frac_value);
- column_decimal->insert_data(reinterpret_cast<char*>(&data), 0);
- } else {
- column_decimal->insert_default();
- }
- }
- break;
- }
- case OLAP_FIELD_TYPE_DECIMAL32: {
- auto column_decimal =
- assert_cast<vectorized::ColumnDecimal<vectorized::Decimal32>*>(column);
- insert_data_directly(batch, column_decimal, start, len);
- }
- case OLAP_FIELD_TYPE_DECIMAL64: {
- auto column_decimal =
- assert_cast<vectorized::ColumnDecimal<vectorized::Decimal64>*>(column);
- insert_data_directly(batch, column_decimal, start, len);
- }
- case OLAP_FIELD_TYPE_DECIMAL128I: {
- auto column_decimal =
- assert_cast<vectorized::ColumnDecimal<vectorized::Decimal128I>*>(column);
- insert_data_directly(batch, column_decimal, start, len);
- }
- case OLAP_FIELD_TYPE_ARRAY: {
- auto array_batch = reinterpret_cast<const ArrayColumnVectorBatch*>(batch);
- auto column_array = assert_cast<vectorized::ColumnArray*>(column);
- auto nested_col = (*column_array->get_data_ptr()).assume_mutable();
-
- auto& offsets_col = column_array->get_offsets();
- auto offset = offsets_col.back();
- for (uint32_t j = 0; j < selected_size; ++j) {
- if (!nullable_mark_array[j]) {
- uint64_t row_idx = j + start;
- auto cv = reinterpret_cast<const CollectionValue*>(batch->cell_ptr(row_idx));
- offset += cv->length();
- _append_data_to_column(array_batch->elements(), array_batch->item_offset(row_idx),
- cv->length(), nested_col);
- offsets_col.push_back(offset);
- } else {
- column_array->insert_default();
- }
- }
- break;
- }
- case OLAP_FIELD_TYPE_INT: {
- auto column_int = assert_cast<vectorized::ColumnVector<vectorized::Int32>*>(column);
- insert_data_directly(batch, column_int, start, len);
- break;
- }
- case OLAP_FIELD_TYPE_BOOL: {
- auto column_int = assert_cast<vectorized::ColumnVector<vectorized::UInt8>*>(column);
- insert_data_directly(batch, column_int, start, len);
- break;
- }
- case OLAP_FIELD_TYPE_TINYINT: {
- auto column_int = assert_cast<vectorized::ColumnVector<vectorized::Int8>*>(column);
- insert_data_directly(batch, column_int, start, len);
- break;
- }
- case OLAP_FIELD_TYPE_SMALLINT: {
- auto column_int = assert_cast<vectorized::ColumnVector<vectorized::Int16>*>(column);
- insert_data_directly(batch, column_int, start, len);
- break;
- }
- case OLAP_FIELD_TYPE_BIGINT: {
- auto column_int = assert_cast<vectorized::ColumnVector<vectorized::Int64>*>(column);
- insert_data_directly(batch, column_int, start, len);
- break;
- }
- case OLAP_FIELD_TYPE_LARGEINT: {
- auto column_int = assert_cast<vectorized::ColumnVector<vectorized::Int128>*>(column);
- insert_data_directly(batch, column_int, start, len);
- break;
- }
- case OLAP_FIELD_TYPE_FLOAT: {
- auto column_float = assert_cast<vectorized::ColumnVector<vectorized::Float32>*>(column);
- insert_data_directly(batch, column_float, start, len);
- break;
- }
- case OLAP_FIELD_TYPE_DOUBLE: {
- auto column_float = assert_cast<vectorized::ColumnVector<vectorized::Float64>*>(column);
- insert_data_directly(batch, column_float, start, len);
- break;
- }
- default: {
- DCHECK(false) << "Invalid type in RowBlockV2:" << batch->type_info()->type();
- }
- }
-
- return Status::OK();
-}
-
-std::string RowBlockRow::debug_string() const {
- std::stringstream ss;
- ss << "[";
- for (int i = 0; i < _block->schema()->num_column_ids(); ++i) {
- if (i != 0) {
- ss << ",";
- }
- ColumnId cid = _block->schema()->column_ids()[i];
- auto col_schema = _block->schema()->column(cid);
- if (col_schema->is_nullable() && is_null(cid)) {
- ss << "NULL";
- } else {
- ss << col_schema->type_info()->to_string(cell_ptr(cid));
- }
- }
- ss << "]";
- return ss.str();
-}
-std::string RowBlockV2::debug_string() {
- std::stringstream ss;
- for (int i = 0; i < num_rows(); ++i) {
- ss << row(i).debug_string();
- if (i != num_rows() - 1) {
- ss << "\n";
- }
- }
- return ss.str();
-}
-} // namespace doris
diff --git a/be/src/olap/row_block2.h b/be/src/olap/row_block2.h
deleted file mode 100644
index 0455036f12..0000000000
--- a/be/src/olap/row_block2.h
+++ /dev/null
@@ -1,162 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#pragma once
-
-#include <cstddef>
-#include <cstdint>
-#include <vector>
-
-#include "common/status.h"
-#include "olap/column_block.h"
-#include "olap/schema.h"
-#include "olap/selection_vector.h"
-#include "olap/types.h"
-#include "runtime/mem_pool.h"
-#include "runtime/memory/mem_tracker.h"
-#include "vec/columns/column.h"
-
-namespace doris {
-
-class MemPool;
-class RowBlockRow;
-class RowCursor;
-
-// This struct contains a block of rows, in which each column's data is stored
-// in block, however it is used by old code, which we don't want to change.
-class RowBlockV2 {
-public:
- RowBlockV2(const Schema& schema, uint16_t capacity);
-
- ~RowBlockV2();
-
- // update number of rows contained in this block
- void set_num_rows(size_t num_rows) { _num_rows = num_rows; }
- // low-level API to get the number of rows contained in this block.
- // note that some rows may have been un-selected by selection_vector(),
- // client should use selected_size() to get the actual number of selected rows.
- size_t num_rows() const { return _num_rows; }
- // return the maximum number of rows that can be contained in this block.
- // invariant: 0 <= num_rows() <= capacity()
- size_t capacity() const { return _capacity; }
- MemPool* pool() const { return _pool.get(); }
-
- // reset the state of the block so that it can be reused for write.
- // all previously returned ColumnBlocks are invalidated after clear(), accessing them
- // will result in undefined behavior.
- void clear() {
- _num_rows = 0;
- _pool->clear();
- _selected_size = _capacity;
- for (int i = 0; i < _selected_size; ++i) {
- _selection_vector[i] = i;
- }
- _delete_state = DEL_NOT_SATISFIED;
- }
-
- // low-level API to access memory for each column block(including data array and nullmap).
- // `cid` must be one of `schema()->column_ids()`.
- ColumnBlock column_block(ColumnId cid) const {
- ColumnVectorBatch* batch = _column_vector_batches[cid].get();
- return {batch, _pool.get()};
- }
-
- // low-level API to access the underlying memory for row at `row_idx`.
- // client should use selection_vector() to iterate over row index of selected rows.
- // TODO(gdy) DO NOT expose raw rows which may be un-selected to clients
- RowBlockRow row(size_t row_idx) const;
-
- const Schema* schema() const { return &_schema; }
-
- uint16_t* selection_vector() const { return _selection_vector; }
-
- uint16_t selected_size() const { return _selected_size; }
-
- void set_selected_size(uint16_t selected_size) { _selected_size = selected_size; }
-
- DelCondSatisfied delete_state() const { return _delete_state; }
-
- void set_delete_state(DelCondSatisfied delete_state) {
- // if the set _delete_state is DEL_PARTIAL_SATISFIED,
- // we can not change _delete_state to DEL_NOT_SATISFIED;
- if (_delete_state == DEL_PARTIAL_SATISFIED && delete_state != DEL_SATISFIED) {
- return;
- }
- _delete_state = delete_state;
- }
- std::string debug_string();
-
-private:
- Status _copy_data_to_column(int cid, vectorized::MutableColumnPtr& mutable_column_ptr);
- Status _append_data_to_column(const ColumnVectorBatch* batch, size_t start, uint32_t len,
- vectorized::MutableColumnPtr& mutable_column_ptr);
-
- const Schema& _schema;
- size_t _capacity;
- // _column_vector_batches[cid] == null if cid is not in `_schema`.
- // memory are not allocated from `_pool` because we don't wan't to reallocate them in clear()
- std::vector<std::unique_ptr<ColumnVectorBatch>> _column_vector_batches;
-
- size_t _num_rows;
- // manages the memory for slice's data
- std::unique_ptr<MemPool> _pool;
-
- // index of selected rows for rows passed the predicate
- uint16_t* _selection_vector;
- // selected rows number
- uint16_t _selected_size;
-
- // block delete state
- DelCondSatisfied _delete_state;
-};
-
-// Stands for a row in RowBlockV2. It is consisted of a RowBlockV2 reference
-// and row index.
-class RowBlockRow {
-public:
- RowBlockRow(const RowBlockV2* block, size_t row_index) : _block(block), _row_index(row_index) {}
-
- const RowBlockV2* row_block() const { return _block; }
- size_t row_index() const { return _row_index; }
-
- ColumnBlock column_block(size_t col_idx) const { return _block->column_block(col_idx); }
- bool is_null(size_t col_idx) const { return column_block(col_idx).is_null(_row_index); }
- void set_is_null(size_t col_idx, bool is_null) {
- return column_block(col_idx).set_is_null(_row_index, is_null);
- }
- uint8_t* mutable_cell_ptr(size_t col_idx) const {
- return column_block(col_idx).mutable_cell_ptr(_row_index);
- }
- const uint8_t* cell_ptr(size_t col_idx) const {
- return column_block(col_idx).cell_ptr(_row_index);
- }
- const Schema* schema() const { return _block->schema(); }
-
- std::string debug_string() const;
-
- ColumnBlockCell cell(uint32_t cid) const { return column_block(cid).cell(_row_index); }
-
-private:
- const RowBlockV2* _block;
- size_t _row_index;
-};
-
-inline RowBlockRow RowBlockV2::row(size_t row_idx) const {
- return RowBlockRow(this, row_idx);
-}
-
-} // namespace doris
diff --git a/be/src/olap/rowset/beta_rowset_reader.cpp b/be/src/olap/rowset/beta_rowset_reader.cpp
index 9daab16ec7..3a52cef373 100644
--- a/be/src/olap/rowset/beta_rowset_reader.cpp
+++ b/be/src/olap/rowset/beta_rowset_reader.cpp
@@ -20,7 +20,6 @@
#include <utility>
#include "olap/delete_handler.h"
-#include "olap/row_block2.h"
#include "olap/row_cursor.h"
#include "olap/schema.h"
#include "olap/tablet_meta.h"
diff --git a/be/src/olap/rowset/beta_rowset_reader.h b/be/src/olap/rowset/beta_rowset_reader.h
index 8bd88b7a92..5082be1ee8 100644
--- a/be/src/olap/rowset/beta_rowset_reader.h
+++ b/be/src/olap/rowset/beta_rowset_reader.h
@@ -18,7 +18,6 @@
#pragma once
#include "olap/iterators.h"
-#include "olap/row_block2.h"
#include "olap/row_cursor.h"
#include "olap/rowset/beta_rowset.h"
#include "olap/rowset/rowset_reader.h"
diff --git a/be/src/olap/rowset/segment_v2/empty_segment_iterator.cpp b/be/src/olap/rowset/segment_v2/empty_segment_iterator.cpp
index 0c9985e884..e9a51ed46a 100644
--- a/be/src/olap/rowset/segment_v2/empty_segment_iterator.cpp
+++ b/be/src/olap/rowset/segment_v2/empty_segment_iterator.cpp
@@ -17,8 +17,6 @@
#include "olap/rowset/segment_v2/empty_segment_iterator.h"
-#include "olap/row_block2.h"
-
namespace doris {
namespace segment_v2 {
diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
index 3ca445cf77..0e970d4831 100644
--- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
@@ -25,7 +25,6 @@
#include "common/status.h"
#include "olap/column_predicate.h"
#include "olap/olap_common.h"
-#include "olap/row_block2.h"
#include "olap/rowset/segment_v2/column_reader.h"
#include "olap/rowset/segment_v2/segment.h"
#include "olap/short_key_index.h"
@@ -861,18 +860,6 @@ Status SegmentIterator::_seek_columns(const std::vector<ColumnId>& column_ids, r
return Status::OK();
}
-Status SegmentIterator::_read_columns(const std::vector<ColumnId>& column_ids, RowBlockV2* block,
- size_t row_offset, size_t nrows) {
- for (auto cid : column_ids) {
- auto column_block = block->column_block(cid);
- ColumnBlockView dst(&column_block, row_offset);
- size_t rows_read = nrows;
- RETURN_IF_ERROR(_column_iterators[_schema.unique_id(cid)]->next_batch(&rows_read, &dst));
- DCHECK_EQ(nrows, rows_read);
- }
- return Status::OK();
-}
-
/* ---------------------- for vectorization implementation ---------------------- */
/**
diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.h b/be/src/olap/rowset/segment_v2/segment_iterator.h
index 2e60393186..6c6932c2f5 100644
--- a/be/src/olap/rowset/segment_v2/segment_iterator.h
+++ b/be/src/olap/rowset/segment_v2/segment_iterator.h
@@ -37,7 +37,6 @@
namespace doris {
class RowCursor;
-class RowBlockV2;
class ShortKeyIndexIterator;
namespace fs {
@@ -150,9 +149,6 @@ private:
Status _seek_columns(const std::vector<ColumnId>& column_ids, rowid_t pos);
// read `nrows` of columns specified by `column_ids` into `block` at `row_offset`.
- Status _read_columns(const std::vector<ColumnId>& column_ids, RowBlockV2* block,
- size_t row_offset, size_t nrows);
-
// for vectorization implementation
Status _read_columns(const std::vector<ColumnId>& column_ids,
vectorized::MutableColumns& column_block, size_t nrows);
diff --git a/be/src/olap/schema.cpp b/be/src/olap/schema.cpp
index 0d7cdb5d36..66b0547dd2 100644
--- a/be/src/olap/schema.cpp
+++ b/be/src/olap/schema.cpp
@@ -17,7 +17,6 @@
#include "olap/schema.h"
-#include "olap/row_block2.h"
#include "olap/uint24.h"
#include "vec/columns/column_complex.h"
#include "vec/columns/column_dictionary.h"
diff --git a/be/src/util/arrow/row_block.cpp b/be/src/util/arrow/row_block.cpp
index 2eb9b0301d..16db9417b6 100644
--- a/be/src/util/arrow/row_block.cpp
+++ b/be/src/util/arrow/row_block.cpp
@@ -30,7 +30,6 @@
#include "olap/column_block.h"
#include "olap/field.h"
#include "olap/olap_common.h"
-#include "olap/row_block2.h"
#include "olap/schema.h"
#include "olap/tablet_schema.h"
#include "util/arrow/utils.h"
@@ -138,158 +137,4 @@ Status convert_to_doris_schema(const arrow::Schema& schema, std::shared_ptr<Sche
return Status::OK();
}
-// Convert data in RowBlockV2 to an Arrow RecordBatch
-// We should keep this function to keep compatible with arrow's type visitor
-// Now we inherit TypeVisitor to use default Visit implementation
-class FromRowBlockConverter : public arrow::TypeVisitor {
-public:
- FromRowBlockConverter(const RowBlockV2& block, const std::shared_ptr<arrow::Schema>& schema,
- arrow::MemoryPool* pool)
- : _block(block), _schema(schema), _pool(pool) {}
-
- ~FromRowBlockConverter() override {}
-
- // Use base class function
- using arrow::TypeVisitor::Visit;
-
-#define PRIMITIVE_VISIT(TYPE) \
- arrow::Status Visit(const arrow::TYPE& type) override { return _visit(type); }
-
- PRIMITIVE_VISIT(Int8Type);
- PRIMITIVE_VISIT(Int16Type);
- PRIMITIVE_VISIT(Int32Type);
- PRIMITIVE_VISIT(Int64Type);
- PRIMITIVE_VISIT(FloatType);
- PRIMITIVE_VISIT(DoubleType);
-
-#undef PRIMITIVE_VISIT
-
- Status convert(std::shared_ptr<arrow::RecordBatch>* out);
-
-private:
- template <typename T>
- typename std::enable_if<std::is_base_of<arrow::PrimitiveCType, T>::value, arrow::Status>::type
- _visit(const T& type) {
- arrow::NumericBuilder<T> builder(_pool);
- size_t num_rows = _block.num_rows();
- ARROW_RETURN_NOT_OK(builder.Reserve(num_rows));
-
- auto column_block = _block.column_block(_cur_field_idx);
- for (size_t i = 0; i < num_rows; ++i) {
- if (column_block.is_null(i)) {
- ARROW_RETURN_NOT_OK(builder.AppendNull());
- } else {
- auto cell_ptr = column_block.cell_ptr(i);
- ARROW_RETURN_NOT_OK(builder.Append(*(typename T::c_type*)cell_ptr));
- }
- }
- return builder.Finish(&_arrays[_cur_field_idx]);
- }
-
-private:
- const RowBlockV2& _block;
- std::shared_ptr<arrow::Schema> _schema;
- arrow::MemoryPool* _pool;
-
- size_t _cur_field_idx = 0;
- std::vector<std::shared_ptr<arrow::Array>> _arrays;
-};
-
-Status FromRowBlockConverter::convert(std::shared_ptr<arrow::RecordBatch>* out) {
- size_t num_fields = _schema->num_fields();
- if (num_fields != _block.schema()->num_column_ids()) {
- return Status::InvalidArgument("Schema not match");
- }
-
- _arrays.resize(num_fields);
- for (int idx = 0; idx < num_fields; ++idx) {
- _cur_field_idx = idx;
- auto arrow_st = arrow::VisitTypeInline(*_schema->field(idx)->type(), this);
- if (!arrow_st.ok()) {
- return to_status(arrow_st);
- }
- }
- *out = arrow::RecordBatch::Make(_schema, _block.num_rows(), std::move(_arrays));
- return Status::OK();
-}
-
-Status convert_to_arrow_batch(const RowBlockV2& block, const std::shared_ptr<arrow::Schema>& schema,
- arrow::MemoryPool* pool,
- std::shared_ptr<arrow::RecordBatch>* result) {
- FromRowBlockConverter converter(block, schema, pool);
- return converter.convert(result);
-}
-
-// Convert Arrow RecordBatch to Doris RowBlockV2
-class ToRowBlockConverter : public arrow::ArrayVisitor {
-public:
- ToRowBlockConverter(const arrow::RecordBatch& batch, const Schema& schema)
- : _batch(batch), _schema(schema) {}
-
- ~ToRowBlockConverter() override {}
-
- using arrow::ArrayVisitor::Visit;
-
-#define PRIMITIVE_VISIT(TYPE) \
- arrow::Status Visit(const arrow::TYPE& array) override { return _visit(array); }
-
- PRIMITIVE_VISIT(Int8Array);
- PRIMITIVE_VISIT(Int16Array);
- PRIMITIVE_VISIT(Int32Array);
- PRIMITIVE_VISIT(Int64Array);
- PRIMITIVE_VISIT(FloatArray);
- PRIMITIVE_VISIT(DoubleArray);
-
-#undef PRIMITIVE_VISIT
-
- Status convert(std::shared_ptr<RowBlockV2>* result);
-
-private:
- template <typename T>
- typename std::enable_if<std::is_base_of<arrow::PrimitiveCType, typename T::TypeClass>::value,
- arrow::Status>::type
- _visit(const T& array) {
- auto raw_values = array.raw_values();
- auto column_block = _output->column_block(_cur_field_idx);
- for (size_t idx = 0; idx < array.length(); ++idx) {
- if (array.IsValid(idx)) {
- auto cell_ptr = column_block.mutable_cell_ptr(idx);
- column_block.set_is_null(idx, false);
- *(typename T::TypeClass::c_type*)cell_ptr = raw_values[idx];
- } else {
- column_block.set_is_null(idx, true);
- }
- }
- return arrow::Status::OK();
- }
-
-private:
- const arrow::RecordBatch& _batch;
- const Schema& _schema;
-
- size_t _cur_field_idx;
-
- std::shared_ptr<RowBlockV2> _output;
-};
-
-Status ToRowBlockConverter::convert(std::shared_ptr<RowBlockV2>* result) {
- size_t num_fields = _schema.num_column_ids();
- if (_batch.schema()->num_fields() != num_fields) {
- return Status::InvalidArgument("Schema not match");
- }
-
- auto num_rows = _batch.num_rows();
- _output.reset(new RowBlockV2(_schema, num_rows));
- for (int idx = 0; idx < num_fields; ++idx) {
- _cur_field_idx = idx;
- auto arrow_st = arrow::VisitArrayInline(*_batch.column(idx), this);
- if (!arrow_st.ok()) {
- return to_status(arrow_st);
- }
- }
- _output->set_num_rows(num_rows);
- *result = std::move(_output);
- return Status::OK();
-}
-
} // namespace doris
diff --git a/be/src/util/arrow/row_block.h b/be/src/util/arrow/row_block.h
index 80e75217b7..012a31426c 100644
--- a/be/src/util/arrow/row_block.h
+++ b/be/src/util/arrow/row_block.h
@@ -21,9 +21,6 @@
#include "common/status.h"
-// Convert Doris RowBlockV2 to/from Arrow RecordBatch.
-// RowBlockV2 is used in Doris storage engine.
-
namespace arrow {
class Schema;
@@ -33,8 +30,6 @@ class RecordBatch;
} // namespace arrow
namespace doris {
-
-class RowBlockV2;
class Schema;
// Convert Doris Schema to Arrow Schema.
@@ -43,10 +38,4 @@ Status convert_to_arrow_schema(const Schema& row_desc, std::shared_ptr<arrow::Sc
// Convert Arrow Schema to Doris Schema.
Status convert_to_doris_schema(const arrow::Schema& schema, std::shared_ptr<Schema>* result);
-// Convert a Doris RowBlockV2 to an Arrow RecordBatch. A valid Arrow Schema
-// who should match RowBlockV2's schema is given. Memory used by result RecordBatch
-// will be allocated from input pool.
-Status convert_to_arrow_batch(const RowBlockV2& block, const std::shared_ptr<arrow::Schema>& schema,
- arrow::MemoryPool* pool, std::shared_ptr<arrow::RecordBatch>* result);
-
} // namespace doris
diff --git a/be/src/vec/olap/vgeneric_iterators.h b/be/src/vec/olap/vgeneric_iterators.h
index e5bb36d0d3..e8a265f9eb 100644
--- a/be/src/vec/olap/vgeneric_iterators.h
+++ b/be/src/vec/olap/vgeneric_iterators.h
@@ -17,7 +17,6 @@
#include "olap/iterators.h"
#include "olap/row.h"
-#include "olap/row_block2.h"
#include "olap/rowset/segment_v2/column_reader.h"
namespace doris {
diff --git a/be/test/olap/block_column_predicate_test.cpp b/be/test/olap/block_column_predicate_test.cpp
index 9ae68fa242..49d7c30e5a 100644
--- a/be/test/olap/block_column_predicate_test.cpp
+++ b/be/test/olap/block_column_predicate_test.cpp
@@ -23,7 +23,6 @@
#include "olap/column_predicate.h"
#include "olap/comparison_predicate.h"
#include "olap/field.h"
-#include "olap/row_block2.h"
#include "olap/wrapper_field.h"
#include "runtime/mem_pool.h"
#include "runtime/string_value.hpp"
diff --git a/be/test/runtime/array_test.cpp b/be/test/runtime/array_test.cpp
index 43294006ab..8953578ef5 100644
--- a/be/test/runtime/array_test.cpp
+++ b/be/test/runtime/array_test.cpp
@@ -31,7 +31,6 @@
#include "io/fs/file_writer.h"
#include "io/fs/local_file_system.h"
#include "olap/field.h"
-#include "olap/row_block2.h"
#include "olap/rowset/segment_v2/column_reader.h"
#include "olap/rowset/segment_v2/column_writer.h"
#include "olap/tablet_schema.h"
@@ -275,39 +274,6 @@ private:
EXPECT_TRUE(file_writer->close().ok());
}
- {
- auto reader = create_column_reader(path, meta, arrays.size());
- EXPECT_NE(reader, nullptr);
- auto rblock = create_readable_block(path);
- EXPECT_NE(rblock, nullptr);
- OlapReaderStatistics stats;
- std::unique_ptr<segment_v2::ColumnIterator> iter(
- new_iterator(rblock.get(), &stats, reader.get()));
- EXPECT_NE(iter, nullptr);
- auto st = iter->seek_to_first();
- EXPECT_TRUE(st.ok()) << st.to_string();
-
- RowBlockV2 block(schema, 1024);
- auto col = block.column_block(0);
- int index = 0;
- size_t rows_read = 1024;
- size_t num_rows = 0;
- do {
- ColumnBlockView dst(&col);
- st = iter->next_batch(&rows_read, &dst);
- EXPECT_TRUE(st.ok());
- num_rows += rows_read;
- for (int i = 0; i < rows_read; ++i) {
- validate(field, arrays[index++],
- reinterpret_cast<const CollectionValue*>(col.cell_ptr(i)));
- }
- EXPECT_TRUE(st.ok());
- } while (rows_read >= 1024);
- auto type_info = get_type_info(column_pb);
- auto tuple_desc = get_tuple_descriptor(_object_pool, type_info.get());
- block.set_selected_size(num_rows);
- test_convert_to_vec_block(block, tuple_desc, field, arrays);
- }
{
auto type_info = get_type_info(column_pb);
auto tuple_desc = get_tuple_descriptor(_object_pool, type_info.get());
@@ -433,24 +399,6 @@ private:
test_write_and_read_column<array_encoding, item_encoding>(column_pb, field, {array});
}
- void test_convert_to_vec_block(RowBlockV2& row_block, const TupleDescriptor* tuple_desc,
- const Field* field,
- const std::vector<const CollectionValue*>& arrays) {
- vectorized::Block block;
- for (const auto slot_desc : tuple_desc->slots()) {
- block.insert(vectorized::ColumnWithTypeAndName(slot_desc->get_empty_mutable_column(),
- slot_desc->get_data_type_ptr(),
- slot_desc->col_name()));
- }
-
- row_block.convert_to_vec_block(&block);
- for (int i = 0; i < arrays.size(); ++i) {
- auto tuple = block.deep_copy_tuple(*tuple_desc, _mem_pool.get(), i, 0, false);
- auto actual = tuple->get_collection_slot(tuple_desc->slots().front()->tuple_offset());
- validate(field, arrays[i], actual);
- }
- }
-
private:
static constexpr size_t MAX_MEMORY_BYTES = 1024 * 1024;
static const std::string TEST_DIR;
diff --git a/be/test/tools/benchmark_tool.cpp b/be/test/tools/benchmark_tool.cpp
index 7b6a296e13..1bc3decb4b 100644
--- a/be/test/tools/benchmark_tool.cpp
+++ b/be/test/tools/benchmark_tool.cpp
@@ -40,7 +40,6 @@
#include "olap/data_dir.h"
#include "olap/in_list_predicate.h"
#include "olap/olap_common.h"
-#include "olap/row_block2.h"
#include "olap/row_cursor.h"
#include "olap/rowset/segment_v2/binary_dict_page.h"
#include "olap/rowset/segment_v2/binary_plain_page.h"
@@ -467,6 +466,8 @@ public:
read_opts.stats = &stats;
std::unique_ptr<RowwiseIterator> iter;
_segment->new_iterator(get_schema(), read_opts, &iter);
+ // Need modify this case
+ /*
RowBlockV2 block(get_schema(), 1024);
int left = _dataset.size();
@@ -476,6 +477,7 @@ public:
iter->next_batch(&block);
left -= rows_read;
}
+ */
}
private:
@@ -512,6 +514,8 @@ public:
read_opts.stats = &stats;
std::unique_ptr<RowwiseIterator> iter;
_segment->new_iterator(get_schema(), read_opts, &iter);
+ // Need modify this case
+ /*
RowBlockV2 block(get_schema(), 1024);
int left = _dataset.size();
@@ -521,6 +525,7 @@ public:
iter->next_batch(&block);
left -= rows_read;
}
+ */
}
private:
diff --git a/be/test/vec/exec/vgeneric_iterators_test.cpp b/be/test/vec/exec/vgeneric_iterators_test.cpp
index 51b0630f49..598cde284b 100644
--- a/be/test/vec/exec/vgeneric_iterators_test.cpp
+++ b/be/test/vec/exec/vgeneric_iterators_test.cpp
@@ -22,7 +22,6 @@
#include <vector>
#include "olap/olap_common.h"
-#include "olap/row_block2.h"
#include "olap/schema.h"
#include "util/slice.h"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org