You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by li...@apache.org on 2025/11/06 13:22:52 UTC
(doris) branch master updated: [Refactor](exec) remove unless time var in block (#57737)
This is an automated email from the ASF dual-hosted git repository.
lihaopeng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 030531a5888 [Refactor](exec) remove unless time var in block (#57737)
030531a5888 is described below
commit 030531a58887883405553f3f299234a4435654d7
Author: HappenLee <ha...@selectdb.com>
AuthorDate: Thu Nov 6 21:22:45 2025 +0800
[Refactor](exec) remove unless time var in block (#57737)
### What problem does this PR solve?
Remove unless time var in block:
```
int64_t _decompress_time_ns = 0;
int64_t _decompressed_bytes = 0;
mutable int64_t _compress_time_ns = 0;
```
---
be/src/exec/rowid_fetcher.cpp | 16 +++-
be/src/pipeline/exec/materialization_opertor.cpp | 5 +-
be/src/runtime/tablets_channel.cpp | 4 +-
.../arrow_flight/arrow_flight_batch_reader.cpp | 5 +-
.../aggregate_functions/aggregate_function_sort.h | 8 +-
be/src/vec/core/block.cpp | 12 ++-
be/src/vec/core/block.h | 14 +--
be/src/vec/exec/format/wal/wal_reader.cpp | 4 +-
be/src/vec/runtime/vdata_stream_recvr.cpp | 4 +-
be/src/vec/runtime/vdata_stream_recvr.h | 7 +-
be/src/vec/sink/varrow_flight_result_writer.cpp | 8 +-
be/src/vec/sink/vdata_stream_sender.cpp | 5 +-
be/src/vec/sink/writer/vtablet_writer.cpp | 3 +-
be/src/vec/sink/writer/vwal_writer.cpp | 3 +-
be/src/vec/spill/spill_reader.cpp | 4 +-
be/src/vec/spill/spill_writer.cpp | 3 +-
be/test/olap/wal/wal_reader_writer_test.cpp | 10 +-
be/test/pipeline/exec/vdata_stream_recvr_test.cpp | 3 +-
.../operator/materialization_shared_state_test.cpp | 18 ++--
be/test/testutil/mock/mock_data_stream_sender.h | 10 +-
be/test/vec/core/block_test.cpp | 102 +++++++++++++++------
be/test/vec/data_types/common_data_type_test.h | 7 +-
22 files changed, 176 insertions(+), 79 deletions(-)
diff --git a/be/src/exec/rowid_fetcher.cpp b/be/src/exec/rowid_fetcher.cpp
index 189923f974c..6902286548c 100644
--- a/be/src/exec/rowid_fetcher.cpp
+++ b/be/src/exec/rowid_fetcher.cpp
@@ -182,7 +182,11 @@ Status RowIDFetcher::_merge_rpc_results(const PMultiGetRequest& request,
}
// Merge partial blocks
vectorized::Block partial_block;
- RETURN_IF_ERROR(partial_block.deserialize(resp.block()));
+ [[maybe_unused]] size_t uncompressed_size = 0;
+ [[maybe_unused]] int64_t uncompressed_time = 0;
+
+ RETURN_IF_ERROR(
+ partial_block.deserialize(resp.block(), &uncompressed_size, &uncompressed_time));
if (partial_block.is_empty_column()) {
return Status::OK();
}
@@ -493,9 +497,10 @@ Status RowIdStorageReader::read_by_rowids(const PMultiGetRequest& request,
<< ", be_exec_version:" << request.be_exec_version();
[[maybe_unused]] size_t compressed_size = 0;
[[maybe_unused]] size_t uncompressed_size = 0;
+ [[maybe_unused]] int64_t compress_time = 0;
int be_exec_version = request.has_be_exec_version() ? request.be_exec_version() : 0;
RETURN_IF_ERROR(result_block.serialize(be_exec_version, response->mutable_block(),
- &uncompressed_size, &compressed_size,
+ &uncompressed_size, &compressed_size, &compress_time,
segment_v2::CompressionTypePB::LZ4));
}
@@ -599,10 +604,11 @@ Status RowIdStorageReader::read_by_rowids(const PMultiGetRequestV2& request,
[[maybe_unused]] size_t compressed_size = 0;
[[maybe_unused]] size_t uncompressed_size = 0;
+ [[maybe_unused]] int64_t compress_time = 0;
int be_exec_version = request.has_be_exec_version() ? request.be_exec_version() : 0;
- RETURN_IF_ERROR(result_blocks[i].serialize(be_exec_version, pblock->mutable_block(),
- &uncompressed_size, &compressed_size,
- segment_v2::CompressionTypePB::LZ4));
+ RETURN_IF_ERROR(result_blocks[i].serialize(
+ be_exec_version, pblock->mutable_block(), &uncompressed_size, &compressed_size,
+ &compress_time, segment_v2::CompressionTypePB::LZ4));
}
// Build file type statistics string
diff --git a/be/src/pipeline/exec/materialization_opertor.cpp b/be/src/pipeline/exec/materialization_opertor.cpp
index 63fcd7b47a6..689d345dd0a 100644
--- a/be/src/pipeline/exec/materialization_opertor.cpp
+++ b/be/src/pipeline/exec/materialization_opertor.cpp
@@ -59,8 +59,11 @@ Status MaterializationSharedState::merge_multi_response() {
for (int i = 0; i < block_order_results.size(); ++i) {
for (auto& [backend_id, rpc_struct] : rpc_struct_map) {
vectorized::Block partial_block;
+ size_t uncompressed_size = 0;
+ int64_t uncompressed_time = 0;
DCHECK(rpc_struct.response.blocks_size() > i);
- RETURN_IF_ERROR(partial_block.deserialize(rpc_struct.response.blocks(i).block()));
+ RETURN_IF_ERROR(partial_block.deserialize(rpc_struct.response.blocks(i).block(),
+ &uncompressed_size, &uncompressed_time));
if (rpc_struct.response.blocks(i).has_profile()) {
auto response_profile =
RuntimeProfile::from_proto(rpc_struct.response.blocks(i).profile());
diff --git a/be/src/runtime/tablets_channel.cpp b/be/src/runtime/tablets_channel.cpp
index 8ebb6bf1aec..0f0b66a46cb 100644
--- a/be/src/runtime/tablets_channel.cpp
+++ b/be/src/runtime/tablets_channel.cpp
@@ -569,7 +569,9 @@ Status BaseTabletsChannel::_write_block_data(
std::unordered_map<int64_t, DorisVector<uint32_t>>& tablet_to_rowidxs,
PTabletWriterAddBlockResult* response) {
vectorized::Block send_data;
- RETURN_IF_ERROR(send_data.deserialize(request.block()));
+ [[maybe_unused]] size_t uncompressed_size = 0;
+ [[maybe_unused]] int64_t uncompressed_time = 0;
+ RETURN_IF_ERROR(send_data.deserialize(request.block(), &uncompressed_size, &uncompressed_time));
CHECK(send_data.rows() == request.tablet_ids_size())
<< "block rows: " << send_data.rows()
<< ", tablet_ids_size: " << request.tablet_ids_size();
diff --git a/be/src/service/arrow_flight/arrow_flight_batch_reader.cpp b/be/src/service/arrow_flight/arrow_flight_batch_reader.cpp
index 07e46cfcfed..24fc977d027 100644
--- a/be/src/service/arrow_flight/arrow_flight_batch_reader.cpp
+++ b/be/src/service/arrow_flight/arrow_flight_batch_reader.cpp
@@ -268,7 +268,10 @@ arrow::Status ArrowFlightBatchRemoteReader::_fetch_data() {
{
SCOPED_ATOMIC_TIMER(&_deserialize_block_timer);
_block = vectorized::Block::create_shared();
- st = _block->deserialize(callback->response_->block());
+ [[maybe_unused]] size_t uncompressed_size = 0;
+ [[maybe_unused]] int64_t uncompressed_time = 0;
+ st = _block->deserialize(callback->response_->block(), &uncompressed_size,
+ &uncompressed_time);
ARROW_RETURN_NOT_OK(to_arrow_status(st));
break;
}
diff --git a/be/src/vec/aggregate_functions/aggregate_function_sort.h b/be/src/vec/aggregate_functions/aggregate_function_sort.h
index 79fdf4979b1..3d8bc2480c2 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_sort.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_sort.h
@@ -72,8 +72,10 @@ struct AggregateFunctionSortData {
PBlock pblock;
size_t uncompressed_bytes = 0;
size_t compressed_bytes = 0;
+ int64_t compressed_time = 0;
auto st = block.serialize(state->be_exec_version(), &pblock, &uncompressed_bytes,
- &compressed_bytes, segment_v2::CompressionTypePB::NO_COMPRESSION);
+ &compressed_bytes, &compressed_time,
+ segment_v2::CompressionTypePB::NO_COMPRESSION);
if (!st.ok()) {
throw doris::Exception(st);
}
@@ -87,7 +89,9 @@ struct AggregateFunctionSortData {
PBlock pblock;
pblock.ParseFromString(data);
- auto st = block.deserialize(pblock);
+ [[maybe_unused]] size_t uncompressed_size = 0;
+ [[maybe_unused]] int64_t uncompressed_time = 0;
+ auto st = block.deserialize(pblock, &uncompressed_size, &uncompressed_time);
// If memory allocate failed during deserialize, st is not ok, throw exception here to
// stop the query.
if (!st.ok()) {
diff --git a/be/src/vec/core/block.cpp b/be/src/vec/core/block.cpp
index 9690e13e1ea..5914ffe3f31 100644
--- a/be/src/vec/core/block.cpp
+++ b/be/src/vec/core/block.cpp
@@ -114,7 +114,8 @@ Block::Block(const std::vector<SlotDescriptor>& slots, size_t block_size,
*this = Block(slot_ptrs, block_size, ignore_trivial_slot);
}
-Status Block::deserialize(const PBlock& pblock) {
+Status Block::deserialize(const PBlock& pblock, size_t* uncompressed_bytes,
+ int64_t* decompress_time) {
swap(Block());
int be_exec_version = pblock.has_be_exec_version() ? pblock.be_exec_version() : 0;
RETURN_IF_ERROR(BeExecVersionManager::check_be_exec_version(be_exec_version));
@@ -123,7 +124,7 @@ Status Block::deserialize(const PBlock& pblock) {
std::string compression_scratch;
if (pblock.compressed()) {
// Decompress
- SCOPED_RAW_TIMER(&_decompress_time_ns);
+ SCOPED_RAW_TIMER(decompress_time);
const char* compressed_data = pblock.column_values().c_str();
size_t compressed_size = pblock.column_values().size();
size_t uncompressed_size = 0;
@@ -146,7 +147,7 @@ Status Block::deserialize(const PBlock& pblock) {
compression_scratch.data());
DCHECK(success) << "snappy::RawUncompress failed";
}
- _decompressed_bytes = uncompressed_size;
+ *uncompressed_bytes = uncompressed_size;
buf = compression_scratch.data();
} else {
buf = pblock.column_values().data();
@@ -960,7 +961,8 @@ Status Block::filter_block(Block* block, size_t filter_column_id, size_t column_
Status Block::serialize(int be_exec_version, PBlock* pblock,
/*std::string* compressed_buffer,*/ size_t* uncompressed_bytes,
- size_t* compressed_bytes, segment_v2::CompressionTypePB compression_type,
+ size_t* compressed_bytes, int64_t* compress_time,
+ segment_v2::CompressionTypePB compression_type,
bool allow_transfer_large_data) const {
RETURN_IF_ERROR(BeExecVersionManager::check_be_exec_version(be_exec_version));
pblock->set_be_exec_version(be_exec_version);
@@ -1000,7 +1002,7 @@ Status Block::serialize(int be_exec_version, PBlock* pblock,
// compress
if (compression_type != segment_v2::NO_COMPRESSION && content_uncompressed_size > 0) {
- SCOPED_RAW_TIMER(&_compress_time_ns);
+ SCOPED_RAW_TIMER(compress_time);
pblock->set_compression_type(compression_type);
pblock->set_uncompressed_size(serialize_bytes);
diff --git a/be/src/vec/core/block.h b/be/src/vec/core/block.h
index b3ecb58ebed..c347a4036d9 100644
--- a/be/src/vec/core/block.h
+++ b/be/src/vec/core/block.h
@@ -77,11 +77,6 @@ private:
Container data;
IndexByName index_by_name;
- int64_t _decompress_time_ns = 0;
- int64_t _decompressed_bytes = 0;
-
- mutable int64_t _compress_time_ns = 0;
-
public:
Block() = default;
Block(std::initializer_list<ColumnWithTypeAndName> il);
@@ -291,10 +286,11 @@ public:
// serialize block to PBlock
Status serialize(int be_exec_version, PBlock* pblock, size_t* uncompressed_bytes,
- size_t* compressed_bytes, segment_v2::CompressionTypePB compression_type,
+ size_t* compressed_bytes, int64_t* compress_time,
+ segment_v2::CompressionTypePB compression_type,
bool allow_transfer_large_data = false) const;
- Status deserialize(const PBlock& pblock);
+ Status deserialize(const PBlock& pblock, size_t* uncompressed_bytes, int64_t* decompress_time);
std::unique_ptr<Block> create_same_struct_block(size_t size, bool is_reserve = false) const;
@@ -362,10 +358,6 @@ public:
// for String type or Array<String> type
void shrink_char_type_column_suffix_zero(const std::vector<size_t>& char_type_idx);
- int64_t get_decompress_time() const { return _decompress_time_ns; }
- int64_t get_decompressed_bytes() const { return _decompressed_bytes; }
- int64_t get_compress_time() const { return _compress_time_ns; }
-
// remove tmp columns in block
// in inverted index apply logic, in order to optimize query performance,
// we built some temporary columns into block
diff --git a/be/src/vec/exec/format/wal/wal_reader.cpp b/be/src/vec/exec/format/wal/wal_reader.cpp
index bb9b4318e9c..2390f210d3e 100644
--- a/be/src/vec/exec/format/wal/wal_reader.cpp
+++ b/be/src/vec/exec/format/wal/wal_reader.cpp
@@ -60,7 +60,9 @@ Status WalReader::get_next_block(Block* block, size_t* read_rows, bool* eof) {
_wal_path);
}
Block src_block;
- RETURN_IF_ERROR(src_block.deserialize(pblock));
+ size_t uncompressed_size = 0;
+ int64_t uncompressed_time = 0;
+ RETURN_IF_ERROR(src_block.deserialize(pblock, &uncompressed_size, &uncompressed_time));
//convert to dst block
Block dst_block;
int index = 0;
diff --git a/be/src/vec/runtime/vdata_stream_recvr.cpp b/be/src/vec/runtime/vdata_stream_recvr.cpp
index 4b2429b8792..dd86476f483 100644
--- a/be/src/vec/runtime/vdata_stream_recvr.cpp
+++ b/be/src/vec/runtime/vdata_stream_recvr.cpp
@@ -106,8 +106,8 @@ Status VDataStreamRecvr::SenderQueue::get_batch(Block* block, bool* eos) {
RETURN_IF_ERROR(block_item.get_block(next_block));
size_t block_byte_size = block_item.block_byte_size();
COUNTER_UPDATE(_recvr->_deserialize_row_batch_timer, block_item.deserialize_time());
- COUNTER_UPDATE(_recvr->_decompress_timer, block->get_decompress_time());
- COUNTER_UPDATE(_recvr->_decompress_bytes, block->get_decompressed_bytes());
+ COUNTER_UPDATE(_recvr->_decompress_timer, block_item.decompress_time());
+ COUNTER_UPDATE(_recvr->_decompress_bytes, block_item.decompress_bytes());
_recvr->_memory_used_counter->update(-(int64_t)block_byte_size);
INJECT_MOCK_SLEEP(std::lock_guard<std::mutex> l(_lock));
sub_blocks_memory_usage(block_byte_size);
diff --git a/be/src/vec/runtime/vdata_stream_recvr.h b/be/src/vec/runtime/vdata_stream_recvr.h
index 1d752b1bad8..1a1c84f3e67 100644
--- a/be/src/vec/runtime/vdata_stream_recvr.h
+++ b/be/src/vec/runtime/vdata_stream_recvr.h
@@ -271,7 +271,8 @@ protected:
DCHECK(_pblock);
SCOPED_RAW_TIMER(&_deserialize_time);
_block = Block::create_unique();
- RETURN_IF_ERROR_OR_CATCH_EXCEPTION(_block->deserialize(*_pblock));
+ RETURN_IF_ERROR_OR_CATCH_EXCEPTION(
+ _block->deserialize(*_pblock, &_decompress_bytes, &_decompress_time));
}
block.swap(_block);
_block.reset();
@@ -280,6 +281,8 @@ protected:
size_t block_byte_size() const { return _block_byte_size; }
int64_t deserialize_time() const { return _deserialize_time; }
+ int64_t decompress_time() const { return _decompress_time; }
+ size_t decompress_bytes() const { return _decompress_bytes; }
BlockItem() = default;
BlockItem(BlockUPtr&& block, size_t block_byte_size)
: _block(std::move(block)), _block_byte_size(block_byte_size) {}
@@ -292,6 +295,8 @@ protected:
std::unique_ptr<PBlock> _pblock;
size_t _block_byte_size = 0;
int64_t _deserialize_time = 0;
+ int64_t _decompress_time = 0;
+ size_t _decompress_bytes = 0;
};
std::list<BlockItem> _block_queue;
diff --git a/be/src/vec/sink/varrow_flight_result_writer.cpp b/be/src/vec/sink/varrow_flight_result_writer.cpp
index 9105ba9e057..b8280785546 100644
--- a/be/src/vec/sink/varrow_flight_result_writer.cpp
+++ b/be/src/vec/sink/varrow_flight_result_writer.cpp
@@ -47,10 +47,12 @@ Status GetArrowResultBatchCtx::on_data(const std::shared_ptr<vectorized::Block>&
if (_result != nullptr) {
auto* arrow_buffer = assert_cast<ArrowFlightResultBlockBuffer*>(buffer);
size_t uncompressed_bytes = 0, compressed_bytes = 0;
+ int64_t compressed_time = 0;
SCOPED_TIMER(arrow_buffer->_serialize_batch_ns_timer);
- RETURN_IF_ERROR(block->serialize(
- arrow_buffer->_be_exec_version, _result->mutable_block(), &uncompressed_bytes,
- &compressed_bytes, arrow_buffer->_fragment_transmission_compression_type, false));
+ RETURN_IF_ERROR(block->serialize(arrow_buffer->_be_exec_version, _result->mutable_block(),
+ &uncompressed_bytes, &compressed_bytes, &compressed_time,
+ arrow_buffer->_fragment_transmission_compression_type,
+ false));
COUNTER_UPDATE(arrow_buffer->_uncompressed_bytes_counter, uncompressed_bytes);
COUNTER_UPDATE(arrow_buffer->_compressed_bytes_counter, compressed_bytes);
_result->set_packet_seq(packet_seq);
diff --git a/be/src/vec/sink/vdata_stream_sender.cpp b/be/src/vec/sink/vdata_stream_sender.cpp
index b4af185adb3..a05ce66ff98 100644
--- a/be/src/vec/sink/vdata_stream_sender.cpp
+++ b/be/src/vec/sink/vdata_stream_sender.cpp
@@ -347,12 +347,13 @@ Status BlockSerializer::serialize_block(const Block* src, PBlock* dest, size_t n
SCOPED_TIMER(_parent->_serialize_batch_timer);
dest->Clear();
size_t uncompressed_bytes = 0, compressed_bytes = 0;
+ int64_t compress_time = 0;
RETURN_IF_ERROR(src->serialize(_parent->_state->be_exec_version(), dest, &uncompressed_bytes,
- &compressed_bytes, _parent->compression_type(),
+ &compressed_bytes, &compress_time, _parent->compression_type(),
_parent->transfer_large_data_by_brpc()));
COUNTER_UPDATE(_parent->_bytes_sent_counter, compressed_bytes * num_receivers);
COUNTER_UPDATE(_parent->_uncompressed_bytes_counter, uncompressed_bytes * num_receivers);
- COUNTER_UPDATE(_parent->_compress_timer, src->get_compress_time());
+ COUNTER_UPDATE(_parent->_compress_timer, compress_time);
#ifndef BE_TEST
_parent->state()->get_query_ctx()->resource_ctx()->io_context()->update_shuffle_send_bytes(
compressed_bytes * num_receivers);
diff --git a/be/src/vec/sink/writer/vtablet_writer.cpp b/be/src/vec/sink/writer/vtablet_writer.cpp
index c7abe1baa14..fb864435cdf 100644
--- a/be/src/vec/sink/writer/vtablet_writer.cpp
+++ b/be/src/vec/sink/writer/vtablet_writer.cpp
@@ -917,8 +917,9 @@ void VNodeChannel::try_send_pending_block(RuntimeState* state) {
if (block.rows() > 0) {
SCOPED_ATOMIC_TIMER(&_serialize_batch_ns);
size_t uncompressed_bytes = 0, compressed_bytes = 0;
+ int64_t compressed_time = 0;
Status st = block.serialize(state->be_exec_version(), request->mutable_block(),
- &uncompressed_bytes, &compressed_bytes,
+ &uncompressed_bytes, &compressed_bytes, &compressed_time,
state->fragement_transmission_compression_type(),
_parent->_transfer_large_data_by_brpc);
TEST_INJECTION_POINT_CALLBACK("VNodeChannel::try_send_block", &st);
diff --git a/be/src/vec/sink/writer/vwal_writer.cpp b/be/src/vec/sink/writer/vwal_writer.cpp
index 5a64b6f7c43..35436da82f1 100644
--- a/be/src/vec/sink/writer/vwal_writer.cpp
+++ b/be/src/vec/sink/writer/vwal_writer.cpp
@@ -73,8 +73,9 @@ Status VWalWriter::write_wal(vectorized::Block* block) {
{ return Status::InternalError("Failed to write wal!"); });
PBlock pblock;
size_t uncompressed_bytes = 0, compressed_bytes = 0;
+ int64_t compressed_time = 0;
RETURN_IF_ERROR(block->serialize(_be_exe_version, &pblock, &uncompressed_bytes,
- &compressed_bytes,
+ &compressed_bytes, &compressed_time,
segment_v2::CompressionTypePB::NO_COMPRESSION));
RETURN_IF_ERROR(_wal_writer->append_blocks(std::vector<PBlock*> {&pblock}));
return Status::OK();
diff --git a/be/src/vec/spill/spill_reader.cpp b/be/src/vec/spill/spill_reader.cpp
index de3eea6b625..bb74d20ec33 100644
--- a/be/src/vec/spill/spill_reader.cpp
+++ b/be/src/vec/spill/spill_reader.cpp
@@ -142,7 +142,9 @@ Status SpillReader::read(Block* block, bool* eos) {
if (!pb_block_.ParseFromArray(result.data, cast_set<int>(result.size))) {
return Status::InternalError("Failed to read spilled block");
}
- RETURN_IF_ERROR(block->deserialize(pb_block_));
+ size_t uncompressed_size = 0;
+ int64_t uncompressed_time = 0;
+ RETURN_IF_ERROR(block->deserialize(pb_block_, &uncompressed_size, &uncompressed_time));
}
COUNTER_UPDATE(_read_block_data_size, block->bytes());
COUNTER_UPDATE(_read_rows_count, block->rows());
diff --git a/be/src/vec/spill/spill_writer.cpp b/be/src/vec/spill/spill_writer.cpp
index a43ba83ccb9..a43162f43ce 100644
--- a/be/src/vec/spill/spill_writer.cpp
+++ b/be/src/vec/spill/spill_writer.cpp
@@ -115,9 +115,10 @@ Status SpillWriter::_write_internal(const Block& block, size_t& written_bytes) {
{
PBlock pblock;
SCOPED_TIMER(_serialize_timer);
+ int64_t compressed_time = 0;
status = block.serialize(
BeExecVersionManager::get_newest_version(), &pblock, &uncompressed_bytes,
- &compressed_bytes,
+ &compressed_bytes, &compressed_time,
segment_v2::CompressionTypePB::ZSTD); // ZSTD for better compression ratio
RETURN_IF_ERROR(status);
int64_t pblock_mem = pblock.ByteSizeLong();
diff --git a/be/test/olap/wal/wal_reader_writer_test.cpp b/be/test/olap/wal/wal_reader_writer_test.cpp
index 94fc8ff91a2..d443cdfa849 100644
--- a/be/test/olap/wal/wal_reader_writer_test.cpp
+++ b/be/test/olap/wal/wal_reader_writer_test.cpp
@@ -64,8 +64,10 @@ void covert_block_to_pb(
segment_v2::CompressionTypePB compression_type = segment_v2::CompressionTypePB::SNAPPY) {
size_t uncompressed_bytes = 0;
size_t compressed_bytes = 0;
- Status st = block.serialize(BeExecVersionManager::get_newest_version(), pblock,
- &uncompressed_bytes, &compressed_bytes, compression_type);
+ int64_t compressed_time = 0;
+ Status st =
+ block.serialize(BeExecVersionManager::get_newest_version(), pblock, &uncompressed_bytes,
+ &compressed_bytes, &compressed_time, compression_type);
EXPECT_TRUE(st.ok());
EXPECT_TRUE(uncompressed_bytes >= compressed_bytes);
EXPECT_EQ(compressed_bytes, pblock->column_values().size());
@@ -132,7 +134,9 @@ TEST_F(WalReaderWriterTest, TestWriteAndRead1) {
break;
}
vectorized::Block block;
- EXPECT_TRUE(block.deserialize(pblock).ok());
+ size_t uncompress_size = 0;
+ int64_t uncompressed_time = 0;
+ EXPECT_TRUE(block.deserialize(pblock, &uncompress_size, &uncompressed_time).ok());
EXPECT_EQ(block_rows, block.rows());
}
static_cast<void>(wal_reader.finalize());
diff --git a/be/test/pipeline/exec/vdata_stream_recvr_test.cpp b/be/test/pipeline/exec/vdata_stream_recvr_test.cpp
index bed8be3e4f3..0bf6691d7d0 100644
--- a/be/test/pipeline/exec/vdata_stream_recvr_test.cpp
+++ b/be/test/pipeline/exec/vdata_stream_recvr_test.cpp
@@ -318,8 +318,9 @@ class MockClosure : public google::protobuf::Closure {
void to_pblock(Block& block, PBlock* pblock) {
size_t uncompressed_bytes = 0;
size_t compressed_bytes = 0;
+ int64_t compressed_time = 0;
EXPECT_TRUE(block.serialize(BeExecVersionManager::get_newest_version(), pblock,
- &uncompressed_bytes, &compressed_bytes,
+ &uncompressed_bytes, &compressed_bytes, &compressed_time,
segment_v2::CompressionTypePB::NO_COMPRESSION));
}
diff --git a/be/test/pipeline/operator/materialization_shared_state_test.cpp b/be/test/pipeline/operator/materialization_shared_state_test.cpp
index 51c097207f1..d96653b2dfe 100644
--- a/be/test/pipeline/operator/materialization_shared_state_test.cpp
+++ b/be/test/pipeline/operator/materialization_shared_state_test.cpp
@@ -119,8 +119,9 @@ TEST_F(MaterializationSharedStateTest, TestMergeMultiResponse) {
auto serialized_block = response_.add_blocks()->mutable_block();
size_t uncompressed_size = 0;
size_t compressed_size = 0;
+ int64_t compress_time = 0;
auto s = resp_block1.serialize(0, serialized_block, &uncompressed_size, &compressed_size,
- CompressionTypePB::LZ4);
+ &compress_time, CompressionTypePB::LZ4);
EXPECT_TRUE(s.ok());
_shared_state->rpc_struct_map[_backend_id1].response = std::move(response_);
@@ -141,8 +142,9 @@ TEST_F(MaterializationSharedStateTest, TestMergeMultiResponse) {
auto serialized_block = response_.add_blocks()->mutable_block();
size_t uncompressed_size = 0;
size_t compressed_size = 0;
+ int64_t compress_time = 0;
auto s = resp_block2.serialize(0, serialized_block, &uncompressed_size, &compressed_size,
- CompressionTypePB::LZ4);
+ &compress_time, CompressionTypePB::LZ4);
EXPECT_TRUE(s.ok());
_shared_state->rpc_struct_map[_backend_id2].response = std::move(response_);
@@ -219,8 +221,9 @@ TEST_F(MaterializationSharedStateTest, TestMergeMultiResponseMultiBlocks) {
auto serialized_block = response_.add_blocks()->mutable_block();
size_t uncompressed_size = 0;
size_t compressed_size = 0;
+ int64_t compress_time = 0;
auto s = resp_block1.serialize(0, serialized_block, &uncompressed_size, &compressed_size,
- CompressionTypePB::LZ4);
+ &compress_time, CompressionTypePB::LZ4);
EXPECT_TRUE(s.ok());
_shared_state->rpc_struct_map[_backend_id1].response = std::move(response_);
@@ -240,8 +243,9 @@ TEST_F(MaterializationSharedStateTest, TestMergeMultiResponseMultiBlocks) {
auto serialized_block = response_.add_blocks()->mutable_block();
size_t uncompressed_size = 0;
size_t compressed_size = 0;
+ int64_t compress_time = 0;
auto s = resp_block2.serialize(0, serialized_block, &uncompressed_size, &compressed_size,
- CompressionTypePB::LZ4);
+ &compress_time, CompressionTypePB::LZ4);
EXPECT_TRUE(s.ok());
_shared_state->rpc_struct_map[_backend_id2].response = std::move(response_);
@@ -260,8 +264,9 @@ TEST_F(MaterializationSharedStateTest, TestMergeMultiResponseMultiBlocks) {
_shared_state->rpc_struct_map[_backend_id1].response.add_blocks()->mutable_block();
size_t uncompressed_size = 0;
size_t compressed_size = 0;
+ int64_t compress_time = 0;
auto s = resp_block1.serialize(0, serialized_block, &uncompressed_size, &compressed_size,
- CompressionTypePB::LZ4);
+ &compress_time, CompressionTypePB::LZ4);
EXPECT_TRUE(s.ok());
_shared_state->response_blocks[1] = resp_block1.clone_empty();
}
@@ -278,8 +283,9 @@ TEST_F(MaterializationSharedStateTest, TestMergeMultiResponseMultiBlocks) {
_shared_state->rpc_struct_map[_backend_id2].response.add_blocks()->mutable_block();
size_t uncompressed_size = 0;
size_t compressed_size = 0;
+ int64_t compress_time = 0;
auto s = resp_block2.serialize(0, serialized_block, &uncompressed_size, &compressed_size,
- CompressionTypePB::LZ4);
+ &compress_time, CompressionTypePB::LZ4);
EXPECT_TRUE(s.ok());
}
diff --git a/be/test/testutil/mock/mock_data_stream_sender.h b/be/test/testutil/mock/mock_data_stream_sender.h
index dc9d2542beb..cd076cca521 100644
--- a/be/test/testutil/mock/mock_data_stream_sender.h
+++ b/be/test/testutil/mock/mock_data_stream_sender.h
@@ -54,7 +54,10 @@ struct MockChannel : public Channel {
return Status::OK();
}
Block nblock;
- RETURN_IF_ERROR_OR_CATCH_EXCEPTION(nblock.deserialize(*_pblock));
+ size_t uncompress_size = 0;
+ int64_t uncompressed_time = 0;
+ RETURN_IF_ERROR_OR_CATCH_EXCEPTION(
+ nblock.deserialize(*_pblock, &uncompress_size, &uncompressed_time));
if (!nblock.empty()) {
RETURN_IF_ERROR(_send_block.merge(std::move(nblock)));
}
@@ -64,7 +67,10 @@ struct MockChannel : public Channel {
Status send_broadcast_block(std::shared_ptr<BroadcastPBlockHolder>& block,
bool eos = false) override {
Block nblock;
- RETURN_IF_ERROR_OR_CATCH_EXCEPTION(nblock.deserialize(*block->get_block()));
+ size_t uncompress_size = 0;
+ int64_t uncompressed_time = 0;
+ RETURN_IF_ERROR_OR_CATCH_EXCEPTION(
+ nblock.deserialize(*block->get_block(), &uncompress_size, &uncompressed_time));
if (!nblock.empty()) {
RETURN_IF_ERROR(_send_block.merge(std::move(nblock)));
}
diff --git a/be/test/vec/core/block_test.cpp b/be/test/vec/core/block_test.cpp
index 07e4254c7fc..001ff8710ef 100644
--- a/be/test/vec/core/block_test.cpp
+++ b/be/test/vec/core/block_test.cpp
@@ -74,8 +74,10 @@ void block_to_pb(
segment_v2::CompressionTypePB compression_type = segment_v2::CompressionTypePB::SNAPPY) {
size_t uncompressed_bytes = 0;
size_t compressed_bytes = 0;
- Status st = block.serialize(BeExecVersionManager::get_newest_version(), pblock,
- &uncompressed_bytes, &compressed_bytes, compression_type);
+ int64_t compress_time = 0;
+ Status st =
+ block.serialize(BeExecVersionManager::get_newest_version(), pblock, &uncompressed_bytes,
+ &compressed_bytes, &compress_time, compression_type);
EXPECT_TRUE(st.ok());
// const column maybe uncompressed_bytes<compressed_bytes
// as the serialize_bytes add some additional byets: STREAMVBYTE_PADDING=16;
@@ -147,7 +149,9 @@ void serialize_and_deserialize_test(segment_v2::CompressionTypePB compression_ty
std::string s1 = pblock.DebugString();
vectorized::Block block2;
- static_cast<void>(block2.deserialize(pblock));
+ size_t uncompress_size = 0;
+ int64_t uncompressed_time = 0;
+ static_cast<void>(block2.deserialize(pblock, &uncompress_size, &uncompressed_time));
PBlock pblock2;
block_to_pb(block2, &pblock2, compression_type);
std::string s2 = pblock2.DebugString();
@@ -169,7 +173,9 @@ void serialize_and_deserialize_test(segment_v2::CompressionTypePB compression_ty
std::string s1 = pblock.DebugString();
vectorized::Block block2;
- static_cast<void>(block2.deserialize(pblock));
+ size_t uncompress_size = 0;
+ int64_t uncompressed_time = 0;
+ static_cast<void>(block2.deserialize(pblock, &uncompress_size, &uncompressed_time));
PBlock pblock2;
block_to_pb(block2, &pblock2, compression_type);
std::string s2 = pblock2.DebugString();
@@ -192,7 +198,9 @@ void serialize_and_deserialize_test(segment_v2::CompressionTypePB compression_ty
std::string s1 = pblock.DebugString();
vectorized::Block block2;
- static_cast<void>(block2.deserialize(pblock));
+ size_t uncompress_size = 0;
+ int64_t uncompressed_time = 0;
+ static_cast<void>(block2.deserialize(pblock, &uncompress_size, &uncompressed_time));
PBlock pblock2;
block_to_pb(block2, &pblock2, compression_type);
std::string s2 = pblock2.DebugString();
@@ -219,7 +227,9 @@ void serialize_and_deserialize_test(segment_v2::CompressionTypePB compression_ty
std::string s1 = pblock.DebugString();
vectorized::Block block2;
- static_cast<void>(block2.deserialize(pblock));
+ size_t uncompress_size = 0;
+ int64_t uncompressed_time = 0;
+ static_cast<void>(block2.deserialize(pblock, &uncompress_size, &uncompressed_time));
PBlock pblock2;
block_to_pb(block2, &pblock2, compression_type);
std::string s2 = pblock2.DebugString();
@@ -240,7 +250,9 @@ void serialize_and_deserialize_test(segment_v2::CompressionTypePB compression_ty
std::string s1 = pblock.DebugString();
vectorized::Block block2;
- static_cast<void>(block2.deserialize(pblock));
+ size_t uncompress_size = 0;
+ int64_t uncompressed_time = 0;
+ static_cast<void>(block2.deserialize(pblock, &uncompress_size, &uncompressed_time));
PBlock pblock2;
block_to_pb(block2, &pblock2, compression_type);
std::string s2 = pblock2.DebugString();
@@ -263,7 +275,9 @@ void serialize_and_deserialize_test(segment_v2::CompressionTypePB compression_ty
std::string s1 = pblock.DebugString();
vectorized::Block block2;
- static_cast<void>(block2.deserialize(pblock));
+ size_t uncompress_size = 0;
+ int64_t uncompressed_time = 0;
+ static_cast<void>(block2.deserialize(pblock, &uncompress_size, &uncompressed_time));
PBlock pblock2;
block_to_pb(block2, &pblock2, compression_type);
std::string s2 = pblock2.DebugString();
@@ -287,7 +301,9 @@ void serialize_and_deserialize_test(segment_v2::CompressionTypePB compression_ty
std::string s1 = pblock.DebugString();
vectorized::Block block2;
- static_cast<void>(block2.deserialize(pblock));
+ size_t uncompress_size = 0;
+ int64_t uncompressed_time = 0;
+ static_cast<void>(block2.deserialize(pblock, &uncompress_size, &uncompressed_time));
PBlock pblock2;
block_to_pb(block2, &pblock2, compression_type);
std::string s2 = pblock2.DebugString();
@@ -303,7 +319,9 @@ void serialize_and_deserialize_test(segment_v2::CompressionTypePB compression_ty
std::string s1 = pblock.DebugString();
vectorized::Block block2;
- static_cast<void>(block2.deserialize(pblock));
+ size_t uncompress_size = 0;
+ int64_t uncompressed_time = 0;
+ static_cast<void>(block2.deserialize(pblock, &uncompress_size, &uncompressed_time));
PBlock pblock2;
block_to_pb(block2, &pblock2, compression_type);
std::string s2 = pblock2.DebugString();
@@ -328,7 +346,9 @@ void serialize_and_deserialize_test_one() {
std::string s1 = pblock.DebugString();
vectorized::Block block2;
- static_cast<void>(block2.deserialize(pblock));
+ size_t uncompress_size = 0;
+ int64_t uncompressed_time = 0;
+ static_cast<void>(block2.deserialize(pblock, &uncompress_size, &uncompressed_time));
PBlock pblock2;
block_to_pb(block2, &pblock2, segment_v2::CompressionTypePB::LZ4);
std::string s2 = pblock2.DebugString();
@@ -354,7 +374,9 @@ void serialize_and_deserialize_test_int() {
std::string s1 = pblock.DebugString();
vectorized::Block block2;
- static_cast<void>(block2.deserialize(pblock));
+ size_t uncompress_size = 0;
+ int64_t uncompressed_time = 0;
+ static_cast<void>(block2.deserialize(pblock, &uncompress_size, &uncompressed_time));
PBlock pblock2;
block_to_pb(block2, &pblock2, segment_v2::CompressionTypePB::LZ4);
std::string s2 = pblock2.DebugString();
@@ -378,7 +400,9 @@ void serialize_and_deserialize_test_int() {
std::string s1 = pblock.DebugString();
vectorized::Block block2;
- static_cast<void>(block2.deserialize(pblock));
+ size_t uncompress_size = 0;
+ int64_t uncompressed_time = 0;
+ static_cast<void>(block2.deserialize(pblock, &uncompress_size, &uncompressed_time));
PBlock pblock2;
block_to_pb(block2, &pblock2, segment_v2::CompressionTypePB::LZ4);
std::string s2 = pblock2.DebugString();
@@ -403,7 +427,9 @@ void serialize_and_deserialize_test_long() {
std::string s1 = pblock.DebugString();
vectorized::Block block2;
- static_cast<void>(block2.deserialize(pblock));
+ size_t uncompress_size = 0;
+ int64_t uncompressed_time = 0;
+ static_cast<void>(block2.deserialize(pblock, &uncompress_size, &uncompressed_time));
PBlock pblock2;
block_to_pb(block2, &pblock2, segment_v2::CompressionTypePB::LZ4);
std::string s2 = pblock2.DebugString();
@@ -427,7 +453,9 @@ void serialize_and_deserialize_test_long() {
std::string s1 = pblock.DebugString();
vectorized::Block block2;
- static_cast<void>(block2.deserialize(pblock));
+ size_t uncompress_size = 0;
+ int64_t uncompressed_time = 0;
+ static_cast<void>(block2.deserialize(pblock, &uncompress_size, &uncompressed_time));
PBlock pblock2;
block_to_pb(block2, &pblock2, segment_v2::CompressionTypePB::LZ4);
std::string s2 = pblock2.DebugString();
@@ -452,7 +480,9 @@ void serialize_and_deserialize_test_string() {
std::string s1 = pblock.DebugString();
vectorized::Block block2;
- static_cast<void>(block2.deserialize(pblock));
+ size_t uncompress_size = 0;
+ int64_t uncompressed_time = 0;
+ static_cast<void>(block2.deserialize(pblock, &uncompress_size, &uncompressed_time));
PBlock pblock2;
block_to_pb(block2, &pblock2, segment_v2::CompressionTypePB::SNAPPY);
std::string s2 = pblock2.DebugString();
@@ -477,7 +507,9 @@ void serialize_and_deserialize_test_string() {
std::string s1 = pblock.DebugString();
vectorized::Block block2;
- static_cast<void>(block2.deserialize(pblock));
+ size_t uncompress_size = 0;
+ int64_t uncompressed_time = 0;
+ static_cast<void>(block2.deserialize(pblock, &uncompress_size, &uncompressed_time));
PBlock pblock2;
block_to_pb(block2, &pblock2, segment_v2::CompressionTypePB::SNAPPY);
std::string s2 = pblock2.DebugString();
@@ -505,7 +537,9 @@ void serialize_and_deserialize_test_nullable() {
std::string s1 = pblock.DebugString();
vectorized::Block block2;
- static_cast<void>(block2.deserialize(pblock));
+ size_t uncompress_size = 0;
+ int64_t uncompressed_time = 0;
+ static_cast<void>(block2.deserialize(pblock, &uncompress_size, &uncompressed_time));
PBlock pblock2;
block_to_pb(block2, &pblock2, segment_v2::CompressionTypePB::LZ4);
std::string s2 = pblock2.DebugString();
@@ -530,7 +564,9 @@ void serialize_and_deserialize_test_nullable() {
std::string s1 = pblock.DebugString();
vectorized::Block block2;
- static_cast<void>(block2.deserialize(pblock));
+ size_t uncompress_size = 0;
+ int64_t uncompressed_time = 0;
+ static_cast<void>(block2.deserialize(pblock, &uncompress_size, &uncompressed_time));
PBlock pblock2;
block_to_pb(block2, &pblock2, segment_v2::CompressionTypePB::LZ4);
std::string s2 = pblock2.DebugString();
@@ -554,7 +590,9 @@ void serialize_and_deserialize_test_nullable() {
std::string s1 = pblock.DebugString();
vectorized::Block block2;
- static_cast<void>(block2.deserialize(pblock));
+ size_t uncompress_size = 0;
+ int64_t uncompressed_time = 0;
+ static_cast<void>(block2.deserialize(pblock, &uncompress_size, &uncompressed_time));
PBlock pblock2;
block_to_pb(block2, &pblock2, segment_v2::CompressionTypePB::SNAPPY);
std::string s2 = pblock2.DebugString();
@@ -576,7 +614,9 @@ void serialize_and_deserialize_test_nullable() {
std::string s1 = pblock.DebugString();
vectorized::Block block2;
- static_cast<void>(block2.deserialize(pblock));
+ size_t uncompress_size = 0;
+ int64_t uncompressed_time = 0;
+ static_cast<void>(block2.deserialize(pblock, &uncompress_size, &uncompressed_time));
PBlock pblock2;
block_to_pb(block2, &pblock2, segment_v2::CompressionTypePB::SNAPPY);
std::string s2 = pblock2.DebugString();
@@ -602,7 +642,9 @@ void serialize_and_deserialize_test_decimal() {
std::string s1 = pblock.DebugString();
vectorized::Block block2;
- static_cast<void>(block2.deserialize(pblock));
+ size_t uncompress_size = 0;
+ int64_t uncompressed_time = 0;
+ static_cast<void>(block2.deserialize(pblock, &uncompress_size, &uncompressed_time));
PBlock pblock2;
block_to_pb(block2, &pblock2, segment_v2::CompressionTypePB::LZ4);
std::string s2 = pblock2.DebugString();
@@ -626,7 +668,9 @@ void serialize_and_deserialize_test_decimal() {
std::string s1 = pblock.DebugString();
vectorized::Block block2;
- static_cast<void>(block2.deserialize(pblock));
+ size_t uncompress_size = 0;
+ int64_t uncompressed_time = 0;
+ static_cast<void>(block2.deserialize(pblock, &uncompress_size, &uncompressed_time));
PBlock pblock2;
block_to_pb(block2, &pblock2, segment_v2::CompressionTypePB::LZ4);
std::string s2 = pblock2.DebugString();
@@ -658,7 +702,9 @@ void serialize_and_deserialize_test_bitmap() {
std::string s1 = pblock.DebugString();
std::string bb1 = block.dump_data(0, 1024);
vectorized::Block block2;
- static_cast<void>(block2.deserialize(pblock));
+ size_t uncompress_size = 0;
+ int64_t uncompressed_time = 0;
+ static_cast<void>(block2.deserialize(pblock, &uncompress_size, &uncompressed_time));
std::string bb2 = block2.dump_data(0, 1024);
EXPECT_EQ(bb1, bb2);
PBlock pblock2;
@@ -688,7 +734,9 @@ void serialize_and_deserialize_test_bitmap() {
std::string s1 = pblock.DebugString();
std::string bb1 = block.dump_data(0, 1024);
vectorized::Block block2;
- static_cast<void>(block2.deserialize(pblock));
+ size_t uncompress_size = 0;
+ int64_t uncompressed_time = 0;
+ static_cast<void>(block2.deserialize(pblock, &uncompress_size, &uncompressed_time));
std::string bb2 = block2.dump_data(0, 1024);
EXPECT_EQ(bb1, bb2);
EXPECT_EQ(block.dump_data_json(0, 1024), block2.dump_data_json(0, 1024));
@@ -709,7 +757,9 @@ void serialize_and_deserialize_test_array() {
block_to_pb(block, &pblock, segment_v2::CompressionTypePB::SNAPPY);
std::string s1 = pblock.DebugString();
vectorized::Block block2;
- static_cast<void>(block2.deserialize(pblock));
+ size_t uncompress_size = 0;
+ int64_t uncompressed_time = 0;
+ static_cast<void>(block2.deserialize(pblock, &uncompress_size, &uncompressed_time));
PBlock pblock2;
block_to_pb(block2, &pblock2, segment_v2::CompressionTypePB::SNAPPY);
std::string s2 = pblock2.DebugString();
diff --git a/be/test/vec/data_types/common_data_type_test.h b/be/test/vec/data_types/common_data_type_test.h
index 4425dd31c94..745b3d19155 100644
--- a/be/test/vec/data_types/common_data_type_test.h
+++ b/be/test/vec/data_types/common_data_type_test.h
@@ -252,13 +252,16 @@ public:
auto pblock = std::make_unique<PBlock>();
size_t uncompressed_bytes = 0;
size_t compressed_bytes = 0;
+ int64_t compress_time = 0;
segment_v2::CompressionTypePB compression_type = segment_v2::CompressionTypePB::ZSTD;
Status st = block->serialize(be_exec_version, pblock.get(), &uncompressed_bytes,
- &compressed_bytes, compression_type);
+ &compressed_bytes, &compress_time, compression_type);
ASSERT_EQ(st.ok(), true);
// deserialize
auto block_1 = std::make_shared<Block>();
- st = block_1->deserialize(*pblock);
+ size_t uncompress_size = 0;
+ int64_t uncompressed_time = 0;
+ st = block_1->deserialize(*pblock, &uncompress_size, &uncompressed_time);
ASSERT_EQ(st.ok(), true);
// check block_1 and block is same
for (auto col_idx = 0; col_idx < block->columns(); ++col_idx) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org