You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by zh...@apache.org on 2023/01/30 06:47:47 UTC
[doris] branch branch-1.2-lts updated: [cherry-pick](load) avoid schema copy to reduce cpu usage (#16154)
This is an automated email from the ASF dual-hosted git repository.
zhangchen pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-1.2-lts by this push:
new 7f1fd5f0ec [cherry-pick](load) avoid schema copy to reduce cpu usage (#16154)
7f1fd5f0ec is described below
commit 7f1fd5f0ec2d1de02e057ddbbb84be71e3f88161
Author: zhannngchen <48...@users.noreply.github.com>
AuthorDate: Mon Jan 30 14:47:40 2023 +0800
[cherry-pick](load) avoid schema copy to reduce cpu usage (#16154)
* [cherry-pick](load) avoid schema copy to reduce cpu usage
* fix ut
---
be/src/olap/delta_writer.cpp | 24 +++++++++++-----------
be/src/olap/delta_writer.h | 4 ++--
be/src/olap/tablet_schema.cpp | 23 ++++++++++-----------
be/src/olap/tablet_schema.h | 4 ++--
be/src/runtime/tablets_channel.cpp | 2 +-
be/test/olap/delta_writer_test.cpp | 24 ++++++++++++++--------
.../olap/engine_storage_migration_task_test.cpp | 6 ++++--
be/test/olap/remote_rowset_gc_test.cpp | 6 ++++--
be/test/olap/tablet_cooldown_test.cpp | 6 ++++--
9 files changed, 55 insertions(+), 44 deletions(-)
diff --git a/be/src/olap/delta_writer.cpp b/be/src/olap/delta_writer.cpp
index ab5d955089..ca47cde07e 100644
--- a/be/src/olap/delta_writer.cpp
+++ b/be/src/olap/delta_writer.cpp
@@ -17,6 +17,7 @@
#include "olap/delta_writer.h"
+#include "exec/tablet_info.h"
#include "olap/base_compaction.h"
#include "olap/cumulative_compaction.h"
#include "olap/data_dir.h"
@@ -132,8 +133,7 @@ Status DeltaWriter::init() {
_req.txn_id, _req.load_id));
}
// build tablet schema in request level
- _build_current_tablet_schema(_req.index_id, _req.ptable_schema_param,
- *_tablet->tablet_schema());
+ _build_current_tablet_schema(_req.index_id, _req.table_schema_param, *_tablet->tablet_schema());
RETURN_NOT_OK(_tablet->create_rowset_writer(_req.txn_id, _req.load_id, PREPARED, OVERLAPPING,
_tablet_schema, &_rowset_writer));
@@ -472,22 +472,22 @@ int64_t DeltaWriter::partition_id() const {
}
void DeltaWriter::_build_current_tablet_schema(int64_t index_id,
- const POlapTableSchemaParam& ptable_schema_param,
+ const OlapTableSchemaParam* table_schema_param,
const TabletSchema& ori_tablet_schema) {
_tablet_schema->copy_from(ori_tablet_schema);
-
// find the right index id
int i = 0;
- for (; i < ptable_schema_param.indexes_size(); i++) {
- if (ptable_schema_param.indexes(i).id() == index_id) break;
+ auto indexes = table_schema_param->indexes();
+ for (; i < indexes.size(); i++) {
+ if (indexes[i]->index_id == index_id) {
+ break;
+ }
}
- if (ptable_schema_param.indexes_size() > 0 &&
- ptable_schema_param.indexes(i).columns_desc_size() != 0 &&
- ptable_schema_param.indexes(i).columns_desc(0).unique_id() >= 0) {
- _tablet_schema->build_current_tablet_schema(index_id, ptable_schema_param.version(),
- ptable_schema_param.indexes(i),
- ori_tablet_schema);
+ if (indexes.size() > 0 && indexes[i]->columns.size() != 0 &&
+ indexes[i]->columns[0]->unique_id() >= 0) {
+ _tablet_schema->build_current_tablet_schema(index_id, table_schema_param->version(),
+ indexes[i], ori_tablet_schema);
}
if (_tablet_schema->schema_version() > ori_tablet_schema.schema_version()) {
_tablet->update_max_version_schema(_tablet_schema);
diff --git a/be/src/olap/delta_writer.h b/be/src/olap/delta_writer.h
index b7f3c0cf65..522bc98ede 100644
--- a/be/src/olap/delta_writer.h
+++ b/be/src/olap/delta_writer.h
@@ -48,7 +48,7 @@ struct WriteRequest {
// slots are in order of tablet's schema
const std::vector<SlotDescriptor*>* slots;
bool is_high_priority = false;
- POlapTableSchemaParam ptable_schema_param = {};
+ OlapTableSchemaParam* table_schema_param;
int64_t index_id = 0;
};
@@ -124,7 +124,7 @@ private:
void _reset_mem_table();
void _build_current_tablet_schema(int64_t index_id,
- const POlapTableSchemaParam& table_schema_param,
+ const OlapTableSchemaParam* table_schema_param,
const TabletSchema& ori_tablet_schema);
void _request_slave_tablet_pull_rowset(PNodeInfo node_info);
diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp
index d9c46888c1..22c031abc8 100644
--- a/be/src/olap/tablet_schema.cpp
+++ b/be/src/olap/tablet_schema.cpp
@@ -19,6 +19,7 @@
#include <gen_cpp/olap_file.pb.h>
+#include "exec/tablet_info.h"
#include "gen_cpp/descriptors.pb.h"
#include "olap/utils.h"
#include "tablet_meta.h"
@@ -617,7 +618,7 @@ std::string TabletSchema::to_key() const {
}
void TabletSchema::build_current_tablet_schema(int64_t index_id, int32_t version,
- const POlapTableIndexSchema& index,
+ const OlapTableIndexSchema* index,
const TabletSchema& ori_tablet_schema) {
// copy from ori_tablet_schema
_keys_type = ori_tablet_schema.keys_type();
@@ -642,26 +643,24 @@ void TabletSchema::build_current_tablet_schema(int64_t index_id, int32_t version
_field_name_to_index.clear();
_field_id_to_index.clear();
- for (auto& pcolumn : index.columns_desc()) {
- TabletColumn column;
- column.init_from_pb(pcolumn);
- if (column.is_key()) {
+ for (auto& column : index->columns) {
+ if (column->is_key()) {
_num_key_columns++;
}
- if (column.is_nullable()) {
+ if (column->is_nullable()) {
_num_null_columns++;
}
- if (column.is_bf_column()) {
+ if (column->is_bf_column()) {
has_bf_columns = true;
}
- if (UNLIKELY(column.name() == DELETE_SIGN)) {
+ if (UNLIKELY(column->name() == DELETE_SIGN)) {
_delete_sign_idx = _num_columns;
- } else if (UNLIKELY(column.name() == SEQUENCE_COL)) {
+ } else if (UNLIKELY(column->name() == SEQUENCE_COL)) {
_sequence_col_idx = _num_columns;
}
- _field_name_to_index[column.name()] = _num_columns;
- _field_id_to_index[column.unique_id()] = _num_columns;
- _cols.emplace_back(std::move(column));
+ _field_name_to_index[column->name()] = _num_columns;
+ _field_id_to_index[column->unique_id()] = _num_columns;
+ _cols.emplace_back(*column);
_num_columns++;
}
diff --git a/be/src/olap/tablet_schema.h b/be/src/olap/tablet_schema.h
index 6702901e83..e04d9c99df 100644
--- a/be/src/olap/tablet_schema.h
+++ b/be/src/olap/tablet_schema.h
@@ -31,7 +31,7 @@ namespace vectorized {
class Block;
}
-class POlapTableIndexSchema;
+struct OlapTableIndexSchema;
class TabletColumn {
public:
@@ -197,7 +197,7 @@ public:
vectorized::Block create_block(bool ignore_dropped_col = true) const;
void build_current_tablet_schema(int64_t index_id, int32_t version,
- const POlapTableIndexSchema& index,
+ const OlapTableIndexSchema* index,
const TabletSchema& out_tablet_schema);
// Merge columns that not exit in current schema, these column is dropped in current schema
diff --git a/be/src/runtime/tablets_channel.cpp b/be/src/runtime/tablets_channel.cpp
index 544f1022e3..36c5430e9f 100644
--- a/be/src/runtime/tablets_channel.cpp
+++ b/be/src/runtime/tablets_channel.cpp
@@ -368,7 +368,7 @@ Status TabletsChannel::_open_all_writers(const PTabletWriterOpenRequest& request
wrequest.tuple_desc = _tuple_desc;
wrequest.slots = index_slots;
wrequest.is_high_priority = _is_high_priority;
- wrequest.ptable_schema_param = request.schema();
+ wrequest.table_schema_param = _schema;
DeltaWriter* writer = nullptr;
auto st = DeltaWriter::open(&wrequest, &writer, _load_id, _is_vec);
diff --git a/be/test/olap/delta_writer_test.cpp b/be/test/olap/delta_writer_test.cpp
index ea2799fdea..16051d1adc 100644
--- a/be/test/olap/delta_writer_test.cpp
+++ b/be/test/olap/delta_writer_test.cpp
@@ -22,6 +22,7 @@
#include <string>
+#include "exec/tablet_info.h"
#include "gen_cpp/Descriptors_types.h"
#include "gen_cpp/PaloInternalService_types.h"
#include "gen_cpp/Types_types.h"
@@ -382,12 +383,13 @@ TEST_F(TestDeltaWriter, open) {
DescriptorTbl* desc_tbl = nullptr;
DescriptorTbl::create(&obj_pool, tdesc_tbl, &desc_tbl);
TupleDescriptor* tuple_desc = desc_tbl->get_tuple_descriptor(0);
+ OlapTableSchemaParam param;
PUniqueId load_id;
load_id.set_hi(0);
load_id.set_lo(0);
WriteRequest write_req = {10003, 270068375, WriteType::LOAD, 20001, 30001,
- load_id, tuple_desc, &tuple_desc->slots(), true};
+ load_id, tuple_desc, &tuple_desc->slots(), true, ¶m};
DeltaWriter* delta_writer = nullptr;
DeltaWriter::open(&write_req, &delta_writer);
EXPECT_NE(delta_writer, nullptr);
@@ -422,12 +424,13 @@ TEST_F(TestDeltaWriter, write) {
DescriptorTbl::create(&obj_pool, tdesc_tbl, &desc_tbl);
TupleDescriptor* tuple_desc = desc_tbl->get_tuple_descriptor(0);
const std::vector<SlotDescriptor*>& slots = tuple_desc->slots();
+ OlapTableSchemaParam param;
PUniqueId load_id;
load_id.set_hi(0);
load_id.set_lo(0);
- WriteRequest write_req = {10004, 270068376, WriteType::LOAD, 20002,
- 30002, load_id, tuple_desc, &(tuple_desc->slots())};
+ WriteRequest write_req = {10004, 270068376, WriteType::LOAD, 20002, 30002,
+ load_id, tuple_desc, &(tuple_desc->slots()), true, ¶m};
DeltaWriter* delta_writer = nullptr;
DeltaWriter::open(&write_req, &delta_writer);
EXPECT_NE(delta_writer, nullptr);
@@ -544,12 +547,13 @@ TEST_F(TestDeltaWriter, vec_write) {
DescriptorTbl::create(&obj_pool, tdesc_tbl, &desc_tbl);
TupleDescriptor* tuple_desc = desc_tbl->get_tuple_descriptor(0);
// const std::vector<SlotDescriptor*>& slots = tuple_desc->slots();
+ OlapTableSchemaParam param;
PUniqueId load_id;
load_id.set_hi(0);
load_id.set_lo(0);
- WriteRequest write_req = {10004, 270068376, WriteType::LOAD, 20002,
- 30002, load_id, tuple_desc, &(tuple_desc->slots())};
+ WriteRequest write_req = {10004, 270068376, WriteType::LOAD, 20002, 30002,
+ load_id, tuple_desc, &(tuple_desc->slots()), false, ¶m};
DeltaWriter* delta_writer = nullptr;
DeltaWriter::open(&write_req, &delta_writer, TUniqueId(), true);
ASSERT_NE(delta_writer, nullptr);
@@ -690,12 +694,13 @@ TEST_F(TestDeltaWriter, sequence_col) {
DescriptorTbl::create(&obj_pool, tdesc_tbl, &desc_tbl);
TupleDescriptor* tuple_desc = desc_tbl->get_tuple_descriptor(0);
const std::vector<SlotDescriptor*>& slots = tuple_desc->slots();
+ OlapTableSchemaParam param;
PUniqueId load_id;
load_id.set_hi(0);
load_id.set_lo(0);
- WriteRequest write_req = {10005, 270068377, WriteType::LOAD, 20003,
- 30003, load_id, tuple_desc, &(tuple_desc->slots())};
+ WriteRequest write_req = {10005, 270068377, WriteType::LOAD, 20003, 30003,
+ load_id, tuple_desc, &(tuple_desc->slots()), true, ¶m};
DeltaWriter* delta_writer = nullptr;
DeltaWriter::open(&write_req, &delta_writer);
EXPECT_NE(delta_writer, nullptr);
@@ -757,12 +762,13 @@ TEST_F(TestDeltaWriter, vec_sequence_col) {
DescriptorTbl* desc_tbl = nullptr;
DescriptorTbl::create(&obj_pool, tdesc_tbl, &desc_tbl);
TupleDescriptor* tuple_desc = desc_tbl->get_tuple_descriptor(0);
+ OlapTableSchemaParam param;
PUniqueId load_id;
load_id.set_hi(0);
load_id.set_lo(0);
- WriteRequest write_req = {10005, 270068377, WriteType::LOAD, 20003,
- 30003, load_id, tuple_desc, &(tuple_desc->slots())};
+ WriteRequest write_req = {10005, 270068377, WriteType::LOAD, 20003, 30003,
+ load_id, tuple_desc, &(tuple_desc->slots()), false, ¶m};
DeltaWriter* delta_writer = nullptr;
DeltaWriter::open(&write_req, &delta_writer, TUniqueId(), true);
ASSERT_NE(delta_writer, nullptr);
diff --git a/be/test/olap/engine_storage_migration_task_test.cpp b/be/test/olap/engine_storage_migration_task_test.cpp
index 8f415ec8aa..af8a3f1116 100644
--- a/be/test/olap/engine_storage_migration_task_test.cpp
+++ b/be/test/olap/engine_storage_migration_task_test.cpp
@@ -22,6 +22,7 @@
#include <string>
+#include "exec/tablet_info.h"
#include "gen_cpp/Descriptors_types.h"
#include "gen_cpp/PaloInternalService_types.h"
#include "gen_cpp/Types_types.h"
@@ -164,12 +165,13 @@ TEST_F(TestEngineStorageMigrationTask, write_and_migration) {
DescriptorTbl::create(&obj_pool, tdesc_tbl, &desc_tbl);
TupleDescriptor* tuple_desc = desc_tbl->get_tuple_descriptor(0);
const std::vector<SlotDescriptor*>& slots = tuple_desc->slots();
+ OlapTableSchemaParam param;
PUniqueId load_id;
load_id.set_hi(0);
load_id.set_lo(0);
- WriteRequest write_req = {10005, 270068377, WriteType::LOAD, 20003,
- 30003, load_id, tuple_desc, &(tuple_desc->slots())};
+ WriteRequest write_req = {10005, 270068377, WriteType::LOAD, 20003, 30003,
+ load_id, tuple_desc, &(tuple_desc->slots()), false, ¶m};
DeltaWriter* delta_writer = nullptr;
DeltaWriter::open(&write_req, &delta_writer);
EXPECT_NE(delta_writer, nullptr);
diff --git a/be/test/olap/remote_rowset_gc_test.cpp b/be/test/olap/remote_rowset_gc_test.cpp
index e1ae7f4995..670d0305d4 100644
--- a/be/test/olap/remote_rowset_gc_test.cpp
+++ b/be/test/olap/remote_rowset_gc_test.cpp
@@ -21,6 +21,7 @@
#include "common/config.h"
#include "common/status.h"
+#include "exec/tablet_info.h"
#include "gen_cpp/internal_service.pb.h"
#include "io/fs/file_system_map.h"
#include "io/fs/s3_file_system.h"
@@ -151,12 +152,13 @@ TEST_F(RemoteRowsetGcTest, normal) {
DescriptorTbl::create(&obj_pool, tdesc_tbl, &desc_tbl);
TupleDescriptor* tuple_desc = desc_tbl->get_tuple_descriptor(0);
auto& slots = tuple_desc->slots();
+ OlapTableSchemaParam param;
PUniqueId load_id;
load_id.set_hi(0);
load_id.set_lo(0);
- WriteRequest write_req = {10005, 270068377, WriteType::LOAD, 20003,
- 30003, load_id, tuple_desc, &(tuple_desc->slots())};
+ WriteRequest write_req = {10005, 270068377, WriteType::LOAD, 20003, 30003,
+ load_id, tuple_desc, &(tuple_desc->slots()), false, ¶m};
DeltaWriter* delta_writer = nullptr;
DeltaWriter::open(&write_req, &delta_writer);
ASSERT_NE(delta_writer, nullptr);
diff --git a/be/test/olap/tablet_cooldown_test.cpp b/be/test/olap/tablet_cooldown_test.cpp
index 761ffa6893..b3a8626227 100644
--- a/be/test/olap/tablet_cooldown_test.cpp
+++ b/be/test/olap/tablet_cooldown_test.cpp
@@ -21,6 +21,7 @@
#include "common/config.h"
#include "common/status.h"
+#include "exec/tablet_info.h"
#include "gen_cpp/internal_service.pb.h"
#include "io/fs/file_system_map.h"
#include "io/fs/s3_file_system.h"
@@ -150,12 +151,13 @@ TEST_F(TabletCooldownTest, normal) {
DescriptorTbl::create(&obj_pool, tdesc_tbl, &desc_tbl);
TupleDescriptor* tuple_desc = desc_tbl->get_tuple_descriptor(0);
auto& slots = tuple_desc->slots();
+ OlapTableSchemaParam param;
PUniqueId load_id;
load_id.set_hi(0);
load_id.set_lo(0);
- WriteRequest write_req = {10005, 270068377, WriteType::LOAD, 20003,
- 30003, load_id, tuple_desc, &(tuple_desc->slots())};
+ WriteRequest write_req = {10005, 270068377, WriteType::LOAD, 20003, 30003,
+ load_id, tuple_desc, &(tuple_desc->slots()), false, ¶m};
DeltaWriter* delta_writer = nullptr;
DeltaWriter::open(&write_req, &delta_writer);
ASSERT_NE(delta_writer, nullptr);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org