You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by ga...@apache.org on 2024/09/10 18:29:37 UTC
(doris) branch branch-3.0 updated (6d381380525 -> 4f4eefa9627)
This is an automated email from the ASF dual-hosted git repository.
gavinchou pushed a change to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
from 6d381380525 [fix](cloud) MS limit max aborted txn num for the same txn label (#40414) (#40606)
new bf8e88aaf60 [feature](cloud) Add lazy commit mechanism for `commit_txn` (#38243)
new 0e113802529 [feature](cloud) Support multi follower fe in cloud (#38388)
new 057c2521bd9 [feature](vault) Support alter storage vault stmt in FE (#37606)
new 5e4f83a91dd [feature](Vault) Support alter hdfs storage vault and alter vault's name (#38685)
new e03b6d1faaf [Enhancement](Group commit) Remove the redundant `iscloud` parameter (#40120)
new a284feb2c9d [fix](cloud-schema-change) Exclude merged dropped columns from projection columns (#40187)
new c0eae4da176 [feature](cloud) export fdb client busyness (#40369)
new 2f3df2d0ef3 [enhance](cloud) Add a policy when be abnormal, tablet delay switch be (#40371)
new 8e332ba11db [fix](cloud) MS limit max aborted txn num for the same txn label (#40414)
new e45fab527e7 [bugfix](iceberg)Fixed random core with writing iceberg partitioned table (#39808)
new 4f4eefa9627 [enhance](Vault) Maintain VaultName to VaultId map in FE master (#37899)
The 11 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/cloud/cloud_meta_mgr.cpp | 1 +
be/src/cloud/cloud_schema_change_job.cpp | 8 +-
be/src/cloud/config.cpp | 4 +
be/src/cloud/config.h | 3 +
.../sink/writer/iceberg/viceberg_table_writer.cpp | 51 +-
.../sink/writer/iceberg/viceberg_table_writer.h | 8 +-
cloud/src/common/bvars.cpp | 3 +
cloud/src/common/bvars.h | 2 +
cloud/src/common/config.h | 10 +
cloud/src/common/metric.cpp | 8 +-
cloud/src/meta-service/CMakeLists.txt | 1 +
cloud/src/meta-service/meta_service.cpp | 250 ++--
cloud/src/meta-service/meta_service.h | 14 +
cloud/src/meta-service/meta_service_http.cpp | 34 +-
cloud/src/meta-service/meta_service_resource.cpp | 697 ++++++++---
cloud/src/meta-service/meta_service_txn.cpp | 1248 ++++++++++++++------
cloud/src/meta-service/txn_kv.cpp | 4 +
cloud/src/meta-service/txn_kv.h | 4 +
cloud/src/meta-service/txn_lazy_committer.cpp | 476 ++++++++
cloud/src/meta-service/txn_lazy_committer.h | 68 ++
cloud/src/recycler/recycler.cpp | 282 ++++-
cloud/src/recycler/recycler.h | 10 +-
cloud/src/recycler/recycler_service.cpp | 18 +-
cloud/src/recycler/recycler_service.h | 5 +-
cloud/src/resource-manager/resource_manager.cpp | 65 +-
cloud/test/CMakeLists.txt | 4 +
cloud/test/meta_service_test.cpp | 352 +++++-
cloud/test/recycler_test.cpp | 100 +-
cloud/test/txn_lazy_commit_test.cpp | 216 ++++
docker/runtime/doris-compose/cluster.py | 29 +-
docker/runtime/doris-compose/command.py | 21 +-
docker/runtime/doris-compose/resource/init_fe.sh | 17 +-
.../main/java/org/apache/doris/common/Config.java | 7 +
.../antlr4/org/apache/doris/nereids/DorisParser.g4 | 1 +
.../org/apache/doris/alter/CloudRollupJobV2.java | 1 +
.../apache/doris/alter/CloudSchemaChangeJobV2.java | 1 +
.../doris/analysis/CreateStorageVaultStmt.java | 25 +
.../org/apache/doris/catalog/HdfsStorageVault.java | 9 +
.../org/apache/doris/catalog/S3StorageVault.java | 12 +
.../org/apache/doris/catalog/StorageVault.java | 12 +
.../org/apache/doris/catalog/StorageVaultMgr.java | 166 ++-
.../doris/cloud/catalog/CloudClusterChecker.java | 34 +-
.../org/apache/doris/cloud/catalog/CloudEnv.java | 26 +-
.../cloud/catalog/CloudInstanceStatusChecker.java | 11 +
.../apache/doris/cloud/catalog/CloudReplica.java | 79 +-
.../doris/cloud/catalog/CloudTabletRebalancer.java | 23 +-
.../cloud/datasource/CloudInternalCatalog.java | 9 +-
.../cloud/planner/CloudGroupCommitPlanner.java | 14 -
.../apache/doris/cloud/rpc/MetaServiceClient.java | 9 +
.../apache/doris/cloud/rpc/MetaServiceProxy.java | 10 +
.../doris/cloud/system/CloudSystemInfoService.java | 23 +-
.../transaction/CloudGlobalTransactionMgr.java | 6 +-
.../apache/doris/datasource/InternalCatalog.java | 3 +-
.../property/constants/S3Properties.java | 32 +-
.../org/apache/doris/httpv2/rest/LoadAction.java | 11 +-
.../org/apache/doris/load/GroupCommitManager.java | 10 +-
.../doris/nereids/parser/LogicalPlanBuilder.java | 10 +
.../apache/doris/nereids/trees/plans/PlanType.java | 1 +
.../plans/commands/AlterStorageVaultCommand.java | 58 +
.../insert/OlapGroupCommitInsertExecutor.java | 2 +-
.../apache/doris/planner/GroupCommitPlanner.java | 2 +-
.../java/org/apache/doris/qe/MasterOpExecutor.java | 7 +-
.../apache/doris/service/FrontendServiceImpl.java | 2 +-
.../doris/cloud/catalog/HdfsStorageVaultTest.java | 10 +-
gensrc/proto/cloud.proto | 11 +
gensrc/thrift/FrontendService.thrift | 1 -
.../test_insert_with_many_buckets.out | 4 +
.../test_insert_with_many_partitions.out | 4 +
...est_insert_with_many_partitions_and_buckets.out | 4 +
.../org/apache/doris/regression/suite/Suite.groovy | 80 ++
.../doris/regression/suite/SuiteCluster.groovy | 23 +-
.../multi_cluster/test_multi_follower.groovy | 176 +++
.../cloud_p0/multi_cluster/test_rebalance.groovy | 2 +-
.../test_insert_with_many_buckets.groovy | 57 +
.../test_insert_with_many_partitions.groovy | 73 ++
..._insert_with_many_partitions_and_buckets.groovy | 79 ++
.../test_merge_dropped_column.groovy | 47 +
.../suites/vaults/alter/alter_hdfs.groovy | 119 ++
.../suites/vaults/alter/alter_s3.groovy | 124 ++
79 files changed, 4522 insertions(+), 921 deletions(-)
create mode 100644 cloud/src/meta-service/txn_lazy_committer.cpp
create mode 100644 cloud/src/meta-service/txn_lazy_committer.h
create mode 100644 cloud/test/txn_lazy_commit_test.cpp
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterStorageVaultCommand.java
create mode 100644 regression-test/data/large_txn_load_p1/test_insert_with_many_buckets.out
create mode 100644 regression-test/data/large_txn_load_p1/test_insert_with_many_partitions.out
create mode 100644 regression-test/data/large_txn_load_p1/test_insert_with_many_partitions_and_buckets.out
create mode 100644 regression-test/suites/cloud_p0/multi_cluster/test_multi_follower.groovy
create mode 100644 regression-test/suites/large_txn_load_p1/test_insert_with_many_buckets.groovy
create mode 100644 regression-test/suites/large_txn_load_p1/test_insert_with_many_partitions.groovy
create mode 100644 regression-test/suites/large_txn_load_p1/test_insert_with_many_partitions_and_buckets.groovy
create mode 100644 regression-test/suites/schema_change_p0/test_merge_dropped_column.groovy
create mode 100644 regression-test/suites/vaults/alter/alter_hdfs.groovy
create mode 100644 regression-test/suites/vaults/alter/alter_s3.groovy
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 09/11: [fix](cloud) MS limit max aborted txn num for the same txn label (#40414)
Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gavinchou pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 8e332ba11db2f78260f23c11373ebcda6b5070d1
Author: Lei Zhang <27...@users.noreply.github.com>
AuthorDate: Sun Sep 8 01:37:27 2024 +0800
[fix](cloud) MS limit max aborted txn num for the same txn label (#40414)
---
cloud/src/common/config.h | 3 +++
1 file changed, 3 insertions(+)
diff --git a/cloud/src/common/config.h b/cloud/src/common/config.h
index ee3c4ba81fc..1cd14adebb6 100644
--- a/cloud/src/common/config.h
+++ b/cloud/src/common/config.h
@@ -218,4 +218,7 @@ CONF_Int32(txn_lazy_max_rowsets_per_batch, "1000");
// max TabletIndexPB num for batch get
CONF_Int32(max_tablet_index_num_per_batch, "1000");
+
+// Max aborted txn num for the same label name
+CONF_mInt64(max_num_aborted_txn, "100");
} // namespace doris::cloud::config
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 10/11: [bugfix](iceberg)Fixed random core with writing iceberg partitioned table (#39808)
Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gavinchou pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit e45fab527e7996471ff117b8599534ad2d078998
Author: wuwenchi <wu...@hotmail.com>
AuthorDate: Fri Aug 23 14:05:49 2024 +0800
[bugfix](iceberg)Fixed random core with writing iceberg partitioned table (#39808)
Use local variables instead of member variables.
---
.../sink/writer/iceberg/viceberg_table_writer.cpp | 51 ++++++++++------------
.../sink/writer/iceberg/viceberg_table_writer.h | 8 ++--
2 files changed, 25 insertions(+), 34 deletions(-)
diff --git a/be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp b/be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp
index a0b9aa6d3f5..280cf8b8107 100644
--- a/be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp
+++ b/be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp
@@ -127,7 +127,7 @@ Status VIcebergTableWriter::write(RuntimeState* state, vectorized::Block& block)
auto writer_iter = _partitions_to_writers.find("");
if (writer_iter == _partitions_to_writers.end()) {
try {
- writer = _create_partition_writer(output_block, -1);
+ writer = _create_partition_writer(nullptr, -1);
} catch (doris::Exception& e) {
return e.to_status();
}
@@ -143,7 +143,7 @@ Status VIcebergTableWriter::write(RuntimeState* state, vectorized::Block& block)
}
_partitions_to_writers.erase(writer_iter);
try {
- writer = _create_partition_writer(output_block, -1, &file_name,
+ writer = _create_partition_writer(nullptr, -1, &file_name,
file_name_index + 1);
} catch (doris::Exception& e) {
return e.to_status();
@@ -162,22 +162,21 @@ Status VIcebergTableWriter::write(RuntimeState* state, vectorized::Block& block)
}
{
+ Block transformed_block;
SCOPED_RAW_TIMER(&_partition_writers_dispatch_ns);
- _transformed_block.clear();
- _transformed_block.reserve(_iceberg_partition_columns.size());
+ transformed_block.reserve(_iceberg_partition_columns.size());
for (auto& iceberg_partition_columns : _iceberg_partition_columns) {
- _transformed_block.insert(iceberg_partition_columns.partition_column_transform().apply(
+ transformed_block.insert(iceberg_partition_columns.partition_column_transform().apply(
output_block, iceberg_partition_columns.source_idx()));
}
for (int i = 0; i < output_block.rows(); ++i) {
std::optional<PartitionData> partition_data;
try {
- partition_data = _get_partition_data(_transformed_block, i);
+ partition_data = _get_partition_data(&transformed_block, i);
} catch (doris::Exception& e) {
return e.to_status();
}
std::string partition_name;
- DCHECK(partition_data.has_value());
try {
partition_name = _partition_to_path(partition_data.value());
} catch (doris::Exception& e) {
@@ -188,7 +187,7 @@ Status VIcebergTableWriter::write(RuntimeState* state, vectorized::Block& block)
const std::string* file_name, int file_name_index,
std::shared_ptr<VIcebergPartitionWriter>& writer_ptr) -> Status {
try {
- auto writer = _create_partition_writer(output_block, position, file_name,
+ auto writer = _create_partition_writer(&transformed_block, position, file_name,
file_name_index);
RETURN_IF_ERROR(writer->open(_state, _profile));
IColumn::Filter filter(output_block.rows(), 0);
@@ -344,30 +343,27 @@ std::vector<std::string> VIcebergTableWriter::_partition_values(
}
std::shared_ptr<VIcebergPartitionWriter> VIcebergTableWriter::_create_partition_writer(
- vectorized::Block& block, int position, const std::string* file_name, int file_name_index) {
+ vectorized::Block* transformed_block, int position, const std::string* file_name,
+ int file_name_index) {
auto& iceberg_table_sink = _t_sink.iceberg_table_sink;
- std::optional<PartitionData> partition_data;
- partition_data = _get_partition_data(_transformed_block, position);
- std::string partition_path;
std::vector<std::string> partition_values;
- if (partition_data.has_value()) {
- partition_path = _partition_to_path(partition_data.value());
- partition_values = _partition_values(partition_data.value());
- }
const std::string& output_path = iceberg_table_sink.output_path;
-
std::string write_path;
std::string original_write_path;
std::string target_path;
- if (partition_path.empty()) {
- original_write_path = iceberg_table_sink.original_output_path;
- target_path = output_path;
- write_path = output_path;
- } else {
+
+ if (transformed_block != nullptr) {
+ PartitionData partition_data = _get_partition_data(transformed_block, position);
+ std::string partition_path = _partition_to_path(partition_data);
+ partition_values = _partition_values(partition_data);
original_write_path =
fmt::format("{}/{}", iceberg_table_sink.original_output_path, partition_path);
target_path = fmt::format("{}/{}", output_path, partition_path);
write_path = fmt::format("{}/{}", output_path, partition_path);
+ } else {
+ original_write_path = iceberg_table_sink.original_output_path;
+ target_path = output_path;
+ write_path = output_path;
}
VIcebergPartitionWriter::WriteInfo write_info = {
@@ -390,18 +386,15 @@ std::shared_ptr<VIcebergPartitionWriter> VIcebergTableWriter::_create_partition_
iceberg_table_sink.hadoop_config);
}
-std::optional<PartitionData> VIcebergTableWriter::_get_partition_data(
- vectorized::Block& transformed_block, int position) {
- if (_iceberg_partition_columns.empty()) {
- return std::nullopt;
- }
-
+PartitionData VIcebergTableWriter::_get_partition_data(vectorized::Block* transformed_block,
+ int position) {
+ DCHECK(!_iceberg_partition_columns.empty());
std::vector<std::any> values;
values.reserve(_iceberg_partition_columns.size());
int column_idx = 0;
for (auto& iceberg_partition_column : _iceberg_partition_columns) {
const vectorized::ColumnWithTypeAndName& partition_column =
- transformed_block.get_by_position(column_idx);
+ transformed_block->get_by_position(column_idx);
const TypeDescriptor& result_type =
iceberg_partition_column.partition_column_transform().get_result_type();
auto value = _get_iceberg_partition_value(result_type, partition_column, position);
diff --git a/be/src/vec/sink/writer/iceberg/viceberg_table_writer.h b/be/src/vec/sink/writer/iceberg/viceberg_table_writer.h
index ae53d3af98e..e53c7020a68 100644
--- a/be/src/vec/sink/writer/iceberg/viceberg_table_writer.h
+++ b/be/src/vec/sink/writer/iceberg/viceberg_table_writer.h
@@ -99,10 +99,10 @@ private:
std::vector<std::string> _partition_values(const doris::iceberg::StructLike& data);
std::shared_ptr<VIcebergPartitionWriter> _create_partition_writer(
- vectorized::Block& block, int position, const std::string* file_name = nullptr,
- int file_name_index = 0);
+ vectorized::Block* transformed_block, int position,
+ const std::string* file_name = nullptr, int file_name_index = 0);
- std::optional<PartitionData> _get_partition_data(vectorized::Block& block, int position);
+ PartitionData _get_partition_data(vectorized::Block* block, int position);
std::any _get_iceberg_partition_value(const TypeDescriptor& type_desc,
const ColumnWithTypeAndName& partition_column,
@@ -129,8 +129,6 @@ private:
VExprContextSPtrs _write_output_vexpr_ctxs;
- Block _transformed_block;
-
size_t _row_count = 0;
// profile counters
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 06/11: [fix](cloud-schema-change) Exclude merged dropped columns from projection columns (#40187)
Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gavinchou pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit a284feb2c9d1fd1f8254ac1a7a403487c4368413
Author: Siyang Tang <82...@users.noreply.github.com>
AuthorDate: Fri Sep 6 00:09:23 2024 +0800
[fix](cloud-schema-change) Exclude merged dropped columns from projection columns (#40187)
## Proposed changes
schema change in cloud mode used to merge dropped columns from delete
predicates to projection columns when reading data from base tablet,
resulted in failure when reading rowsets after columns dropped.
---
be/src/cloud/cloud_schema_change_job.cpp | 8 ++--
.../test_merge_dropped_column.groovy | 47 ++++++++++++++++++++++
2 files changed, 51 insertions(+), 4 deletions(-)
diff --git a/be/src/cloud/cloud_schema_change_job.cpp b/be/src/cloud/cloud_schema_change_job.cpp
index 254a0d8d966..acab3aa80ad 100644
--- a/be/src/cloud/cloud_schema_change_job.cpp
+++ b/be/src/cloud/cloud_schema_change_job.cpp
@@ -100,6 +100,10 @@ Status CloudSchemaChangeJob::process_alter_tablet(const TAlterTabletReqV2& reque
_base_tablet_schema->update_tablet_columns(*_base_tablet->tablet_schema(), request.columns);
_new_tablet_schema = _new_tablet->tablet_schema();
+ std::vector<ColumnId> return_columns;
+ return_columns.resize(_base_tablet_schema->num_columns());
+ std::iota(return_columns.begin(), return_columns.end(), 0);
+
// delete handlers to filter out deleted rows
DeleteHandler delete_handler;
std::vector<RowsetMetaSharedPtr> delete_predicates;
@@ -112,10 +116,6 @@ Status CloudSchemaChangeJob::process_alter_tablet(const TAlterTabletReqV2& reque
}
RETURN_IF_ERROR(delete_handler.init(_base_tablet_schema, delete_predicates, base_max_version));
- std::vector<ColumnId> return_columns;
- return_columns.resize(_base_tablet_schema->num_columns());
- std::iota(return_columns.begin(), return_columns.end(), 0);
-
// reader_context is stack variables, it's lifetime MUST keep the same with rs_readers
RowsetReaderContext reader_context;
reader_context.reader_type = ReaderType::READER_ALTER_TABLE;
diff --git a/regression-test/suites/schema_change_p0/test_merge_dropped_column.groovy b/regression-test/suites/schema_change_p0/test_merge_dropped_column.groovy
new file mode 100644
index 00000000000..40aed871e76
--- /dev/null
+++ b/regression-test/suites/schema_change_p0/test_merge_dropped_column.groovy
@@ -0,0 +1,47 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_merge_dropped_column") {
+ def tableName = "test_merge_dropped_column"
+ sql """ drop table if exists ${tableName} """
+ sql """ create table ${tableName}
+ (
+ col1 SMALLINT NOT NULL,
+ col2 BIGINT NOT NULL,
+ col3 TEXT NOT NULL,
+ col4 FLOAT NOT NULL,
+ col5 TINYINT NOT NULL
+ ) DUPLICATE KEY(`col1`) DISTRIBUTED BY HASH (`col1`) BUCKETS 1
+ PROPERTIES (
+ "disable_auto_compaction"="true",
+ "replication_num" = "1"
+ );
+ """
+ sql """ INSERT INTO ${tableName} VALUES(5, 6, "wwwe", 1.2, 3) """
+ sql """ DELETE FROM ${tableName} WHERE col1 = 0 """
+
+ sql """ ALTER TABLE ${tableName} DROP COLUMN col3 """
+ sql """ INSERT INTO ${tableName} VALUES(7, 1212, 2.22, 2) """
+ // sql """ ALTER TABLE ${tableName} ADD COLUMN col3 TEXT NOT DEFAULT NULL """
+
+ sql """ ALTER TABLE ${tableName} MODIFY COLUMN col2 TEXT NOT NULL"""
+
+ waitForSchemaChangeDone {
+ sql """SHOW ALTER TABLE COLUMN WHERE IndexName='${tableName}' ORDER BY createtime DESC LIMIT 1"""
+ time 600
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 01/11: [feature](cloud) Add lazy commit mechanism for `commit_txn` (#38243)
Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gavinchou pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit bf8e88aaf6059b97464b4c797ae383e91b323850
Author: Lei Zhang <27...@users.noreply.github.com>
AuthorDate: Mon Sep 2 00:50:37 2024 +0800
[feature](cloud) Add lazy commit mechanism for `commit_txn` (#38243)
Motivation:
* In cloud mode we use `foundationdb` for storing rowset meta, when we
load data with many rowsets, `commit_txn` will failed because
`foundationdb` currently limits all transactions to be below 10 MB in
size, So we spilt `commit_txn` into several sub fdb txn to solve the
problem
How:
Now `commit_txn` main flow like this
sub txn 1:
1. update partition `VersionPB` with txn_id
2. submit async task to txn_lazy_committer
txn_lazy_committer:
sub txn 2:
convert tmp rowset meta per batch
sub txn 3:
convert tmp rowset meta per batch
......
sub txn n:
make txn visible and remove txn_id in `VersionPB`
---
be/src/cloud/cloud_meta_mgr.cpp | 1 +
be/src/cloud/config.cpp | 4 +
be/src/cloud/config.h | 3 +
cloud/src/common/config.h | 7 +
cloud/src/meta-service/CMakeLists.txt | 1 +
cloud/src/meta-service/meta_service.cpp | 250 ++--
cloud/src/meta-service/meta_service.h | 2 +
cloud/src/meta-service/meta_service_txn.cpp | 1248 ++++++++++++++------
cloud/src/meta-service/txn_lazy_committer.cpp | 476 ++++++++
cloud/src/meta-service/txn_lazy_committer.h | 68 ++
cloud/src/recycler/recycler.cpp | 282 ++++-
cloud/src/recycler/recycler.h | 10 +-
cloud/src/recycler/recycler_service.cpp | 18 +-
cloud/src/recycler/recycler_service.h | 5 +-
cloud/test/CMakeLists.txt | 4 +
cloud/test/meta_service_test.cpp | 2 +-
cloud/test/recycler_test.cpp | 100 +-
cloud/test/txn_lazy_commit_test.cpp | 216 ++++
.../main/java/org/apache/doris/common/Config.java | 4 +
.../org/apache/doris/alter/CloudRollupJobV2.java | 1 +
.../apache/doris/alter/CloudSchemaChangeJobV2.java | 1 +
.../cloud/datasource/CloudInternalCatalog.java | 3 +-
.../transaction/CloudGlobalTransactionMgr.java | 6 +-
gensrc/proto/cloud.proto | 4 +
.../test_insert_with_many_buckets.out | 4 +
.../test_insert_with_many_partitions.out | 4 +
...est_insert_with_many_partitions_and_buckets.out | 4 +
.../test_insert_with_many_buckets.groovy | 57 +
.../test_insert_with_many_partitions.groovy | 73 ++
..._insert_with_many_partitions_and_buckets.groovy | 79 ++
30 files changed, 2394 insertions(+), 543 deletions(-)
diff --git a/be/src/cloud/cloud_meta_mgr.cpp b/be/src/cloud/cloud_meta_mgr.cpp
index 4ca44f67cec..3f5f8df8de8 100644
--- a/be/src/cloud/cloud_meta_mgr.cpp
+++ b/be/src/cloud/cloud_meta_mgr.cpp
@@ -855,6 +855,7 @@ Status CloudMetaMgr::commit_txn(const StreamLoadContext& ctx, bool is_2pc) {
req.set_db_id(ctx.db_id);
req.set_txn_id(ctx.txn_id);
req.set_is_2pc(is_2pc);
+ req.set_enable_txn_lazy_commit(config::enable_cloud_txn_lazy_commit);
auto st = retry_rpc("commit txn", req, &res, &MetaService_Stub::commit_txn);
if (st.ok()) {
diff --git a/be/src/cloud/config.cpp b/be/src/cloud/config.cpp
index 82c466120e9..1b8256bf932 100644
--- a/be/src/cloud/config.cpp
+++ b/be/src/cloud/config.cpp
@@ -60,4 +60,8 @@ DEFINE_mBool(save_load_error_log_to_s3, "false");
DEFINE_mInt32(sync_load_for_tablets_thread, "32");
+DEFINE_mBool(enable_new_tablet_do_compaction, "false");
+
+DEFINE_Bool(enable_cloud_txn_lazy_commit, "false");
+
} // namespace doris::config
diff --git a/be/src/cloud/config.h b/be/src/cloud/config.h
index 02e7014801e..560c3de97cc 100644
--- a/be/src/cloud/config.h
+++ b/be/src/cloud/config.h
@@ -94,4 +94,7 @@ DECLARE_mBool(save_load_error_log_to_s3);
// the theads which sync the datas which loaded in other clusters
DECLARE_mInt32(sync_load_for_tablets_thread);
+// enable large txn lazy commit in meta-service `commit_txn`
+DECLARE_mBool(enable_cloud_txn_lazy_commit);
+
} // namespace doris::config
diff --git a/cloud/src/common/config.h b/cloud/src/common/config.h
index a28143bb705..ee3c4ba81fc 100644
--- a/cloud/src/common/config.h
+++ b/cloud/src/common/config.h
@@ -211,4 +211,11 @@ CONF_mInt64(max_s3_client_retry, "10");
// Max aborted txn num for the same label name
CONF_mInt64(max_num_aborted_txn, "100");
+CONF_Bool(enable_cloud_txn_lazy_commit, "true");
+CONF_Int32(txn_lazy_commit_rowsets_thresold, "1000");
+CONF_Int32(txn_lazy_commit_num_threads, "8");
+CONF_Int32(txn_lazy_max_rowsets_per_batch, "1000");
+
+// max TabletIndexPB num for batch get
+CONF_Int32(max_tablet_index_num_per_batch, "1000");
} // namespace doris::cloud::config
diff --git a/cloud/src/meta-service/CMakeLists.txt b/cloud/src/meta-service/CMakeLists.txt
index 33dd9b6c0ce..c7c4887a068 100644
--- a/cloud/src/meta-service/CMakeLists.txt
+++ b/cloud/src/meta-service/CMakeLists.txt
@@ -24,4 +24,5 @@ add_library(MetaService
doris_txn.cpp
mem_txn_kv.cpp
http_encode_key.cpp
+ txn_lazy_committer.cpp
)
diff --git a/cloud/src/meta-service/meta_service.cpp b/cloud/src/meta-service/meta_service.cpp
index 2213bb48387..5f3d64b24c9 100644
--- a/cloud/src/meta-service/meta_service.cpp
+++ b/cloud/src/meta-service/meta_service.cpp
@@ -75,6 +75,7 @@ MetaServiceImpl::MetaServiceImpl(std::shared_ptr<TxnKv> txn_kv,
resource_mgr_ = resource_mgr;
rate_limiter_ = rate_limiter;
rate_limiter_->init(this);
+ txn_lazy_committer_ = std::make_shared<TxnLazyCommitter>(txn_kv_);
}
MetaServiceImpl::~MetaServiceImpl() = default;
@@ -237,6 +238,7 @@ void MetaServiceImpl::get_version(::google::protobuf::RpcController* controller,
partition_version_key({instance_id, db_id, table_id, partition_id}, &ver_key);
}
+ code = MetaServiceCode::OK;
std::unique_ptr<Transaction> txn;
TxnErrorCode err = txn_kv_->create_txn(&txn);
if (err != TxnErrorCode::TXN_OK) {
@@ -265,6 +267,12 @@ void MetaServiceImpl::get_version(::google::protobuf::RpcController* controller,
msg = "malformed version value";
return;
}
+ if (!version_pb.has_version()) {
+ msg = "not found";
+ code = MetaServiceCode::VERSION_NOT_FOUND;
+ return;
+ }
+
response->set_version(version_pb.version());
response->add_version_update_time_ms(version_pb.update_time_ms());
}
@@ -329,6 +337,7 @@ void MetaServiceImpl::batch_get_version(::google::protobuf::RpcController* contr
std::vector<std::optional<std::string>> version_values;
version_keys.reserve(BATCH_SIZE);
version_values.reserve(BATCH_SIZE);
+
while ((code == MetaServiceCode::OK || code == MetaServiceCode::KV_TXN_TOO_OLD) &&
response->versions_size() < num_acquired) {
std::unique_ptr<Transaction> txn;
@@ -390,8 +399,15 @@ void MetaServiceImpl::batch_get_version(::google::protobuf::RpcController* contr
msg = "malformed version value";
break;
}
- response->add_versions(version_pb.version());
- response->add_version_update_time_ms(version_pb.update_time_ms());
+
+ if (!version_pb.has_version()) {
+ // return -1 if the target version is not exists.
+ response->add_versions(-1);
+ response->add_version_update_time_ms(-1);
+ } else {
+ response->add_versions(version_pb.version());
+ response->add_version_update_time_ms(version_pb.update_time_ms());
+ }
}
}
}
@@ -403,9 +419,9 @@ void MetaServiceImpl::batch_get_version(::google::protobuf::RpcController* contr
}
}
-void internal_create_tablet(MetaServiceCode& code, std::string& msg,
- const doris::TabletMetaCloudPB& meta, std::shared_ptr<TxnKv> txn_kv,
- const std::string& instance_id,
+void internal_create_tablet(const CreateTabletsRequest* request, MetaServiceCode& code,
+ std::string& msg, const doris::TabletMetaCloudPB& meta,
+ std::shared_ptr<TxnKv> txn_kv, const std::string& instance_id,
std::set<std::pair<int64_t, int32_t>>& saved_schema) {
doris::TabletMetaCloudPB tablet_meta(meta);
bool has_first_rowset = tablet_meta.rs_metas_size() > 0;
@@ -496,7 +512,9 @@ void internal_create_tablet(MetaServiceCode& code, std::string& msg,
MetaTabletIdxKeyInfo key_info1 {instance_id, tablet_id};
meta_tablet_idx_key(key_info1, &key1);
TabletIndexPB tablet_table;
- // tablet_table.set_db_id(db_id);
+ if (request->has_db_id()) {
+ tablet_table.set_db_id(request->db_id());
+ }
tablet_table.set_table_id(table_id);
tablet_table.set_index_id(index_id);
tablet_table.set_partition_id(partition_id);
@@ -618,11 +636,12 @@ void MetaServiceImpl::create_tablets(::google::protobuf::RpcController* controll
msg = fmt::format("failed to get vault id, vault name={}", name);
return;
}
+
// [index_id, schema_version]
std::set<std::pair<int64_t, int32_t>> saved_schema;
TEST_SYNC_POINT_RETURN_WITH_VOID("create_tablets");
for (auto& tablet_meta : request->tablet_metas()) {
- internal_create_tablet(code, msg, tablet_meta, txn_kv_, instance_id, saved_schema);
+ internal_create_tablet(request, code, msg, tablet_meta, txn_kv_, instance_id, saved_schema);
if (code != MetaServiceCode::OK) {
return;
}
@@ -1420,104 +1439,157 @@ void MetaServiceImpl::get_rowset(::google::protobuf::RpcController* controller,
int64_t req_cc_cnt = request->cumulative_compaction_cnt();
int64_t req_cp = request->cumulative_point();
- std::unique_ptr<Transaction> txn;
- TxnErrorCode err = txn_kv_->create_txn(&txn);
- if (err != TxnErrorCode::TXN_OK) {
- code = cast_as<ErrCategory::CREATE>(err);
- msg = "failed to create txn";
- return;
- }
+ do {
+ std::unique_ptr<Transaction> txn;
+ TxnErrorCode err = txn_kv_->create_txn(&txn);
+ if (err != TxnErrorCode::TXN_OK) {
+ code = cast_as<ErrCategory::CREATE>(err);
+ msg = "failed to create txn";
+ LOG(WARNING) << msg;
+ return;
+ }
- TabletIndexPB idx(request->idx());
- // Get tablet id index from kv
- if (!idx.has_table_id() || !idx.has_index_id() || !idx.has_partition_id()) {
+ TabletIndexPB idx;
+ // Get tablet id index from kv
get_tablet_idx(code, msg, txn.get(), instance_id, tablet_id, idx);
- if (code != MetaServiceCode::OK) return;
- }
- // TODO(plat1ko): Judge if tablet has been dropped (in dropped index/partition)
+ if (code != MetaServiceCode::OK) {
+ return;
+ }
+ DCHECK(request->has_idx());
- TabletStatsPB tablet_stat;
- internal_get_tablet_stats(code, msg, txn.get(), instance_id, idx, tablet_stat, true);
- if (code != MetaServiceCode::OK) return;
- VLOG_DEBUG << "tablet_id=" << tablet_id << " stats=" << proto_to_json(tablet_stat);
+ if (idx.has_db_id()) {
+ // there is maybe a lazy commit txn when call get_rowset
+ // we need advance lazy commit txn here
+ std::string ver_val;
+ std::string ver_key = partition_version_key(
+ {instance_id, idx.db_id(), idx.table_id(), idx.partition_id()});
+ err = txn->get(ver_key, &ver_val);
+ if (TxnErrorCode::TXN_OK != err && TxnErrorCode::TXN_KEY_NOT_FOUND != err) {
+ code = cast_as<ErrCategory::READ>(err);
+ ss << "failed to get partiton version, tablet_id=" << tablet_id
+ << " key=" << hex(ver_key) << " err=" << err;
+ msg = ss.str();
+ LOG(WARNING) << msg;
+ return;
+ }
+
+ if (TxnErrorCode::TXN_OK == err) {
+ VersionPB version_pb;
+ if (!version_pb.ParseFromString(ver_val)) {
+ code = MetaServiceCode::PROTOBUF_PARSE_ERR;
+ ss << "failed to parse version pb db_id=" << idx.db_id()
+ << " table_id=" << idx.table_id() << " partition_id" << idx.partition_id()
+ << " key=" << hex(ver_key);
+ msg = ss.str();
+ LOG(WARNING) << msg;
+ return;
+ }
+
+ if (version_pb.pending_txn_ids_size() > 0) {
+ DCHECK(version_pb.pending_txn_ids_size() == 1);
+ txn.reset();
+ std::shared_ptr<TxnLazyCommitTask> task =
+ txn_lazy_committer_->submit(instance_id, version_pb.pending_txn_ids(0));
+
+ std::tie(code, msg) = task->wait();
+ if (code != MetaServiceCode::OK) {
+ LOG(WARNING) << "advance_last_txn failed last_txn="
+ << version_pb.pending_txn_ids(0) << " code=" << code
+ << "msg=" << msg;
+ return;
+ }
+ continue;
+ }
+ }
+ }
- int64_t bc_cnt = tablet_stat.base_compaction_cnt();
- int64_t cc_cnt = tablet_stat.cumulative_compaction_cnt();
- int64_t cp = tablet_stat.cumulative_point();
+ // TODO(plat1ko): Judge if tablet has been dropped (in dropped index/partition)
- response->mutable_stats()->CopyFrom(tablet_stat);
+ TabletStatsPB tablet_stat;
+ internal_get_tablet_stats(code, msg, txn.get(), instance_id, idx, tablet_stat, true);
+ if (code != MetaServiceCode::OK) return;
+ VLOG_DEBUG << "tablet_id=" << tablet_id << " stats=" << proto_to_json(tablet_stat);
- int64_t req_start = request->start_version();
- int64_t req_end = request->end_version();
- req_end = req_end < 0 ? std::numeric_limits<int64_t>::max() - 1 : req_end;
+ int64_t bc_cnt = tablet_stat.base_compaction_cnt();
+ int64_t cc_cnt = tablet_stat.cumulative_compaction_cnt();
+ int64_t cp = tablet_stat.cumulative_point();
- //==========================================================================
- // Find version ranges to be synchronized due to compaction
- //==========================================================================
- if (req_bc_cnt > bc_cnt || req_cc_cnt > cc_cnt || req_cp > cp) {
- code = MetaServiceCode::INVALID_ARGUMENT;
- ss << "no valid compaction_cnt or cumulative_point given. req_bc_cnt=" << req_bc_cnt
- << ", bc_cnt=" << bc_cnt << ", req_cc_cnt=" << req_cc_cnt << ", cc_cnt=" << cc_cnt
- << ", req_cp=" << req_cp << ", cp=" << cp;
- msg = ss.str();
- return;
- }
- auto versions = calc_sync_versions(req_bc_cnt, bc_cnt, req_cc_cnt, cc_cnt, req_cp, cp,
- req_start, req_end);
- for (auto [start, end] : versions) {
- internal_get_rowset(txn.get(), start, end, instance_id, tablet_id, code, msg, response);
- if (code != MetaServiceCode::OK) {
- return;
- }
- }
+ response->mutable_stats()->CopyFrom(tablet_stat);
- // get referenced schema
- std::unordered_map<int32_t, doris::TabletSchemaCloudPB*> version_to_schema;
- for (auto& rowset_meta : *response->mutable_rowset_meta()) {
- if (rowset_meta.has_tablet_schema()) {
- version_to_schema.emplace(rowset_meta.tablet_schema().schema_version(),
- rowset_meta.mutable_tablet_schema());
- rowset_meta.set_schema_version(rowset_meta.tablet_schema().schema_version());
- }
- rowset_meta.set_index_id(idx.index_id());
- }
- bool need_read_schema_dict = false;
- auto arena = response->GetArena();
- for (auto& rowset_meta : *response->mutable_rowset_meta()) {
- if (rowset_meta.has_schema_dict_key_list()) {
- need_read_schema_dict = true;
- }
- if (rowset_meta.has_tablet_schema()) continue;
- if (!rowset_meta.has_schema_version()) {
+ int64_t req_start = request->start_version();
+ int64_t req_end = request->end_version();
+ req_end = req_end < 0 ? std::numeric_limits<int64_t>::max() - 1 : req_end;
+
+ //==========================================================================
+ // Find version ranges to be synchronized due to compaction
+ //==========================================================================
+ if (req_bc_cnt > bc_cnt || req_cc_cnt > cc_cnt || req_cp > cp) {
code = MetaServiceCode::INVALID_ARGUMENT;
- msg = fmt::format(
- "rowset_meta must have either schema or schema_version, "
- "rowset_version=[{}-{}]",
- rowset_meta.start_version(), rowset_meta.end_version());
+ ss << "no valid compaction_cnt or cumulative_point given. req_bc_cnt=" << req_bc_cnt
+ << ", bc_cnt=" << bc_cnt << ", req_cc_cnt=" << req_cc_cnt << ", cc_cnt=" << cc_cnt
+ << ", req_cp=" << req_cp << ", cp=" << cp;
+ msg = ss.str();
return;
}
- if (auto it = version_to_schema.find(rowset_meta.schema_version());
- it != version_to_schema.end()) {
- if (arena != nullptr) {
- rowset_meta.set_allocated_tablet_schema(it->second);
- } else {
- rowset_meta.mutable_tablet_schema()->CopyFrom(*it->second);
+ auto versions = calc_sync_versions(req_bc_cnt, bc_cnt, req_cc_cnt, cc_cnt, req_cp, cp,
+ req_start, req_end);
+ for (auto [start, end] : versions) {
+ internal_get_rowset(txn.get(), start, end, instance_id, tablet_id, code, msg, response);
+ if (code != MetaServiceCode::OK) {
+ return;
}
- } else {
- auto key = meta_schema_key({instance_id, idx.index_id(), rowset_meta.schema_version()});
- if (!try_fetch_and_parse_schema(txn.get(), rowset_meta, key, code, msg)) {
+ }
+
+ // get referenced schema
+ std::unordered_map<int32_t, doris::TabletSchemaCloudPB*> version_to_schema;
+ for (auto& rowset_meta : *response->mutable_rowset_meta()) {
+ if (rowset_meta.has_tablet_schema()) {
+ version_to_schema.emplace(rowset_meta.tablet_schema().schema_version(),
+ rowset_meta.mutable_tablet_schema());
+ rowset_meta.set_schema_version(rowset_meta.tablet_schema().schema_version());
+ }
+ rowset_meta.set_index_id(idx.index_id());
+ }
+ bool need_read_schema_dict = false;
+ auto arena = response->GetArena();
+ for (auto& rowset_meta : *response->mutable_rowset_meta()) {
+ if (rowset_meta.has_schema_dict_key_list()) {
+ need_read_schema_dict = true;
+ }
+ if (rowset_meta.has_tablet_schema()) continue;
+ if (!rowset_meta.has_schema_version()) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ msg = fmt::format(
+ "rowset_meta must have either schema or schema_version, "
+ "rowset_version=[{}-{}]",
+ rowset_meta.start_version(), rowset_meta.end_version());
return;
}
- version_to_schema.emplace(rowset_meta.schema_version(),
- rowset_meta.mutable_tablet_schema());
+ if (auto it = version_to_schema.find(rowset_meta.schema_version());
+ it != version_to_schema.end()) {
+ if (arena != nullptr) {
+ rowset_meta.set_allocated_tablet_schema(it->second);
+ } else {
+ rowset_meta.mutable_tablet_schema()->CopyFrom(*it->second);
+ }
+ } else {
+ auto key = meta_schema_key(
+ {instance_id, idx.index_id(), rowset_meta.schema_version()});
+ if (!try_fetch_and_parse_schema(txn.get(), rowset_meta, key, code, msg)) {
+ return;
+ }
+ version_to_schema.emplace(rowset_meta.schema_version(),
+ rowset_meta.mutable_tablet_schema());
+ }
}
- }
- if (need_read_schema_dict) {
- read_schema_from_dict(code, msg, instance_id, idx.index_id(), txn.get(),
- response->mutable_rowset_meta());
- if (code != MetaServiceCode::OK) return;
- }
+ if (need_read_schema_dict) {
+ read_schema_from_dict(code, msg, instance_id, idx.index_id(), txn.get(),
+ response->mutable_rowset_meta());
+ if (code != MetaServiceCode::OK) return;
+ }
+ break;
+ } while (true);
}
void MetaServiceImpl::get_tablet_stats(::google::protobuf::RpcController* controller,
diff --git a/cloud/src/meta-service/meta_service.h b/cloud/src/meta-service/meta_service.h
index c64a55b89a2..5d2ae55ffeb 100644
--- a/cloud/src/meta-service/meta_service.h
+++ b/cloud/src/meta-service/meta_service.h
@@ -30,6 +30,7 @@
#include "common/config.h"
#include "cpp/sync_point.h"
#include "meta-service/txn_kv.h"
+#include "meta-service/txn_lazy_committer.h"
#include "rate-limiter/rate_limiter.h"
#include "resource-manager/resource_manager.h"
@@ -291,6 +292,7 @@ private:
std::shared_ptr<TxnKv> txn_kv_;
std::shared_ptr<ResourceManager> resource_mgr_;
std::shared_ptr<RateLimiter> rate_limiter_;
+ std::shared_ptr<TxnLazyCommitter> txn_lazy_committer_;
};
class MetaServiceProxy final : public MetaService {
diff --git a/cloud/src/meta-service/meta_service_txn.cpp b/cloud/src/meta-service/meta_service_txn.cpp
index 63f7030e076..2c5b4d6ec9a 100644
--- a/cloud/src/meta-service/meta_service_txn.cpp
+++ b/cloud/src/meta-service/meta_service_txn.cpp
@@ -765,7 +765,7 @@ void MetaServiceImpl::reset_rl_progress(::google::protobuf::RpcController* contr
}
void scan_tmp_rowset(
- const std::string& instance_id, int64_t txn_id, std::shared_ptr<TxnKv>& txn_kv,
+ const std::string& instance_id, int64_t txn_id, std::shared_ptr<TxnKv> txn_kv,
MetaServiceCode& code, std::string& msg, int64_t* db_id,
std::vector<std::pair<std::string, doris::RowsetMetaCloudPB>>* tmp_rowsets_meta) {
// Create a readonly txn for scan tmp rowset
@@ -774,7 +774,7 @@ void scan_tmp_rowset(
TxnErrorCode err = txn_kv->create_txn(&txn);
if (err != TxnErrorCode::TXN_OK) {
code = cast_as<ErrCategory::CREATE>(err);
- ss << "filed to create txn, txn_id=" << txn_id << " err=" << err;
+ ss << "failed to create txn, txn_id=" << txn_id << " err=" << err;
msg = ss.str();
LOG(WARNING) << msg;
return;
@@ -909,6 +909,7 @@ void update_tablet_stats(const StatsTabletKeyInfo& info, const TabletStats& stat
LOG(INFO) << "put stats_tablet_key key=" << hex(key);
}
}
+
/**
* 0. Extract txn_id from request
* 1. Get db id from TxnKv with txn_id
@@ -925,94 +926,482 @@ void update_tablet_stats(const StatsTabletKeyInfo& info, const TabletStats& stat
*/
void commit_txn_immediately(
const CommitTxnRequest* request, CommitTxnResponse* response,
- std::shared_ptr<TxnKv>& txn_kv, MetaServiceCode& code, std::string& msg,
- const std::string& instance_id, int64_t db_id,
+ std::shared_ptr<TxnKv>& txn_kv, std::shared_ptr<TxnLazyCommitter>& txn_lazy_committer,
+ MetaServiceCode& code, std::string& msg, const std::string& instance_id, int64_t db_id,
std::vector<std::pair<std::string, doris::RowsetMetaCloudPB>>& tmp_rowsets_meta) {
std::stringstream ss;
int64_t txn_id = request->txn_id();
- std::unique_ptr<Transaction> txn;
- TxnErrorCode err = txn_kv->create_txn(&txn);
- if (err != TxnErrorCode::TXN_OK) {
- code = cast_as<ErrCategory::CREATE>(err);
- ss << "filed to create txn, txn_id=" << txn_id << " err=" << err;
- msg = ss.str();
- LOG(WARNING) << msg;
- return;
- }
+ do {
+ int64_t last_pending_txn_id = 0;
+ std::unique_ptr<Transaction> txn;
+ TxnErrorCode err = txn_kv->create_txn(&txn);
+ if (err != TxnErrorCode::TXN_OK) {
+ code = cast_as<ErrCategory::CREATE>(err);
+ ss << "failed to create txn, txn_id=" << txn_id << " err=" << err;
+ msg = ss.str();
+ LOG(WARNING) << msg;
+ return;
+ }
- // Get txn info with db_id and txn_id
- std::string info_val; // Will be reused when saving updated txn
- const std::string info_key = txn_info_key({instance_id, db_id, txn_id});
- err = txn->get(info_key, &info_val);
- if (err != TxnErrorCode::TXN_OK) {
- code = err == TxnErrorCode::TXN_KEY_NOT_FOUND ? MetaServiceCode::TXN_ID_NOT_FOUND
- : cast_as<ErrCategory::READ>(err);
- if (err == TxnErrorCode::TXN_KEY_NOT_FOUND) {
- ss << "transaction [" << txn_id << "] not found, db_id=" << db_id;
- } else {
- ss << "failed to get txn_info, db_id=" << db_id << " txn_id=" << txn_id
- << " err=" << err;
+ // Get txn info with db_id and txn_id
+ std::string info_val; // Will be reused when saving updated txn
+ const std::string info_key = txn_info_key({instance_id, db_id, txn_id});
+ err = txn->get(info_key, &info_val);
+ if (err != TxnErrorCode::TXN_OK) {
+ code = err == TxnErrorCode::TXN_KEY_NOT_FOUND ? MetaServiceCode::TXN_ID_NOT_FOUND
+ : cast_as<ErrCategory::READ>(err);
+ if (err == TxnErrorCode::TXN_KEY_NOT_FOUND) {
+ ss << "transaction [" << txn_id << "] not found, db_id=" << db_id;
+ } else {
+ ss << "failed to get txn_info, db_id=" << db_id << " txn_id=" << txn_id
+ << " err=" << err;
+ }
+ msg = ss.str();
+ LOG(WARNING) << msg;
+ return;
}
- msg = ss.str();
- LOG(WARNING) << msg;
- return;
- }
- TxnInfoPB txn_info;
- if (!txn_info.ParseFromString(info_val)) {
- code = MetaServiceCode::PROTOBUF_PARSE_ERR;
- ss << "failed to parse txn_info, db_id=" << db_id << " txn_id=" << txn_id;
- msg = ss.str();
- LOG(WARNING) << msg;
- return;
- }
+ TxnInfoPB txn_info;
+ if (!txn_info.ParseFromString(info_val)) {
+ code = MetaServiceCode::PROTOBUF_PARSE_ERR;
+ ss << "failed to parse txn_info, db_id=" << db_id << " txn_id=" << txn_id;
+ msg = ss.str();
+ LOG(WARNING) << msg;
+ return;
+ }
- // TODO: do more check like txn state, 2PC etc.
- DCHECK(txn_info.txn_id() == txn_id);
- if (txn_info.status() == TxnStatusPB::TXN_STATUS_ABORTED) {
- code = MetaServiceCode::TXN_ALREADY_ABORTED;
- ss << "transaction [" << txn_id << "] is already aborted, db_id=" << db_id;
- msg = ss.str();
- LOG(WARNING) << msg;
- return;
- }
+ // TODO: do more check like txn state, 2PC etc.
+ DCHECK(txn_info.txn_id() == txn_id);
+ if (txn_info.status() == TxnStatusPB::TXN_STATUS_ABORTED) {
+ code = MetaServiceCode::TXN_ALREADY_ABORTED;
+ ss << "transaction [" << txn_id << "] is already aborted, db_id=" << db_id;
+ msg = ss.str();
+ LOG(WARNING) << msg;
+ return;
+ }
- if (txn_info.status() == TxnStatusPB::TXN_STATUS_VISIBLE) {
- if (request->has_is_2pc() && request->is_2pc()) {
- code = MetaServiceCode::TXN_ALREADY_VISIBLE;
- ss << "transaction [" << txn_id << "] is already visible, not pre-committed.";
+ if (txn_info.status() == TxnStatusPB::TXN_STATUS_VISIBLE) {
+ if (request->has_is_2pc() && request->is_2pc()) {
+ code = MetaServiceCode::TXN_ALREADY_VISIBLE;
+ ss << "transaction [" << txn_id << "] is already visible, not pre-committed.";
+ msg = ss.str();
+ LOG(INFO) << msg;
+ response->mutable_txn_info()->CopyFrom(txn_info);
+ return;
+ }
+ code = MetaServiceCode::OK;
+ ss << "transaction is already visible: db_id=" << db_id << " txn_id=" << txn_id;
msg = ss.str();
LOG(INFO) << msg;
response->mutable_txn_info()->CopyFrom(txn_info);
return;
}
- code = MetaServiceCode::OK;
- ss << "transaction is already visible: db_id=" << db_id << " txn_id=" << txn_id;
- msg = ss.str();
- LOG(INFO) << msg;
- response->mutable_txn_info()->CopyFrom(txn_info);
- return;
- }
- if (request->has_is_2pc() && request->is_2pc() &&
- txn_info.status() == TxnStatusPB::TXN_STATUS_PREPARED) {
- code = MetaServiceCode::TXN_INVALID_STATUS;
- ss << "transaction is prepare, not pre-committed: db_id=" << db_id << " txn_id" << txn_id;
- msg = ss.str();
- LOG(WARNING) << msg;
- return;
- }
+ if (request->has_is_2pc() && request->is_2pc() &&
+ txn_info.status() == TxnStatusPB::TXN_STATUS_PREPARED) {
+ code = MetaServiceCode::TXN_INVALID_STATUS;
+ ss << "transaction is prepare, not pre-committed: db_id=" << db_id << " txn_id"
+ << txn_id;
+ msg = ss.str();
+ LOG(WARNING) << msg;
+ return;
+ }
- LOG(INFO) << "txn_id=" << txn_id << " txn_info=" << txn_info.ShortDebugString();
+ LOG(INFO) << "txn_id=" << txn_id << " txn_info=" << txn_info.ShortDebugString();
- // Prepare rowset meta and new_versions
+ // Prepare rowset meta and new_versions
+ // Read tablet indexes in batch.
+ std::vector<std::string> tablet_idx_keys;
+ for (auto& [_, i] : tmp_rowsets_meta) {
+ tablet_idx_keys.push_back(meta_tablet_idx_key({instance_id, i.tablet_id()}));
+ }
+ std::vector<std::optional<std::string>> tablet_idx_values;
+ err = txn->batch_get(&tablet_idx_values, tablet_idx_keys,
+ Transaction::BatchGetOptions(false));
+ if (err != TxnErrorCode::TXN_OK) {
+ code = cast_as<ErrCategory::READ>(err);
+ ss << "failed to get tablet table index ids, err=" << err;
+ msg = ss.str();
+ LOG(WARNING) << msg << " txn_id=" << txn_id;
+ return;
+ }
+
+ size_t total_rowsets = tmp_rowsets_meta.size();
+ // tablet_id -> {table/index/partition}_id
+ std::unordered_map<int64_t, TabletIndexPB> tablet_ids;
+ // table_id -> tablets_ids
+ std::unordered_map<int64_t, std::vector<int64_t>> table_id_tablet_ids;
+ for (size_t i = 0; i < total_rowsets; i++) {
+ uint64_t tablet_id = tmp_rowsets_meta[i].second.tablet_id();
+ if (!tablet_idx_values[i].has_value()) [[unlikely]] {
+ // The value must existed
+ code = MetaServiceCode::KV_TXN_GET_ERR;
+ ss << "failed to get tablet table index ids, err=not found"
+ << " tablet_id=" << tablet_id << " key=" << hex(tablet_idx_keys[i]);
+ msg = ss.str();
+ LOG(WARNING) << msg << " err=" << err << " txn_id=" << txn_id;
+ return;
+ }
+ if (!tablet_ids[tablet_id].ParseFromString(tablet_idx_values[i].value())) [[unlikely]] {
+ code = MetaServiceCode::PROTOBUF_PARSE_ERR;
+ ss << "malformed tablet index value tablet_id=" << tablet_id
+ << " txn_id=" << txn_id;
+ msg = ss.str();
+ LOG(WARNING) << msg;
+ return;
+ }
+ table_id_tablet_ids[tablet_ids[tablet_id].table_id()].push_back(tablet_id);
+ VLOG_DEBUG << "tablet_id:" << tablet_id
+ << " value:" << tablet_ids[tablet_id].ShortDebugString();
+ }
+
+ tablet_idx_keys.clear();
+ tablet_idx_values.clear();
+
+ // {table/partition} -> version
+ std::unordered_map<std::string, uint64_t> new_versions;
+ std::vector<std::string> version_keys;
+ for (auto& [_, i] : tmp_rowsets_meta) {
+ int64_t tablet_id = i.tablet_id();
+ int64_t table_id = tablet_ids[tablet_id].table_id();
+ int64_t partition_id = i.partition_id();
+ std::string ver_key =
+ partition_version_key({instance_id, db_id, table_id, partition_id});
+ if (new_versions.count(ver_key) == 0) {
+ new_versions.insert({ver_key, 0});
+ version_keys.push_back(std::move(ver_key));
+ }
+ }
+ std::vector<std::optional<std::string>> version_values;
+ err = txn->batch_get(&version_values, version_keys);
+ if (err != TxnErrorCode::TXN_OK) {
+ code = cast_as<ErrCategory::READ>(err);
+ ss << "failed to get partition versions, err=" << err;
+ msg = ss.str();
+ LOG(WARNING) << msg << " txn_id=" << txn_id;
+ return;
+ }
+ size_t total_versions = version_keys.size();
+ for (size_t i = 0; i < total_versions; i++) {
+ int64_t version;
+ if (version_values[i].has_value()) {
+ VersionPB version_pb;
+ if (!version_pb.ParseFromString(version_values[i].value())) {
+ code = MetaServiceCode::PROTOBUF_PARSE_ERR;
+ ss << "failed to parse version pb txn_id=" << txn_id
+ << " key=" << hex(version_keys[i]);
+ msg = ss.str();
+ return;
+ }
+ if (version_pb.pending_txn_ids_size() > 0) {
+ DCHECK(version_pb.pending_txn_ids_size() == 1);
+ last_pending_txn_id = version_pb.pending_txn_ids(0);
+ DCHECK(last_pending_txn_id > 0);
+ break;
+ }
+ version = version_pb.version();
+ } else {
+ version = 1;
+ }
+ new_versions[version_keys[i]] = version + 1;
+ last_pending_txn_id = 0;
+ }
+ version_keys.clear();
+ version_values.clear();
+
+ if (last_pending_txn_id > 0) {
+ txn.reset();
+ std::shared_ptr<TxnLazyCommitTask> task =
+ txn_lazy_committer->submit(instance_id, last_pending_txn_id);
+
+ std::tie(code, msg) = task->wait();
+ if (code != MetaServiceCode::OK) {
+ LOG(WARNING) << "advance_last_txn failed last_txn=" << last_pending_txn_id
+ << " code=" << code << "msg=" << msg;
+ return;
+ }
+ last_pending_txn_id = 0;
+ continue;
+ }
+
+ std::vector<std::pair<std::string, std::string>> rowsets;
+ std::unordered_map<int64_t, TabletStats> tablet_stats; // tablet_id -> stats
+ rowsets.reserve(tmp_rowsets_meta.size());
+ for (auto& [_, i] : tmp_rowsets_meta) {
+ int64_t tablet_id = i.tablet_id();
+ int64_t table_id = tablet_ids[tablet_id].table_id();
+ int64_t partition_id = i.partition_id();
+ std::string ver_key =
+ partition_version_key({instance_id, db_id, table_id, partition_id});
+ if (new_versions[ver_key] == 0) [[unlikely]] {
+ // it is impossible.
+ code = MetaServiceCode::UNDEFINED_ERR;
+ ss << "failed to get partition version key, the target version not exists in "
+ "new_versions."
+ << " txn_id=" << txn_id;
+ msg = ss.str();
+ LOG(ERROR) << msg;
+ return;
+ }
+
+ // Update rowset version
+ int64_t new_version = new_versions[ver_key];
+ i.set_start_version(new_version);
+ i.set_end_version(new_version);
+
+ std::string key = meta_rowset_key({instance_id, tablet_id, i.end_version()});
+ std::string val;
+ if (!i.SerializeToString(&val)) {
+ code = MetaServiceCode::PROTOBUF_SERIALIZE_ERR;
+ ss << "failed to serialize rowset_meta, txn_id=" << txn_id;
+ msg = ss.str();
+ return;
+ }
+ rowsets.emplace_back(std::move(key), std::move(val));
+
+ // Accumulate affected rows
+ auto& stats = tablet_stats[tablet_id];
+ stats.data_size += i.data_disk_size();
+ stats.num_rows += i.num_rows();
+ ++stats.num_rowsets;
+ stats.num_segs += i.num_segments();
+ } // for tmp_rowsets_meta
+
+ // process mow table, check lock and remove pending key
+ std::vector<std::string> lock_keys;
+ lock_keys.reserve(request->mow_table_ids().size());
+ for (auto table_id : request->mow_table_ids()) {
+ lock_keys.push_back(meta_delete_bitmap_update_lock_key({instance_id, table_id, -1}));
+ }
+ std::vector<std::optional<std::string>> lock_values;
+ err = txn->batch_get(&lock_values, lock_keys);
+ if (err != TxnErrorCode::TXN_OK) {
+ ss << "failed to get delete bitmap update lock key info, instance_id=" << instance_id
+ << " err=" << err;
+ msg = ss.str();
+ code = cast_as<ErrCategory::READ>(err);
+ LOG(WARNING) << msg << " txn_id=" << txn_id;
+ return;
+ }
+ size_t total_locks = lock_keys.size();
+ for (size_t i = 0; i < total_locks; i++) {
+ int64_t table_id = request->mow_table_ids(i);
+ // When the key does not exist, it means the lock has been acquired
+ // by another transaction and successfully committed.
+ if (!lock_values[i].has_value()) {
+ ss << "get delete bitmap update lock info, lock is expired"
+ << " table_id=" << table_id << " key=" << hex(lock_keys[i]);
+ code = MetaServiceCode::LOCK_EXPIRED;
+ msg = ss.str();
+ LOG(WARNING) << msg << " txn_id=" << txn_id;
+ return;
+ }
+
+ DeleteBitmapUpdateLockPB lock_info;
+ if (!lock_info.ParseFromString(lock_values[i].value())) [[unlikely]] {
+ code = MetaServiceCode::PROTOBUF_PARSE_ERR;
+ msg = "failed to parse DeleteBitmapUpdateLockPB";
+ LOG(WARNING) << msg << " txn_id=" << txn_id;
+ return;
+ }
+ if (lock_info.lock_id() != request->txn_id()) {
+ msg = "lock is expired";
+ code = MetaServiceCode::LOCK_EXPIRED;
+ return;
+ }
+ txn->remove(lock_keys[i]);
+ LOG(INFO) << "xxx remove delete bitmap lock, lock_key=" << hex(lock_keys[i])
+ << " txn_id=" << txn_id;
+
+ for (auto tablet_id : table_id_tablet_ids[table_id]) {
+ std::string pending_key = meta_pending_delete_bitmap_key({instance_id, tablet_id});
+ txn->remove(pending_key);
+ LOG(INFO) << "xxx remove delete bitmap pending key, pending_key="
+ << hex(pending_key) << " txn_id=" << txn_id;
+ }
+ }
+ lock_keys.clear();
+ lock_values.clear();
+
+ // Save rowset meta
+ for (auto& i : rowsets) {
+ size_t rowset_size = i.first.size() + i.second.size();
+ txn->put(i.first, i.second);
+ LOG(INFO) << "xxx put rowset_key=" << hex(i.first) << " txn_id=" << txn_id
+ << " rowset_size=" << rowset_size;
+ }
+
+ // Save versions
+ int64_t version_update_time_ms =
+ duration_cast<milliseconds>(system_clock::now().time_since_epoch()).count();
+ response->set_version_update_time_ms(version_update_time_ms);
+ for (auto& i : new_versions) {
+ std::string ver_val;
+ VersionPB version_pb;
+ version_pb.set_version(i.second);
+ version_pb.set_update_time_ms(version_update_time_ms);
+ if (!version_pb.SerializeToString(&ver_val)) {
+ code = MetaServiceCode::PROTOBUF_SERIALIZE_ERR;
+ ss << "failed to serialize version_pb when saving, txn_id=" << txn_id;
+ msg = ss.str();
+ return;
+ }
+
+ txn->put(i.first, ver_val);
+ LOG(INFO) << "xxx put partition_version_key=" << hex(i.first) << " version:" << i.second
+ << " txn_id=" << txn_id << " update_time=" << version_update_time_ms;
+
+ std::string_view ver_key = i.first;
+ ver_key.remove_prefix(1); // Remove key space
+ // PartitionVersionKeyInfo {instance_id, db_id, table_id, partition_id}
+ std::vector<std::tuple<std::variant<int64_t, std::string>, int, int>> out;
+ int ret = decode_key(&ver_key, &out);
+ if (ret != 0) [[unlikely]] {
+ // decode version key error means this is something wrong,
+ // we can not continue this txn
+ LOG(WARNING) << "failed to decode key, ret=" << ret << " key=" << hex(ver_key);
+ code = MetaServiceCode::UNDEFINED_ERR;
+ msg = "decode version key error";
+ return;
+ }
+
+ int64_t table_id = std::get<int64_t>(std::get<0>(out[4]));
+ int64_t partition_id = std::get<int64_t>(std::get<0>(out[5]));
+ VLOG_DEBUG << " table_id=" << table_id << " partition_id=" << partition_id;
+
+ response->add_table_ids(table_id);
+ response->add_partition_ids(partition_id);
+ response->add_versions(i.second);
+ }
+
+ // Save table versions
+ for (auto& i : table_id_tablet_ids) {
+ std::string ver_key = table_version_key({instance_id, db_id, i.first});
+ txn->atomic_add(ver_key, 1);
+ LOG(INFO) << "xxx atomic add table_version_key=" << hex(ver_key)
+ << " txn_id=" << txn_id;
+ }
+
+ LOG(INFO) << " before update txn_info=" << txn_info.ShortDebugString();
+
+ // Update txn_info
+ txn_info.set_status(TxnStatusPB::TXN_STATUS_VISIBLE);
+
+ auto now_time = system_clock::now();
+ uint64_t commit_time = duration_cast<milliseconds>(now_time.time_since_epoch()).count();
+ if ((txn_info.prepare_time() + txn_info.timeout_ms()) < commit_time) {
+ code = MetaServiceCode::UNDEFINED_ERR;
+ msg = fmt::format("txn is expired, not allow to commit txn_id={}", txn_id);
+ LOG(INFO) << msg << " prepare_time=" << txn_info.prepare_time()
+ << " timeout_ms=" << txn_info.timeout_ms() << " commit_time=" << commit_time;
+ return;
+ }
+ txn_info.set_commit_time(commit_time);
+ txn_info.set_finish_time(commit_time);
+ if (request->has_commit_attachment()) {
+ txn_info.mutable_commit_attachment()->CopyFrom(request->commit_attachment());
+ }
+ LOG(INFO) << "after update txn_info=" << txn_info.ShortDebugString();
+ info_val.clear();
+ if (!txn_info.SerializeToString(&info_val)) {
+ code = MetaServiceCode::PROTOBUF_SERIALIZE_ERR;
+ ss << "failed to serialize txn_info when saving, txn_id=" << txn_id;
+ msg = ss.str();
+ return;
+ }
+ txn->put(info_key, info_val);
+ LOG(INFO) << "xxx put info_key=" << hex(info_key) << " txn_id=" << txn_id;
+
+ // Update stats of affected tablet
+ for (auto& [tablet_id, stats] : tablet_stats) {
+ DCHECK(tablet_ids.count(tablet_id));
+ auto& tablet_idx = tablet_ids[tablet_id];
+ StatsTabletKeyInfo info {instance_id, tablet_idx.table_id(), tablet_idx.index_id(),
+ tablet_idx.partition_id(), tablet_id};
+ update_tablet_stats(info, stats, txn, code, msg);
+ if (code != MetaServiceCode::OK) return;
+ }
+ // Remove tmp rowset meta
+ for (auto& [k, _] : tmp_rowsets_meta) {
+ txn->remove(k);
+ LOG(INFO) << "xxx remove tmp_rowset_key=" << hex(k) << " txn_id=" << txn_id;
+ }
+
+ const std::string running_key = txn_running_key({instance_id, db_id, txn_id});
+ LOG(INFO) << "xxx remove running_key=" << hex(running_key) << " txn_id=" << txn_id;
+ txn->remove(running_key);
+
+ std::string recycle_val;
+ std::string recycle_key = recycle_txn_key({instance_id, db_id, txn_id});
+ RecycleTxnPB recycle_pb;
+ recycle_pb.set_creation_time(commit_time);
+ recycle_pb.set_label(txn_info.label());
+
+ if (!recycle_pb.SerializeToString(&recycle_val)) {
+ code = MetaServiceCode::PROTOBUF_SERIALIZE_ERR;
+ ss << "failed to serialize recycle_pb, txn_id=" << txn_id;
+ msg = ss.str();
+ return;
+ }
+ txn->put(recycle_key, recycle_val);
+
+ if (txn_info.load_job_source_type() ==
+ LoadJobSourceTypePB::LOAD_JOB_SRC_TYPE_ROUTINE_LOAD_TASK) {
+ put_routine_load_progress(code, msg, instance_id, request, txn.get(), db_id);
+ }
+
+ LOG(INFO) << "xxx commit_txn put recycle_key key=" << hex(recycle_key)
+ << " txn_id=" << txn_id;
+ LOG(INFO) << "commit_txn put_size=" << txn->put_bytes()
+ << " del_size=" << txn->delete_bytes() << " num_put_keys=" << txn->num_put_keys()
+ << " num_del_keys=" << txn->num_del_keys()
+ << " txn_size=" << txn->approximate_bytes() << " txn_id=" << txn_id;
+
+ // Finally we are done...
+ err = txn->commit();
+ if (err != TxnErrorCode::TXN_OK) {
+ code = cast_as<ErrCategory::COMMIT>(err);
+ ss << "failed to commit kv txn, txn_id=" << txn_id << " err=" << err;
+ msg = ss.str();
+ return;
+ }
+
+ // calculate table stats from tablets stats
+ std::map<int64_t /*table_id*/, TableStats> table_stats;
+ std::vector<int64_t> base_tablet_ids(request->base_tablet_ids().begin(),
+ request->base_tablet_ids().end());
+ calc_table_stats(tablet_ids, tablet_stats, table_stats, base_tablet_ids);
+ for (const auto& pair : table_stats) {
+ TableStatsPB* stats_pb = response->add_table_stats();
+ auto table_id = pair.first;
+ auto stats = pair.second;
+ get_pb_from_tablestats(stats, stats_pb);
+ stats_pb->set_table_id(table_id);
+ VLOG_DEBUG << "Add TableStats to CommitTxnResponse. txn_id=" << txn_id
+ << " table_id=" << table_id
+ << " updated_row_count=" << stats_pb->updated_row_count();
+ }
+ response->mutable_txn_info()->CopyFrom(txn_info);
+ break;
+ } while (true);
+} // end commit_txn_immediately
+
+void get_tablet_indexes(
+ const std::string& instance_id, int64_t txn_id,
+ const std::vector<std::pair<std::string, doris::RowsetMetaCloudPB>>& tmp_rowsets_meta,
+ std::unique_ptr<Transaction>& txn, MetaServiceCode& code, std::string& msg,
+ std::unordered_map<int64_t, TabletIndexPB>* tablet_ids,
+ std::unordered_map<int64_t, std::vector<int64_t>>* table_id_tablet_ids,
+ bool* need_repair_tablet_idx) {
// Read tablet indexes in batch.
+ std::stringstream ss;
std::vector<std::string> tablet_idx_keys;
for (auto& [_, i] : tmp_rowsets_meta) {
tablet_idx_keys.push_back(meta_tablet_idx_key({instance_id, i.tablet_id()}));
}
std::vector<std::optional<std::string>> tablet_idx_values;
- err = txn->batch_get(&tablet_idx_values, tablet_idx_keys, Transaction::BatchGetOptions(false));
+ TxnErrorCode err =
+ txn->batch_get(&tablet_idx_values, tablet_idx_keys, Transaction::BatchGetOptions(true));
if (err != TxnErrorCode::TXN_OK) {
code = cast_as<ErrCategory::READ>(err);
ss << "failed to get tablet table index ids, err=" << err;
@@ -1021,12 +1410,7 @@ void commit_txn_immediately(
return;
}
- size_t total_rowsets = tmp_rowsets_meta.size();
- // tablet_id -> {table/index/partition}_id
- std::unordered_map<int64_t, TabletIndexPB> tablet_ids;
- // table_id -> tablets_ids
- std::unordered_map<int64_t, std::vector<int64_t>> table_id_tablet_ids;
- for (size_t i = 0; i < total_rowsets; i++) {
+ for (size_t i = 0; i < tmp_rowsets_meta.size(); i++) {
uint64_t tablet_id = tmp_rowsets_meta[i].second.tablet_id();
if (!tablet_idx_values[i].has_value()) [[unlikely]] {
// The value must existed
@@ -1037,353 +1421,488 @@ void commit_txn_immediately(
LOG(WARNING) << msg << " err=" << err << " txn_id=" << txn_id;
return;
}
- if (!tablet_ids[tablet_id].ParseFromString(tablet_idx_values[i].value())) [[unlikely]] {
+ if (!(*tablet_ids)[tablet_id].ParseFromString(tablet_idx_values[i].value())) [[unlikely]] {
code = MetaServiceCode::PROTOBUF_PARSE_ERR;
ss << "malformed tablet index value tablet_id=" << tablet_id << " txn_id=" << txn_id;
msg = ss.str();
LOG(WARNING) << msg;
return;
}
- table_id_tablet_ids[tablet_ids[tablet_id].table_id()].push_back(tablet_id);
+ if (!(*tablet_ids)[tablet_id].has_db_id()) {
+ *need_repair_tablet_idx = true;
+ }
+ (*table_id_tablet_ids)[(*tablet_ids)[tablet_id].table_id()].push_back(tablet_id);
VLOG_DEBUG << "tablet_id:" << tablet_id
- << " value:" << tablet_ids[tablet_id].ShortDebugString();
+ << " value:" << (*tablet_ids)[tablet_id].ShortDebugString();
}
tablet_idx_keys.clear();
tablet_idx_values.clear();
+}
- // {table/partition} -> version
- std::unordered_map<std::string, uint64_t> new_versions;
- std::vector<std::string> version_keys;
+// rewrite TabletIndexPB for fill db_id, in case of historical reasons
+// TabletIndexPB missing db_id
+void repair_tablet_index(
+ std::shared_ptr<TxnKv>& txn_kv, MetaServiceCode& code, std::string& msg,
+ const std::string& instance_id, int64_t db_id, int64_t txn_id,
+ const std::vector<std::pair<std::string, doris::RowsetMetaCloudPB>>& tmp_rowsets_meta) {
+ std::stringstream ss;
+ std::vector<std::string> tablet_idx_keys;
for (auto& [_, i] : tmp_rowsets_meta) {
- int64_t tablet_id = i.tablet_id();
- int64_t table_id = tablet_ids[tablet_id].table_id();
- int64_t partition_id = i.partition_id();
- std::string ver_key = partition_version_key({instance_id, db_id, table_id, partition_id});
- if (new_versions.count(ver_key) == 0) {
- new_versions.insert({ver_key, 0});
- version_keys.push_back(std::move(ver_key));
- }
- }
- std::vector<std::optional<std::string>> version_values;
- err = txn->batch_get(&version_values, version_keys);
- if (err != TxnErrorCode::TXN_OK) {
- code = cast_as<ErrCategory::READ>(err);
- ss << "failed to get partition versions, err=" << err;
- msg = ss.str();
- LOG(WARNING) << msg << " txn_id=" << txn_id;
- return;
+ tablet_idx_keys.push_back(meta_tablet_idx_key({instance_id, i.tablet_id()}));
}
- size_t total_versions = version_keys.size();
- for (size_t i = 0; i < total_versions; i++) {
- int64_t version;
- if (version_values[i].has_value()) {
- VersionPB version_pb;
- if (!version_pb.ParseFromString(version_values[i].value())) {
+
+ for (size_t i = 0; i < tablet_idx_keys.size(); i += config::max_tablet_index_num_per_batch) {
+ size_t end = (i + config::max_tablet_index_num_per_batch) > tablet_idx_keys.size()
+ ? tablet_idx_keys.size()
+ : i + config::max_tablet_index_num_per_batch;
+ const std::vector<std::string> sub_tablet_idx_keys(tablet_idx_keys.begin() + i,
+ tablet_idx_keys.begin() + end);
+
+ std::unique_ptr<Transaction> txn;
+ TxnErrorCode err = txn_kv->create_txn(&txn);
+ if (err != TxnErrorCode::TXN_OK) {
+ code = cast_as<ErrCategory::CREATE>(err);
+ ss << "failed to create txn, txn_id=" << txn_id << " err=" << err;
+ msg = ss.str();
+ LOG(WARNING) << msg;
+ return;
+ }
+
+ std::vector<std::optional<std::string>> tablet_idx_values;
+ // batch get snapshot is false
+ err = txn->batch_get(&tablet_idx_values, sub_tablet_idx_keys,
+ Transaction::BatchGetOptions(false));
+ if (err != TxnErrorCode::TXN_OK) {
+ code = cast_as<ErrCategory::READ>(err);
+ ss << "failed to get tablet table index ids, err=" << err;
+ msg = ss.str();
+ LOG(WARNING) << msg << " txn_id=" << txn_id;
+ return;
+ }
+ DCHECK(tablet_idx_values.size() <= config::max_tablet_index_num_per_batch);
+
+ for (size_t j = 0; j < sub_tablet_idx_keys.size(); j++) {
+ if (!tablet_idx_values[j].has_value()) [[unlikely]] {
+ // The value must existed
+ code = MetaServiceCode::KV_TXN_GET_ERR;
+ ss << "failed to get tablet table index ids, err=not found"
+ << " key=" << hex(tablet_idx_keys[j]);
+ msg = ss.str();
+ LOG(WARNING) << msg << " err=" << err << " txn_id=" << txn_id;
+ return;
+ }
+ TabletIndexPB tablet_idx_pb;
+ if (!tablet_idx_pb.ParseFromString(tablet_idx_values[j].value())) [[unlikely]] {
code = MetaServiceCode::PROTOBUF_PARSE_ERR;
- ss << "failed to parse version pb txn_id=" << txn_id
- << " key=" << hex(version_keys[i]);
+ ss << "malformed tablet index value key=" << hex(tablet_idx_keys[j])
+ << " txn_id=" << txn_id;
msg = ss.str();
+ LOG(WARNING) << msg;
+ return;
+ }
+
+ if (!tablet_idx_pb.has_db_id()) {
+ tablet_idx_pb.set_db_id(db_id);
+ std::string idx_val;
+ if (!tablet_idx_pb.SerializeToString(&idx_val)) {
+ code = MetaServiceCode::PROTOBUF_SERIALIZE_ERR;
+ ss << "failed to serialize tablet index value key=" << hex(tablet_idx_keys[j])
+ << " txn_id=" << txn_id;
+ msg = ss.str();
+ LOG(WARNING) << msg;
+ return;
+ }
+ txn->put(sub_tablet_idx_keys[j], idx_val);
+ LOG(INFO) << " repaire tablet index txn_id=" << txn_id
+ << " tablet_idx_pb:" << tablet_idx_pb.ShortDebugString()
+ << " key=" << hex(sub_tablet_idx_keys[j]);
+ }
+ }
+
+ err = txn->commit();
+ if (err != TxnErrorCode::TXN_OK) {
+ code = cast_as<ErrCategory::COMMIT>(err);
+ ss << "failed to commit kv txn, txn_id=" << txn_id << " err=" << err;
+ msg = ss.str();
+ LOG(WARNING) << msg;
+ return;
+ }
+ }
+}
+
+void commit_txn_eventually(
+ const CommitTxnRequest* request, CommitTxnResponse* response,
+ std::shared_ptr<TxnKv>& txn_kv, std::shared_ptr<TxnLazyCommitter>& txn_lazy_committer,
+ MetaServiceCode& code, std::string& msg, const std::string& instance_id, int64_t db_id,
+ const std::vector<std::pair<std::string, doris::RowsetMetaCloudPB>>& tmp_rowsets_meta) {
+ std::stringstream ss;
+ TxnErrorCode err = TxnErrorCode::TXN_OK;
+ int64_t txn_id = request->txn_id();
+
+ do {
+ int64_t last_pending_txn_id = 0;
+ std::unique_ptr<Transaction> txn;
+ err = txn_kv->create_txn(&txn);
+ if (err != TxnErrorCode::TXN_OK) {
+ code = cast_as<ErrCategory::CREATE>(err);
+ ss << "failed to create txn, txn_id=" << txn_id << " err=" << err;
+ msg = ss.str();
+ LOG(WARNING) << msg;
+ return;
+ }
+
+ // tablet_id -> {table/index/partition}_id
+ std::unordered_map<int64_t, TabletIndexPB> tablet_ids;
+ // table_id -> tablets_ids
+ std::unordered_map<int64_t, std::vector<int64_t>> table_id_tablet_ids;
+ bool need_repair_tablet_idx = false;
+ get_tablet_indexes(instance_id, txn_id, tmp_rowsets_meta, txn, code, msg, &tablet_ids,
+ &table_id_tablet_ids, &need_repair_tablet_idx);
+ if (code != MetaServiceCode::OK) {
+ LOG(WARNING) << "get_tablet_indexes failed, txn_id=" << txn_id << " code=" << code;
+ return;
+ }
+
+ if (need_repair_tablet_idx) {
+ txn.reset();
+ repair_tablet_index(txn_kv, code, msg, instance_id, db_id, txn_id, tmp_rowsets_meta);
+ if (code != MetaServiceCode::OK) {
+ LOG(WARNING) << "repair_tablet_index failed, txn_id=" << txn_id << " code=" << code;
return;
}
- version = version_pb.version();
- } else {
- version = 1;
+ continue;
+ }
+
+ // <partition_version_key, version>
+ std::unordered_map<std::string, uint64_t> new_versions;
+ std::vector<std::string> version_keys;
+ for (auto& [_, i] : tmp_rowsets_meta) {
+ int64_t tablet_id = i.tablet_id();
+ int64_t table_id = tablet_ids[tablet_id].table_id();
+ int64_t partition_id = i.partition_id();
+ std::string ver_key =
+ partition_version_key({instance_id, db_id, table_id, partition_id});
+ if (new_versions.count(ver_key) == 0) {
+ new_versions.insert({ver_key, 0});
+ version_keys.push_back(std::move(ver_key));
+ }
}
- new_versions[version_keys[i]] = version + 1;
- }
- version_keys.clear();
- version_values.clear();
- std::vector<std::pair<std::string, std::string>> rowsets;
- std::unordered_map<int64_t, TabletStats> tablet_stats; // tablet_id -> stats
- rowsets.reserve(tmp_rowsets_meta.size());
- for (auto& [_, i] : tmp_rowsets_meta) {
- int64_t tablet_id = i.tablet_id();
- int64_t table_id = tablet_ids[tablet_id].table_id();
- int64_t partition_id = i.partition_id();
- std::string ver_key = partition_version_key({instance_id, db_id, table_id, partition_id});
- if (new_versions[ver_key] == 0) [[unlikely]] {
- // it is impossible.
- code = MetaServiceCode::UNDEFINED_ERR;
- ss << "failed to get partition version key, the target version not exists in "
- "new_versions."
- << " txn_id=" << txn_id;
+ std::vector<std::optional<std::string>> version_values;
+ err = txn->batch_get(&version_values, version_keys);
+ if (err != TxnErrorCode::TXN_OK) {
+ code = cast_as<ErrCategory::READ>(err);
+ ss << "failed to get partition versions, err=" << err;
msg = ss.str();
- LOG(ERROR) << msg;
+ LOG(WARNING) << msg << " txn_id=" << txn_id;
return;
}
- // Update rowset version
- int64_t new_version = new_versions[ver_key];
- i.set_start_version(new_version);
- i.set_end_version(new_version);
+ for (size_t i = 0; i < version_keys.size(); i++) {
+ int64_t version;
+ if (version_values[i].has_value()) {
+ VersionPB version_pb;
+ if (!version_pb.ParseFromString(version_values[i].value())) {
+ code = MetaServiceCode::PROTOBUF_PARSE_ERR;
+ ss << "failed to parse version pb txn_id=" << txn_id
+ << " key=" << hex(version_keys[i]);
+ msg = ss.str();
+ return;
+ }
+ if (version_pb.pending_txn_ids_size() > 0) {
+ DCHECK(version_pb.pending_txn_ids_size() == 1);
+ last_pending_txn_id = version_pb.pending_txn_ids(0);
+ DCHECK(last_pending_txn_id > 0);
+ break;
+ }
+ version = version_pb.version();
+ } else {
+ version = 1;
+ }
+ new_versions[version_keys[i]] = version;
+ last_pending_txn_id = 0;
+ }
+
+ if (last_pending_txn_id > 0) {
+ txn.reset();
+ std::shared_ptr<TxnLazyCommitTask> task =
+ txn_lazy_committer->submit(instance_id, last_pending_txn_id);
- std::string key = meta_rowset_key({instance_id, tablet_id, i.end_version()});
- std::string val;
- if (!i.SerializeToString(&val)) {
- code = MetaServiceCode::PROTOBUF_SERIALIZE_ERR;
- ss << "failed to serialize rowset_meta, txn_id=" << txn_id;
+ std::tie(code, msg) = task->wait();
+ if (code != MetaServiceCode::OK) {
+ LOG(WARNING) << "advance_last_txn failed last_txn=" << last_pending_txn_id
+ << " code=" << code << "msg=" << msg;
+ return;
+ }
+
+ last_pending_txn_id = 0;
+ // there maybe concurrent commit_txn_eventually, so we need continue to make sure
+ // partition versionPB has no txn_id
+ continue;
+ }
+
+ std::string info_val;
+ const std::string info_key = txn_info_key({instance_id, db_id, txn_id});
+ err = txn->get(info_key, &info_val);
+ if (err != TxnErrorCode::TXN_OK) {
+ code = err == TxnErrorCode::TXN_KEY_NOT_FOUND ? MetaServiceCode::TXN_ID_NOT_FOUND
+ : cast_as<ErrCategory::READ>(err);
+ ss << "failed to get txn_info, db_id=" << db_id << " txn_id=" << txn_id
+ << " err=" << err;
msg = ss.str();
+ LOG(WARNING) << msg;
return;
}
- rowsets.emplace_back(std::move(key), std::move(val));
- // Accumulate affected rows
- auto& stats = tablet_stats[tablet_id];
- stats.data_size += i.data_disk_size();
- stats.num_rows += i.num_rows();
- ++stats.num_rowsets;
- stats.num_segs += i.num_segments();
- } // for tmp_rowsets_meta
-
- // process mow table, check lock and remove pending key
- std::vector<std::string> lock_keys;
- lock_keys.reserve(request->mow_table_ids().size());
- for (auto table_id : request->mow_table_ids()) {
- lock_keys.push_back(meta_delete_bitmap_update_lock_key({instance_id, table_id, -1}));
- }
- std::vector<std::optional<std::string>> lock_values;
- err = txn->batch_get(&lock_values, lock_keys);
- if (err != TxnErrorCode::TXN_OK) {
- ss << "failed to get delete bitmap update lock key info, instance_id=" << instance_id
- << " err=" << err;
- msg = ss.str();
- code = cast_as<ErrCategory::READ>(err);
- LOG(WARNING) << msg << " txn_id=" << txn_id;
- return;
- }
- size_t total_locks = lock_keys.size();
- for (size_t i = 0; i < total_locks; i++) {
- int64_t table_id = request->mow_table_ids(i);
- // When the key does not exist, it means the lock has been acquired
- // by another transaction and successfully committed.
- if (!lock_values[i].has_value()) {
- ss << "get delete bitmap update lock info, lock is expired"
- << " table_id=" << table_id << " key=" << hex(lock_keys[i]);
- code = MetaServiceCode::LOCK_EXPIRED;
+ TxnInfoPB txn_info;
+ if (!txn_info.ParseFromString(info_val)) {
+ code = MetaServiceCode::PROTOBUF_PARSE_ERR;
+ ss << "failed to parse txn_info, db_id=" << db_id << " txn_id=" << txn_id;
msg = ss.str();
- LOG(WARNING) << msg << " txn_id=" << txn_id;
+ LOG(WARNING) << msg;
return;
}
+ LOG(INFO) << "txn_id=" << txn_id << " txn_info=" << txn_info.ShortDebugString();
- DeleteBitmapUpdateLockPB lock_info;
- if (!lock_info.ParseFromString(lock_values[i].value())) [[unlikely]] {
- code = MetaServiceCode::PROTOBUF_PARSE_ERR;
- msg = "failed to parse DeleteBitmapUpdateLockPB";
- LOG(WARNING) << msg << " txn_id=" << txn_id;
+ DCHECK(txn_info.txn_id() == txn_id);
+ if (txn_info.status() == TxnStatusPB::TXN_STATUS_ABORTED) {
+ code = MetaServiceCode::TXN_ALREADY_ABORTED;
+ ss << "transaction is already aborted: db_id=" << db_id << " txn_id=" << txn_id;
+ msg = ss.str();
+ LOG(WARNING) << msg;
return;
}
- if (lock_info.lock_id() != request->txn_id()) {
- msg = "lock is expired";
- code = MetaServiceCode::LOCK_EXPIRED;
+
+ if (txn_info.status() == TxnStatusPB::TXN_STATUS_VISIBLE) {
+ if (request->has_is_2pc() && request->is_2pc()) {
+ code = MetaServiceCode::TXN_ALREADY_VISIBLE;
+ ss << "transaction [" << txn_id << "] is already visible, not pre-committed.";
+ msg = ss.str();
+ LOG(INFO) << msg;
+ response->mutable_txn_info()->CopyFrom(txn_info);
+ return;
+ }
+ code = MetaServiceCode::OK;
+ ss << "transaction is already visible: db_id=" << db_id << " txn_id=" << txn_id;
+ msg = ss.str();
+ LOG(INFO) << msg;
+ response->mutable_txn_info()->CopyFrom(txn_info);
return;
}
- txn->remove(lock_keys[i]);
- LOG(INFO) << "xxx remove delete bitmap lock, lock_key=" << hex(lock_keys[i])
- << " txn_id=" << txn_id;
- for (auto tablet_id : table_id_tablet_ids[table_id]) {
- std::string pending_key = meta_pending_delete_bitmap_key({instance_id, tablet_id});
- txn->remove(pending_key);
- LOG(INFO) << "xxx remove delete bitmap pending key, pending_key=" << hex(pending_key)
- << " txn_id=" << txn_id;
+ if (request->has_is_2pc() && request->is_2pc() &&
+ txn_info.status() == TxnStatusPB::TXN_STATUS_PREPARED) {
+ code = MetaServiceCode::TXN_INVALID_STATUS;
+ ss << "transaction is prepare, not pre-committed: db_id=" << db_id << " txn_id"
+ << txn_id;
+ msg = ss.str();
+ LOG(WARNING) << msg;
+ return;
}
- }
- lock_keys.clear();
- lock_values.clear();
- // Save rowset meta
- for (auto& i : rowsets) {
- size_t rowset_size = i.first.size() + i.second.size();
- txn->put(i.first, i.second);
- LOG(INFO) << "xxx put rowset_key=" << hex(i.first) << " txn_id=" << txn_id
- << " rowset_size=" << rowset_size;
- }
+ auto now_time = system_clock::now();
+ uint64_t commit_time = duration_cast<milliseconds>(now_time.time_since_epoch()).count();
+ if ((txn_info.prepare_time() + txn_info.timeout_ms()) < commit_time) {
+ code = MetaServiceCode::UNDEFINED_ERR;
+ msg = fmt::format("txn is expired, not allow to commit txn_id={}", txn_id);
+ LOG(INFO) << msg << " prepare_time=" << txn_info.prepare_time()
+ << " timeout_ms=" << txn_info.timeout_ms() << " commit_time=" << commit_time;
+ return;
+ }
+ txn_info.set_commit_time(commit_time);
+ txn_info.set_finish_time(commit_time);
+ if (request->has_commit_attachment()) {
+ txn_info.mutable_commit_attachment()->CopyFrom(request->commit_attachment());
+ }
+ DCHECK(txn_info.status() != TxnStatusPB::TXN_STATUS_COMMITTED);
+ // set status TXN_STATUS_COMMITTED not TXN_STATUS_VISIBLE !!!
+ // lazy commit task will advance txn to make txn visible
+ txn_info.set_status(TxnStatusPB::TXN_STATUS_COMMITTED);
- // Save versions
- int64_t version_update_time_ms =
- duration_cast<milliseconds>(system_clock::now().time_since_epoch()).count();
- response->set_version_update_time_ms(version_update_time_ms);
- for (auto& i : new_versions) {
- std::string ver_val;
- VersionPB version_pb;
- version_pb.set_version(i.second);
- version_pb.set_update_time_ms(version_update_time_ms);
- if (!version_pb.SerializeToString(&ver_val)) {
+ LOG(INFO) << "after update txn_id= " << txn_id
+ << " txn_info=" << txn_info.ShortDebugString();
+ info_val.clear();
+ if (!txn_info.SerializeToString(&info_val)) {
code = MetaServiceCode::PROTOBUF_SERIALIZE_ERR;
- ss << "failed to serialize version_pb when saving, txn_id=" << txn_id;
+ ss << "failed to serialize txn_info when saving, txn_id=" << txn_id;
msg = ss.str();
return;
}
- txn->put(i.first, ver_val);
- LOG(INFO) << "xxx put partition_version_key=" << hex(i.first) << " version:" << i.second
- << " txn_id=" << txn_id << " update_time=" << version_update_time_ms;
+ txn->put(info_key, info_val);
+ LOG(INFO) << "put info_key=" << hex(info_key) << " txn_id=" << txn_id;
- std::string_view ver_key = i.first;
- ver_key.remove_prefix(1); // Remove key space
- // PartitionVersionKeyInfo {instance_id, db_id, table_id, partition_id}
- std::vector<std::tuple<std::variant<int64_t, std::string>, int, int>> out;
- int ret = decode_key(&ver_key, &out);
- if (ret != 0) [[unlikely]] {
- // decode version key error means this is something wrong,
- // we can not continue this txn
- LOG(WARNING) << "failed to decode key, ret=" << ret << " key=" << hex(ver_key);
- code = MetaServiceCode::UNDEFINED_ERR;
- msg = "decode version key error";
- return;
+ if (txn_info.load_job_source_type() ==
+ LoadJobSourceTypePB::LOAD_JOB_SRC_TYPE_ROUTINE_LOAD_TASK) {
+ put_routine_load_progress(code, msg, instance_id, request, txn.get(), db_id);
}
- int64_t table_id = std::get<int64_t>(std::get<0>(out[4]));
- int64_t partition_id = std::get<int64_t>(std::get<0>(out[5]));
- VLOG_DEBUG << " table_id=" << table_id << " partition_id=" << partition_id;
+ // save versions for partition
+ int64_t version_update_time_ms =
+ duration_cast<milliseconds>(system_clock::now().time_since_epoch()).count();
+ response->set_version_update_time_ms(version_update_time_ms);
+ for (auto& i : new_versions) {
+ std::string ver_val;
+ VersionPB version_pb;
+ version_pb.add_pending_txn_ids(txn_id);
+ version_pb.set_update_time_ms(version_update_time_ms);
+ if (i.second > 1) {
+ version_pb.set_version(i.second);
+ }
- response->add_table_ids(table_id);
- response->add_partition_ids(partition_id);
- response->add_versions(i.second);
- }
+ if (!version_pb.SerializeToString(&ver_val)) {
+ code = MetaServiceCode::PROTOBUF_SERIALIZE_ERR;
+ ss << "failed to serialize version_pb when saving, txn_id=" << txn_id
+ << " partiton_key=" << hex(i.first);
+ msg = ss.str();
+ return;
+ }
- // Save table versions
- for (auto& i : table_id_tablet_ids) {
- std::string ver_key = table_version_key({instance_id, db_id, i.first});
- txn->atomic_add(ver_key, 1);
- LOG(INFO) << "xxx atomic add table_version_key=" << hex(ver_key) << " txn_id=" << txn_id;
- }
+ txn->put(i.first, ver_val);
+ LOG(INFO) << "put partition_version_key=" << hex(i.first) << " version:" << i.second
+ << " txn_id=" << txn_id << " update_time=" << version_update_time_ms;
+
+ std::string_view ver_key = i.first;
+ ver_key.remove_prefix(1); // Remove key space
+ // PartitionVersionKeyInfo {instance_id, db_id, table_id, partition_id}
+ std::vector<std::tuple<std::variant<int64_t, std::string>, int, int>> out;
+ int ret = decode_key(&ver_key, &out);
+ if (ret != 0) [[unlikely]] {
+ // decode version key error means this is something wrong,
+ // we can not continue this txn
+ LOG(WARNING) << "failed to decode key, ret=" << ret << " key=" << hex(ver_key);
+ code = MetaServiceCode::UNDEFINED_ERR;
+ msg = "decode version key error";
+ return;
+ }
- LOG(INFO) << " before update txn_info=" << txn_info.ShortDebugString();
+ int64_t table_id = std::get<int64_t>(std::get<0>(out[4]));
+ int64_t partition_id = std::get<int64_t>(std::get<0>(out[5]));
+ VLOG_DEBUG << "txn_id=" << txn_id << " table_id=" << table_id
+ << " partition_id=" << partition_id << " version=" << i.second;
- // Update txn_info
- txn_info.set_status(TxnStatusPB::TXN_STATUS_VISIBLE);
+ response->add_table_ids(table_id);
+ response->add_partition_ids(partition_id);
+ response->add_versions(i.second + 1);
+ }
- auto now_time = system_clock::now();
- uint64_t commit_time = duration_cast<milliseconds>(now_time.time_since_epoch()).count();
- if ((txn_info.prepare_time() + txn_info.timeout_ms()) < commit_time) {
- code = MetaServiceCode::UNDEFINED_ERR;
- msg = fmt::format("txn is expired, not allow to commit txn_id={}", txn_id);
- LOG(INFO) << msg << " prepare_time=" << txn_info.prepare_time()
- << " timeout_ms=" << txn_info.timeout_ms() << " commit_time=" << commit_time;
- return;
- }
- txn_info.set_commit_time(commit_time);
- txn_info.set_finish_time(commit_time);
- if (request->has_commit_attachment()) {
- txn_info.mutable_commit_attachment()->CopyFrom(request->commit_attachment());
- }
- LOG(INFO) << "after update txn_info=" << txn_info.ShortDebugString();
- info_val.clear();
- if (!txn_info.SerializeToString(&info_val)) {
- code = MetaServiceCode::PROTOBUF_SERIALIZE_ERR;
- ss << "failed to serialize txn_info when saving, txn_id=" << txn_id;
- msg = ss.str();
- return;
- }
- txn->put(info_key, info_val);
- LOG(INFO) << "xxx put info_key=" << hex(info_key) << " txn_id=" << txn_id;
+ // process mow table, check lock and remove pending key
+ std::vector<std::string> lock_keys;
+ lock_keys.reserve(request->mow_table_ids().size());
+ for (auto table_id : request->mow_table_ids()) {
+ lock_keys.push_back(meta_delete_bitmap_update_lock_key({instance_id, table_id, -1}));
+ }
+ std::vector<std::optional<std::string>> lock_values;
+ err = txn->batch_get(&lock_values, lock_keys);
+ if (err != TxnErrorCode::TXN_OK) {
+ ss << "failed to get delete bitmap update lock key info, instance_id=" << instance_id
+ << " err=" << err;
+ msg = ss.str();
+ code = cast_as<ErrCategory::READ>(err);
+ LOG(WARNING) << msg << " txn_id=" << txn_id;
+ return;
+ }
- // Update stats of affected tablet
- for (auto& [tablet_id, stats] : tablet_stats) {
- DCHECK(tablet_ids.count(tablet_id));
- auto& tablet_idx = tablet_ids[tablet_id];
- StatsTabletKeyInfo info {instance_id, tablet_idx.table_id(), tablet_idx.index_id(),
- tablet_idx.partition_id(), tablet_id};
- update_tablet_stats(info, stats, txn, code, msg);
- if (code != MetaServiceCode::OK) return;
- }
- // Remove tmp rowset meta
- for (auto& [k, _] : tmp_rowsets_meta) {
- txn->remove(k);
- LOG(INFO) << "xxx remove tmp_rowset_key=" << hex(k) << " txn_id=" << txn_id;
- }
+ for (size_t i = 0; i < lock_keys.size(); i++) {
+ int64_t table_id = request->mow_table_ids(i);
+ // When the key does not exist, it means the lock has been acquired
+ // by another transaction and successfully committed.
+ if (!lock_values[i].has_value()) {
+ ss << "get delete bitmap update lock info, lock is expired"
+ << " table_id=" << table_id << " key=" << hex(lock_keys[i]);
+ code = MetaServiceCode::LOCK_EXPIRED;
+ msg = ss.str();
+ LOG(WARNING) << msg << " txn_id=" << txn_id;
+ return;
+ }
- const std::string running_key = txn_running_key({instance_id, db_id, txn_id});
- LOG(INFO) << "xxx remove running_key=" << hex(running_key) << " txn_id=" << txn_id;
- txn->remove(running_key);
+ DeleteBitmapUpdateLockPB lock_info;
+ if (!lock_info.ParseFromString(lock_values[i].value())) [[unlikely]] {
+ code = MetaServiceCode::PROTOBUF_PARSE_ERR;
+ msg = "failed to parse DeleteBitmapUpdateLockPB";
+ LOG(WARNING) << msg << " txn_id=" << txn_id;
+ return;
+ }
+ if (lock_info.lock_id() != request->txn_id()) {
+ msg = "lock is expired";
+ code = MetaServiceCode::LOCK_EXPIRED;
+ return;
+ }
+ txn->remove(lock_keys[i]);
+ LOG(INFO) << "xxx remove delete bitmap lock, lock_key=" << hex(lock_keys[i])
+ << " txn_id=" << txn_id;
- std::string recycle_val;
- std::string recycle_key = recycle_txn_key({instance_id, db_id, txn_id});
- RecycleTxnPB recycle_pb;
- recycle_pb.set_creation_time(commit_time);
- recycle_pb.set_label(txn_info.label());
+ for (auto tablet_id : table_id_tablet_ids[table_id]) {
+ std::string pending_key = meta_pending_delete_bitmap_key({instance_id, tablet_id});
+ txn->remove(pending_key);
+ LOG(INFO) << "xxx remove delete bitmap pending key, pending_key="
+ << hex(pending_key) << " txn_id=" << txn_id;
+ }
+ }
+ lock_keys.clear();
+ lock_values.clear();
- if (!recycle_pb.SerializeToString(&recycle_val)) {
- code = MetaServiceCode::PROTOBUF_SERIALIZE_ERR;
- ss << "failed to serialize recycle_pb, txn_id=" << txn_id;
- msg = ss.str();
- return;
- }
- txn->put(recycle_key, recycle_val);
+ // Save table versions
+ for (auto& i : table_id_tablet_ids) {
+ std::string ver_key = table_version_key({instance_id, db_id, i.first});
+ txn->atomic_add(ver_key, 1);
+ LOG(INFO) << "xxx atomic add table_version_key=" << hex(ver_key)
+ << " txn_id=" << txn_id;
+ }
- if (txn_info.load_job_source_type() ==
- LoadJobSourceTypePB::LOAD_JOB_SRC_TYPE_ROUTINE_LOAD_TASK) {
- put_routine_load_progress(code, msg, instance_id, request, txn.get(), db_id);
- }
+ LOG(INFO) << "commit_txn_eventually put_size=" << txn->put_bytes()
+ << " del_size=" << txn->delete_bytes() << " num_put_keys=" << txn->num_put_keys()
+ << " num_del_keys=" << txn->num_del_keys()
+ << " txn_size=" << txn->approximate_bytes() << " txn_id=" << txn_id;
- LOG(INFO) << "xxx commit_txn put recycle_key key=" << hex(recycle_key) << " txn_id=" << txn_id;
- LOG(INFO) << "commit_txn put_size=" << txn->put_bytes() << " del_size=" << txn->delete_bytes()
- << " num_put_keys=" << txn->num_put_keys() << " num_del_keys=" << txn->num_del_keys()
- << " txn_size=" << txn->approximate_bytes() << " txn_id=" << txn_id;
+ err = txn->commit();
+ if (err != TxnErrorCode::TXN_OK) {
+ code = cast_as<ErrCategory::COMMIT>(err);
+ ss << "failed to commit kv txn, txn_id=" << txn_id << " err=" << err;
+ msg = ss.str();
+ return;
+ }
- // Finally we are done...
- err = txn->commit();
- if (err != TxnErrorCode::TXN_OK) {
- if ((err == TxnErrorCode::TXN_VALUE_TOO_LARGE ||
- err == TxnErrorCode::TXN_BYTES_TOO_LARGE) &&
- !tmp_rowsets_meta.empty()) {
- size_t max_size = 0, max_idx = 0, max_num_segments = 0,
- min_num_segments = std::numeric_limits<size_t>::max(), avg_num_segments = 0;
- for (size_t i = 0; i < tmp_rowsets_meta.size(); i++) {
- auto& [k, v] = tmp_rowsets_meta[i];
- if (v.ByteSizeLong() > max_size) {
- max_size = v.ByteSizeLong();
- max_idx = i;
- }
- if (v.num_segments() > max_num_segments) {
- max_num_segments = v.num_segments();
- }
- if (v.num_segments() < min_num_segments) {
- min_num_segments = v.num_segments();
- }
- avg_num_segments += v.num_segments();
- }
- if (!tmp_rowsets_meta.empty()) {
- avg_num_segments /= tmp_rowsets_meta.size();
- }
- LOG(WARNING) << "failed to commit kv txn"
- << ", err=" << err << ", txn_id=" << txn_id
- << ", total_rowsets=" << tmp_rowsets_meta.size()
- << ", avg_num_segments=" << avg_num_segments
- << ", min_num_segments=" << min_num_segments
- << ", max_num_segments=" << max_num_segments
- << ", largest_rowset_size=" << max_size
- << ", largest_rowset_key=" << hex(tmp_rowsets_meta[max_idx].first)
- << ", largest_rowset_value="
- << tmp_rowsets_meta[max_idx].second.ShortDebugString();
+ std::shared_ptr<TxnLazyCommitTask> task = txn_lazy_committer->submit(instance_id, txn_id);
+ std::pair<MetaServiceCode, std::string> ret = task->wait();
+ if (ret.first != MetaServiceCode::OK) {
+ LOG(WARNING) << "txn lazy commit failed txn_id=" << txn_id << " code=" << ret.first
+ << "msg=" << ret.second;
}
- code = cast_as<ErrCategory::COMMIT>(err);
- ss << "failed to commit kv txn, txn_id=" << txn_id << " err=" << err;
- msg = ss.str();
- return;
- }
- // calculate table stats from tablets stats
- std::map<int64_t /*table_id*/, TableStats> table_stats;
- std::vector<int64_t> base_tablet_ids(request->base_tablet_ids().begin(),
- request->base_tablet_ids().end());
- calc_table_stats(tablet_ids, tablet_stats, table_stats, base_tablet_ids);
- for (const auto& pair : table_stats) {
- TableStatsPB* stats_pb = response->add_table_stats();
- auto table_id = pair.first;
- auto stats = pair.second;
- get_pb_from_tablestats(stats, stats_pb);
- stats_pb->set_table_id(table_id);
- VLOG_DEBUG << "Add TableStats to CommitTxnResponse. txn_id=" << txn_id
- << " table_id=" << table_id
- << " updated_row_count=" << stats_pb->updated_row_count();
- }
+ std::unordered_map<int64_t, TabletStats> tablet_stats; // tablet_id -> stats
+ for (auto& [_, i] : tmp_rowsets_meta) {
+ // Accumulate affected rows
+ auto& stats = tablet_stats[i.tablet_id()];
+ stats.data_size += i.data_disk_size();
+ stats.num_rows += i.num_rows();
+ ++stats.num_rowsets;
+ stats.num_segs += i.num_segments();
+ }
- response->mutable_txn_info()->CopyFrom(txn_info);
-} // end commit_txn
+ // calculate table stats from tablets stats
+ std::map<int64_t /*table_id*/, TableStats> table_stats;
+ std::vector<int64_t> base_tablet_ids(request->base_tablet_ids().begin(),
+ request->base_tablet_ids().end());
+ calc_table_stats(tablet_ids, tablet_stats, table_stats, base_tablet_ids);
+ for (const auto& pair : table_stats) {
+ TableStatsPB* stats_pb = response->add_table_stats();
+ auto table_id = pair.first;
+ auto stats = pair.second;
+ get_pb_from_tablestats(stats, stats_pb);
+ stats_pb->set_table_id(table_id);
+ VLOG_DEBUG << "Add TableStats to CommitTxnResponse. txn_id=" << txn_id
+ << " table_id=" << table_id
+ << " updated_row_count=" << stats_pb->updated_row_count();
+ }
+
+ // txn set visible for fe callback
+ txn_info.set_status(TxnStatusPB::TXN_STATUS_VISIBLE);
+ response->mutable_txn_info()->CopyFrom(txn_info);
+ break;
+ } while (true);
+}
/**
* This process is generally the same as commit_txn, the difference is that
@@ -2009,8 +2528,19 @@ void MetaServiceImpl::commit_txn(::google::protobuf::RpcController* controller,
LOG(WARNING) << "scan_tmp_rowset failed, txn_id=" << txn_id << " code=" << code;
return;
}
- commit_txn_immediately(request, response, txn_kv_, code, msg, instance_id, db_id,
- tmp_rowsets_meta);
+
+ if (request->has_is_2pc() && !request->is_2pc() && request->has_enable_txn_lazy_commit() &&
+ request->enable_txn_lazy_commit() && config::enable_cloud_txn_lazy_commit &&
+ (tmp_rowsets_meta.size() >= config::txn_lazy_commit_rowsets_thresold)) {
+ LOG(INFO) << "txn_id=" << txn_id << " commit_txn_eventually"
+ << " size=" << tmp_rowsets_meta.size();
+ commit_txn_eventually(request, response, txn_kv_, txn_lazy_committer_, code, msg,
+ instance_id, db_id, tmp_rowsets_meta);
+ return;
+ }
+
+ commit_txn_immediately(request, response, txn_kv_, txn_lazy_committer_, code, msg, instance_id,
+ db_id, tmp_rowsets_meta);
}
static void _abort_txn(const std::string& instance_id, const AbortTxnRequest* request,
diff --git a/cloud/src/meta-service/txn_lazy_committer.cpp b/cloud/src/meta-service/txn_lazy_committer.cpp
new file mode 100644
index 00000000000..5273a5c9683
--- /dev/null
+++ b/cloud/src/meta-service/txn_lazy_committer.cpp
@@ -0,0 +1,476 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "txn_lazy_committer.h"
+
+#include <chrono>
+
+#include "common/logging.h"
+#include "common/util.h"
+#include "meta-service/keys.h"
+#include "meta-service/meta_service_helper.h"
+#include "meta-service/meta_service_tablet_stats.h"
+
+using namespace std::chrono;
+
+namespace doris::cloud {
+
+void scan_tmp_rowset(
+ const std::string& instance_id, int64_t txn_id, std::shared_ptr<TxnKv> txn_kv,
+ MetaServiceCode& code, std::string& msg, int64_t* db_id,
+ std::vector<std::pair<std::string, doris::RowsetMetaCloudPB>>* tmp_rowsets_meta);
+
+void update_tablet_stats(const StatsTabletKeyInfo& info, const TabletStats& stats,
+ std::unique_ptr<Transaction>& txn, MetaServiceCode& code,
+ std::string& msg);
+
+void convert_tmp_rowsets(
+ const std::string& instance_id, int64_t txn_id, std::shared_ptr<TxnKv> txn_kv,
+ MetaServiceCode& code, std::string& msg, int64_t db_id,
+ std::vector<std::pair<std::string, doris::RowsetMetaCloudPB>>& tmp_rowsets_meta,
+ std::unordered_map<int64_t, TabletIndexPB>& tablet_ids) {
+ std::stringstream ss;
+ std::unique_ptr<Transaction> txn;
+ TxnErrorCode err = txn_kv->create_txn(&txn);
+ if (err != TxnErrorCode::TXN_OK) {
+ code = cast_as<ErrCategory::CREATE>(err);
+ ss << "failed to create txn, txn_id=" << txn_id << " err=" << err;
+ msg = ss.str();
+ LOG(WARNING) << msg;
+ return;
+ }
+
+ // partition_id -> VersionPB
+ std::unordered_map<int64_t, VersionPB> partition_versions;
+ // tablet_id -> stats
+ std::unordered_map<int64_t, TabletStats> tablet_stats;
+
+ for (auto& [tmp_rowset_key, tmp_rowset_pb] : tmp_rowsets_meta) {
+ std::string tmp_rowst_data;
+ err = txn->get(tmp_rowset_key, &tmp_rowst_data);
+ if (TxnErrorCode::TXN_KEY_NOT_FOUND == err) {
+ // the tmp rowset has been converted
+ VLOG_DEBUG << "tmp rowset has been converted, key=" << hex(tmp_rowset_key);
+ continue;
+ }
+
+ if (TxnErrorCode::TXN_OK != err) {
+ code = cast_as<ErrCategory::READ>(err);
+ ss << "failed to get tmp_rowset_key, txn_id=" << txn_id
+ << " key=" << hex(tmp_rowset_key) << " err=" << err;
+ msg = ss.str();
+ LOG(WARNING) << msg;
+ return;
+ }
+
+ if (!tablet_ids.contains(tmp_rowset_pb.tablet_id())) {
+ std::string tablet_idx_key =
+ meta_tablet_idx_key({instance_id, tmp_rowset_pb.tablet_id()});
+ std::string tablet_idx_val;
+ err = txn->get(tablet_idx_key, &tablet_idx_val, true);
+ if (TxnErrorCode::TXN_OK != err) {
+ code = err == TxnErrorCode::TXN_KEY_NOT_FOUND ? MetaServiceCode::TXN_ID_NOT_FOUND
+ : cast_as<ErrCategory::READ>(err);
+ ss << "failed to get tablet idx, txn_id=" << txn_id
+ << " key=" << hex(tablet_idx_key) << " err=" << err;
+ msg = ss.str();
+ LOG(WARNING) << msg;
+ return;
+ }
+
+ TabletIndexPB tablet_idx_pb;
+ if (!tablet_idx_pb.ParseFromString(tablet_idx_val)) {
+ code = MetaServiceCode::PROTOBUF_PARSE_ERR;
+ ss << "failed to parse tablet idx pb txn_id=" << txn_id
+ << " key=" << hex(tablet_idx_key);
+ msg = ss.str();
+ return;
+ }
+ tablet_ids.emplace(tmp_rowset_pb.tablet_id(), tablet_idx_pb);
+ }
+ const TabletIndexPB& tablet_idx_pb = tablet_ids[tmp_rowset_pb.tablet_id()];
+
+ if (!partition_versions.contains(tmp_rowset_pb.partition_id())) {
+ std::string ver_val;
+ std::string ver_key = partition_version_key(
+ {instance_id, db_id, tablet_idx_pb.table_id(), tmp_rowset_pb.partition_id()});
+ err = txn->get(ver_key, &ver_val);
+ if (TxnErrorCode::TXN_OK != err) {
+ code = err == TxnErrorCode::TXN_KEY_NOT_FOUND ? MetaServiceCode::TXN_ID_NOT_FOUND
+ : cast_as<ErrCategory::READ>(err);
+ ss << "failed to get partiton version, txn_id=" << txn_id << " key=" << hex(ver_key)
+ << " err=" << err;
+ msg = ss.str();
+ LOG(WARNING) << msg;
+ return;
+ }
+ VersionPB version_pb;
+ if (!version_pb.ParseFromString(ver_val)) {
+ code = MetaServiceCode::PROTOBUF_PARSE_ERR;
+ ss << "failed to parse version pb txn_id=" << txn_id << " key=" << hex(ver_key);
+ msg = ss.str();
+ return;
+ }
+ LOG(INFO) << "txn_id=" << txn_id << " key=" << hex(ver_key)
+ << " version_pb:" << version_pb.ShortDebugString();
+ partition_versions.emplace(tmp_rowset_pb.partition_id(), version_pb);
+ }
+
+ const VersionPB& version_pb = partition_versions[tmp_rowset_pb.partition_id()];
+ DCHECK(version_pb.pending_txn_ids_size() > 0);
+ DCHECK_EQ(version_pb.pending_txn_ids(0), txn_id);
+
+ int64_t version = version_pb.has_version() ? (version_pb.version() + 1) : 2;
+
+ std::string rowset_key = meta_rowset_key({instance_id, tmp_rowset_pb.tablet_id(), version});
+ std::string rowset_val;
+ err = txn->get(rowset_key, &rowset_val);
+ if (TxnErrorCode::TXN_OK == err) {
+ // tmp rowset key has been converted
+ continue;
+ }
+
+ if (err != TxnErrorCode::TXN_KEY_NOT_FOUND) {
+ code = cast_as<ErrCategory::READ>(err);
+ ss << "failed to get rowset_key, txn_id=" << txn_id << " key=" << hex(rowset_key)
+ << " err=" << err;
+ msg = ss.str();
+ LOG(WARNING) << msg;
+ return;
+ }
+ DCHECK(err == TxnErrorCode::TXN_KEY_NOT_FOUND);
+
+ tmp_rowset_pb.set_start_version(version);
+ tmp_rowset_pb.set_end_version(version);
+
+ rowset_val.clear();
+ if (!tmp_rowset_pb.SerializeToString(&rowset_val)) {
+ code = MetaServiceCode::PROTOBUF_SERIALIZE_ERR;
+ ss << "failed to serialize rowset_meta, txn_id=" << txn_id
+ << " key=" << hex(rowset_key);
+ msg = ss.str();
+ return;
+ }
+
+ txn->put(rowset_key, rowset_val);
+ LOG(INFO) << "put rowset_key=" << hex(rowset_key) << " txn_id=" << txn_id
+ << " rowset_size=" << rowset_key.size() + rowset_val.size();
+
+ // Accumulate affected rows
+ auto& stats = tablet_stats[tmp_rowset_pb.tablet_id()];
+ stats.data_size += tmp_rowset_pb.data_disk_size();
+ stats.num_rows += tmp_rowset_pb.num_rows();
+ ++stats.num_rowsets;
+ stats.num_segs += tmp_rowset_pb.num_segments();
+ }
+
+ DCHECK(partition_versions.size() == 1);
+
+ for (auto& [tablet_id, stats] : tablet_stats) {
+ DCHECK(tablet_ids.count(tablet_id));
+ auto& tablet_idx = tablet_ids[tablet_id];
+ StatsTabletKeyInfo info {instance_id, tablet_idx.table_id(), tablet_idx.index_id(),
+ tablet_idx.partition_id(), tablet_id};
+ update_tablet_stats(info, stats, txn, code, msg);
+ if (code != MetaServiceCode::OK) return;
+ }
+
+ err = txn->commit();
+ if (err != TxnErrorCode::TXN_OK) {
+ code = cast_as<ErrCategory::COMMIT>(err);
+ ss << "failed to commit kv txn, txn_id=" << txn_id << " err=" << err;
+ msg = ss.str();
+ return;
+ }
+}
+
+void make_committed_txn_visible(const std::string& instance_id, int64_t db_id, int64_t txn_id,
+ std::shared_ptr<TxnKv> txn_kv, MetaServiceCode& code,
+ std::string& msg) {
+ // 1. visible txn info
+ // 2. remove running key and put recycle txn key
+
+ std::stringstream ss;
+ std::unique_ptr<Transaction> txn;
+ TxnErrorCode err = txn_kv->create_txn(&txn);
+ if (err != TxnErrorCode::TXN_OK) {
+ code = cast_as<ErrCategory::CREATE>(err);
+ ss << "failed to create txn, txn_id=" << txn_id << " err=" << err;
+ msg = ss.str();
+ LOG(WARNING) << msg;
+ return;
+ }
+
+ std::string info_val;
+ const std::string info_key = txn_info_key({instance_id, db_id, txn_id});
+ err = txn->get(info_key, &info_val);
+ if (err != TxnErrorCode::TXN_OK) {
+ code = err == TxnErrorCode::TXN_KEY_NOT_FOUND ? MetaServiceCode::TXN_ID_NOT_FOUND
+ : cast_as<ErrCategory::READ>(err);
+ ss << "failed to get txn_info, db_id=" << db_id << " txn_id=" << txn_id << " err=" << err;
+ msg = ss.str();
+ LOG(WARNING) << msg;
+ return;
+ }
+
+ TxnInfoPB txn_info;
+ if (!txn_info.ParseFromString(info_val)) {
+ code = MetaServiceCode::PROTOBUF_PARSE_ERR;
+ ss << "failed to parse txn_info, txn_id=" << txn_id;
+ msg = ss.str();
+ LOG(WARNING) << msg;
+ return;
+ }
+
+ VLOG_DEBUG << "txn_info:" << txn_info.ShortDebugString();
+ DCHECK((txn_info.status() == TxnStatusPB::TXN_STATUS_COMMITTED) ||
+ (txn_info.status() == TxnStatusPB::TXN_STATUS_VISIBLE));
+
+ if (txn_info.status() == TxnStatusPB::TXN_STATUS_COMMITTED) {
+ txn_info.set_status(TxnStatusPB::TXN_STATUS_VISIBLE);
+
+ if (!txn_info.SerializeToString(&info_val)) {
+ code = MetaServiceCode::PROTOBUF_SERIALIZE_ERR;
+ ss << "failed to serialize txn_info when saving, txn_id=" << txn_id;
+ msg = ss.str();
+ return;
+ }
+ txn->put(info_key, info_val);
+ LOG(INFO) << "put info_key=" << hex(info_key) << " txn_id=" << txn_id;
+
+ const std::string running_key = txn_running_key({instance_id, db_id, txn_id});
+ LOG(INFO) << "remove running_key=" << hex(running_key) << " txn_id=" << txn_id;
+ txn->remove(running_key);
+
+ std::string recycle_val;
+ std::string recycle_key = recycle_txn_key({instance_id, db_id, txn_id});
+ RecycleTxnPB recycle_pb;
+ auto now_time = system_clock::now();
+ uint64_t visible_time = duration_cast<milliseconds>(now_time.time_since_epoch()).count();
+ recycle_pb.set_creation_time(visible_time);
+ recycle_pb.set_label(txn_info.label());
+
+ if (!recycle_pb.SerializeToString(&recycle_val)) {
+ code = MetaServiceCode::PROTOBUF_SERIALIZE_ERR;
+ ss << "failed to serialize recycle_pb, txn_id=" << txn_id;
+ msg = ss.str();
+ return;
+ }
+
+ txn->put(recycle_key, recycle_val);
+ LOG(INFO) << "put recycle_key=" << hex(recycle_key) << " txn_id=" << txn_id;
+
+ err = txn->commit();
+ if (err != TxnErrorCode::TXN_OK) {
+ code = cast_as<ErrCategory::COMMIT>(err);
+ ss << "failed to commit kv txn, txn_id=" << txn_id << " err=" << err;
+ msg = ss.str();
+ return;
+ }
+ }
+}
+
+TxnLazyCommitTask::TxnLazyCommitTask(const std::string& instance_id, int64_t txn_id,
+ std::shared_ptr<TxnKv> txn_kv,
+ TxnLazyCommitter* txn_lazy_committer)
+ : instance_id_(instance_id),
+ txn_id_(txn_id),
+ txn_kv_(txn_kv),
+ txn_lazy_committer_(txn_lazy_committer) {
+ DCHECK(txn_id > 0);
+}
+
+void TxnLazyCommitTask::commit() {
+ std::stringstream ss;
+ int retry_times = 0;
+ do {
+ LOG(INFO) << "lazy task commit txn_id=" << txn_id_ << " retry_times=" << retry_times;
+ do {
+ code_ = MetaServiceCode::OK;
+ msg_.clear();
+ int64_t db_id;
+ std::vector<std::pair<std::string, doris::RowsetMetaCloudPB>> tmp_rowset_metas;
+ scan_tmp_rowset(instance_id_, txn_id_, txn_kv_, code_, msg_, &db_id, &tmp_rowset_metas);
+ if (code_ != MetaServiceCode::OK) {
+ LOG(WARNING) << "scan_tmp_rowset failed, txn_id=" << txn_id_ << " code=" << code_;
+ break;
+ }
+
+ VLOG_DEBUG << "txn_id=" << txn_id_
+ << " tmp_rowset_metas.size()=" << tmp_rowset_metas.size();
+ if (tmp_rowset_metas.size() == 0) {
+ LOG(INFO) << "empty tmp_rowset_metas, txn_id=" << txn_id_;
+ }
+
+ // <partition_id, tmp_rowsets>
+ std::unordered_map<int64_t,
+ std::vector<std::pair<std::string, doris::RowsetMetaCloudPB>>>
+ partition_to_tmp_rowset_metas;
+ for (auto& [tmp_rowset_key, tmp_rowset_pb] : tmp_rowset_metas) {
+ partition_to_tmp_rowset_metas[tmp_rowset_pb.partition_id()].emplace_back();
+ partition_to_tmp_rowset_metas[tmp_rowset_pb.partition_id()].back().first =
+ tmp_rowset_key;
+ partition_to_tmp_rowset_metas[tmp_rowset_pb.partition_id()].back().second =
+ tmp_rowset_pb;
+ }
+
+ // tablet_id -> TabletIndexPB
+ std::unordered_map<int64_t, TabletIndexPB> tablet_ids;
+ for (auto& [partition_id, tmp_rowset_metas] : partition_to_tmp_rowset_metas) {
+ for (size_t i = 0; i < tmp_rowset_metas.size();
+ i += config::txn_lazy_max_rowsets_per_batch) {
+ size_t end =
+ (i + config::txn_lazy_max_rowsets_per_batch) > tmp_rowset_metas.size()
+ ? tmp_rowset_metas.size()
+ : i + config::txn_lazy_max_rowsets_per_batch;
+ std::vector<std::pair<std::string, doris::RowsetMetaCloudPB>>
+ sub_partition_tmp_rowset_metas(tmp_rowset_metas.begin() + i,
+ tmp_rowset_metas.begin() + end);
+ convert_tmp_rowsets(instance_id_, txn_id_, txn_kv_, code_, msg_, db_id,
+ sub_partition_tmp_rowset_metas, tablet_ids);
+ if (code_ != MetaServiceCode::OK) break;
+ }
+ if (code_ != MetaServiceCode::OK) break;
+
+ DCHECK(tmp_rowset_metas.size() > 0);
+ std::unique_ptr<Transaction> txn;
+ TxnErrorCode err = txn_kv_->create_txn(&txn);
+ if (err != TxnErrorCode::TXN_OK) {
+ code_ = cast_as<ErrCategory::CREATE>(err);
+ ss << "failed to create txn, txn_id=" << txn_id_ << " err=" << err;
+ msg_ = ss.str();
+ LOG(WARNING) << msg_;
+ break;
+ }
+
+ int64_t table_id = -1;
+ for (auto& [tmp_rowset_key, tmp_rowset_pb] : tmp_rowset_metas) {
+ if (table_id <= 0) {
+ table_id = tablet_ids[tmp_rowset_pb.tablet_id()].table_id();
+ }
+ txn->remove(tmp_rowset_key);
+ LOG(INFO) << "remove tmp_rowset_key=" << hex(tmp_rowset_key)
+ << " txn_id=" << txn_id_;
+ }
+
+ DCHECK(table_id > 0);
+ DCHECK(partition_id > 0);
+
+ std::string ver_val;
+ std::string ver_key =
+ partition_version_key({instance_id_, db_id, table_id, partition_id});
+ err = txn->get(ver_key, &ver_val);
+ if (TxnErrorCode::TXN_OK != err) {
+ code_ = err == TxnErrorCode::TXN_KEY_NOT_FOUND
+ ? MetaServiceCode::TXN_ID_NOT_FOUND
+ : cast_as<ErrCategory::READ>(err);
+ ss << "failed to get partiton version, txn_id=" << txn_id_
+ << " key=" << hex(ver_key) << " err=" << err;
+ msg_ = ss.str();
+ LOG(WARNING) << msg_;
+ break;
+ }
+ VersionPB version_pb;
+ if (!version_pb.ParseFromString(ver_val)) {
+ code_ = MetaServiceCode::PROTOBUF_PARSE_ERR;
+ ss << "failed to parse version pb txn_id=" << txn_id_
+ << " key=" << hex(ver_key);
+ msg_ = ss.str();
+ break;
+ }
+
+ if (version_pb.pending_txn_ids_size() > 0 &&
+ version_pb.pending_txn_ids(0) == txn_id_) {
+ DCHECK(version_pb.pending_txn_ids_size() == 1);
+ version_pb.clear_pending_txn_ids();
+ ver_val.clear();
+
+ if (version_pb.has_version()) {
+ version_pb.set_version(version_pb.version() + 1);
+ } else {
+ // first commit txn version is 2
+ version_pb.set_version(2);
+ }
+ if (!version_pb.SerializeToString(&ver_val)) {
+ code_ = MetaServiceCode::PROTOBUF_SERIALIZE_ERR;
+ ss << "failed to serialize version_pb when saving, txn_id=" << txn_id_;
+ msg_ = ss.str();
+ return;
+ }
+ txn->put(ver_key, ver_val);
+ LOG(INFO) << "put ver_key=" << hex(ver_key) << " txn_id=" << txn_id_
+ << " version_pb=" << version_pb.ShortDebugString();
+
+ err = txn->commit();
+ if (err != TxnErrorCode::TXN_OK) {
+ code_ = cast_as<ErrCategory::COMMIT>(err);
+ ss << "failed to commit kv txn, txn_id=" << txn_id_ << " err=" << err;
+ msg_ = ss.str();
+ break;
+ }
+ }
+ }
+ make_committed_txn_visible(instance_id_, db_id, txn_id_, txn_kv_, code_, msg_);
+ } while (false);
+ } while (code_ == MetaServiceCode::KV_TXN_CONFLICT &&
+ retry_times++ < config::txn_store_retry_times);
+ {
+ std::unique_lock<std::mutex> lock(mutex_);
+ this->finished_ = true;
+ }
+ this->cond_.notify_all();
+}
+
+std::pair<MetaServiceCode, std::string> TxnLazyCommitTask::wait() {
+ {
+ std::unique_lock<std::mutex> lock(mutex_);
+ cond_.wait(lock, [this]() { return this->finished_ == true; });
+ }
+ txn_lazy_committer_->remove(txn_id_);
+ return std::make_pair(this->code_, this->msg_);
+}
+
+TxnLazyCommitter::TxnLazyCommitter(std::shared_ptr<TxnKv> txn_kv) : txn_kv_(txn_kv) {
+ worker_pool_ = std::make_unique<SimpleThreadPool>(config::txn_lazy_commit_num_threads);
+ worker_pool_->start();
+}
+
+std::shared_ptr<TxnLazyCommitTask> TxnLazyCommitter::submit(const std::string& instance_id,
+ int64_t txn_id) {
+ std::shared_ptr<TxnLazyCommitTask> task;
+ {
+ std::unique_lock<std::mutex> lock(mutex_);
+ auto iter = running_tasks_.find(txn_id);
+ if (iter != running_tasks_.end()) {
+ return iter->second;
+ }
+
+ task = std::make_shared<TxnLazyCommitTask>(instance_id, txn_id, txn_kv_, this);
+ running_tasks_.emplace(txn_id, task);
+ }
+
+ worker_pool_->submit([task]() { task->commit(); });
+ DCHECK(task != nullptr);
+ return task;
+}
+
+void TxnLazyCommitter::remove(int64_t txn_id) {
+ std::unique_lock<std::mutex> lock(mutex_);
+ running_tasks_.erase(txn_id);
+}
+
+} // namespace doris::cloud
\ No newline at end of file
diff --git a/cloud/src/meta-service/txn_lazy_committer.h b/cloud/src/meta-service/txn_lazy_committer.h
new file mode 100644
index 00000000000..dca618739ec
--- /dev/null
+++ b/cloud/src/meta-service/txn_lazy_committer.h
@@ -0,0 +1,68 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <gen_cpp/cloud.pb.h>
+
+#include <atomic>
+
+#include "common/simple_thread_pool.h"
+#include "meta-service/txn_kv.h"
+
+namespace doris::cloud {
+
+class TxnLazyCommitter;
+
+class TxnLazyCommitTask {
+public:
+ TxnLazyCommitTask(const std::string& instance_id, int64_t txn_id, std::shared_ptr<TxnKv> txn_kv,
+ TxnLazyCommitter* txn_lazy_committer);
+ void commit();
+
+ std::pair<MetaServiceCode, std::string> wait();
+
+private:
+ friend class TxnLazyCommitter;
+
+ std::string instance_id_;
+ int64_t txn_id_;
+ std::shared_ptr<TxnKv> txn_kv_;
+ std::mutex mutex_;
+ std::condition_variable cond_;
+ bool finished_ = false;
+ MetaServiceCode code_ = MetaServiceCode::OK;
+ std::string msg_;
+ TxnLazyCommitter* txn_lazy_committer_;
+};
+
+class TxnLazyCommitter {
+public:
+ TxnLazyCommitter(std::shared_ptr<TxnKv> txn_kv);
+ std::shared_ptr<TxnLazyCommitTask> submit(const std::string& instance_id, int64_t txn_id);
+ void remove(int64_t txn_id);
+
+private:
+ std::shared_ptr<TxnKv> txn_kv_;
+
+ std::unique_ptr<SimpleThreadPool> worker_pool_;
+
+ std::mutex mutex_;
+ // <txn_id, TxnLazyCommitTask>
+ std::unordered_map<int64_t, std::shared_ptr<TxnLazyCommitTask>> running_tasks_;
+};
+} // namespace doris::cloud
\ No newline at end of file
diff --git a/cloud/src/recycler/recycler.cpp b/cloud/src/recycler/recycler.cpp
index 486e143f30b..76d4a7ca767 100644
--- a/cloud/src/recycler/recycler.cpp
+++ b/cloud/src/recycler/recycler.cpp
@@ -167,6 +167,7 @@ static inline void check_recycle_task(const std::string& instance_id, const std:
Recycler::Recycler(std::shared_ptr<TxnKv> txn_kv) : txn_kv_(std::move(txn_kv)) {
ip_port_ = std::string(butil::my_ip_cstr()) + ":" + std::to_string(config::brpc_listen_port);
+
auto s3_producer_pool = std::make_shared<SimpleThreadPool>(config::recycle_pool_parallelism);
s3_producer_pool->start();
auto recycle_tablet_pool = std::make_shared<SimpleThreadPool>(config::recycle_pool_parallelism);
@@ -177,6 +178,8 @@ Recycler::Recycler(std::shared_ptr<TxnKv> txn_kv) : txn_kv_(std::move(txn_kv)) {
_thread_pool_group =
RecyclerThreadPoolGroup(std::move(s3_producer_pool), std::move(recycle_tablet_pool),
std::move(group_recycle_function_pool));
+
+ txn_lazy_committer_ = std::make_shared<TxnLazyCommitter>(txn_kv_);
}
Recycler::~Recycler() {
@@ -241,8 +244,9 @@ void Recycler::recycle_callback() {
// skip instance in recycling
if (recycling_instance_map_.count(instance_id)) continue;
}
- auto instance_recycler =
- std::make_shared<InstanceRecycler>(txn_kv_, instance, _thread_pool_group);
+ auto instance_recycler = std::make_shared<InstanceRecycler>(
+ txn_kv_, instance, _thread_pool_group, txn_lazy_committer_);
+
if (instance_recycler->init() != 0) {
LOG(WARNING) << "failed to init instance recycler, instance_id=" << instance_id;
continue;
@@ -342,7 +346,8 @@ int Recycler::start(brpc::Server* server) {
if (server) {
// Add service
- auto recycler_service = new RecyclerServiceImpl(txn_kv_, this, checker_.get());
+ auto recycler_service =
+ new RecyclerServiceImpl(txn_kv_, this, checker_.get(), txn_lazy_committer_);
server->AddService(recycler_service, brpc::SERVER_OWNS_SERVICE);
}
@@ -459,12 +464,14 @@ private:
};
InstanceRecycler::InstanceRecycler(std::shared_ptr<TxnKv> txn_kv, const InstanceInfoPB& instance,
- RecyclerThreadPoolGroup thread_pool_group)
+ RecyclerThreadPoolGroup thread_pool_group,
+ std::shared_ptr<TxnLazyCommitter> txn_lazy_committer)
: txn_kv_(std::move(txn_kv)),
instance_id_(instance.instance_id()),
instance_info_(instance),
inverted_index_id_cache_(std::make_unique<InvertedIndexIdCache>(instance_id_, txn_kv_)),
- _thread_pool_group(std::move(thread_pool_group)) {}
+ _thread_pool_group(std::move(thread_pool_group)),
+ txn_lazy_committer_(std::move(txn_lazy_committer)) {};
InstanceRecycler::~InstanceRecycler() = default;
@@ -829,6 +836,79 @@ int InstanceRecycler::recycle_indexes() {
return scan_and_recycle(index_key0, index_key1, std::move(recycle_func), std::move(loop_done));
}
+bool check_lazy_txn_finished(std::shared_ptr<TxnKv> txn_kv, const std::string instance_id,
+ int64_t tablet_id) {
+ std::unique_ptr<Transaction> txn;
+ TxnErrorCode err = txn_kv->create_txn(&txn);
+ if (err != TxnErrorCode::TXN_OK) {
+ LOG(WARNING) << "failed to create txn, instance_id=" << instance_id
+ << " tablet_id=" << tablet_id << " err=" << err;
+ return false;
+ }
+
+ std::string tablet_idx_key = meta_tablet_idx_key({instance_id, tablet_id});
+ std::string tablet_idx_val;
+ err = txn->get(tablet_idx_key, &tablet_idx_val);
+ if (TxnErrorCode::TXN_OK != err) {
+ LOG(WARNING) << "failed to get tablet index, instance_id=" << instance_id
+ << " tablet_id=" << tablet_id << " err=" << err
+ << " key=" << hex(tablet_idx_key);
+ return false;
+ }
+
+ TabletIndexPB tablet_idx_pb;
+ if (!tablet_idx_pb.ParseFromString(tablet_idx_val)) {
+ LOG(WARNING) << "failed to parse tablet_idx_pb, instance_id=" << instance_id
+ << " tablet_id=" << tablet_id;
+ return false;
+ }
+
+ if (!tablet_idx_pb.has_db_id()) {
+ return true;
+ }
+
+ std::string ver_val;
+ std::string ver_key =
+ partition_version_key({instance_id, tablet_idx_pb.db_id(), tablet_idx_pb.table_id(),
+ tablet_idx_pb.partition_id()});
+ err = txn->get(ver_key, &ver_val);
+
+ if (TxnErrorCode::TXN_KEY_NOT_FOUND == err) {
+ return true;
+ }
+
+ if (TxnErrorCode::TXN_OK != err) {
+ LOG(WARNING) << "failed to get partition version, instance_id=" << instance_id
+ << " db_id=" << tablet_idx_pb.db_id()
+ << " table_id=" << tablet_idx_pb.table_id()
+ << " partition_id=" << tablet_idx_pb.partition_id()
+ << " tablet_id=" << tablet_id << " key=" << hex(ver_key) << " err=" << err;
+ return false;
+ }
+
+ VersionPB version_pb;
+ if (!version_pb.ParseFromString(ver_val)) {
+ LOG(WARNING) << "failed to parse version_pb, instance_id=" << instance_id
+ << " db_id=" << tablet_idx_pb.db_id()
+ << " table_id=" << tablet_idx_pb.table_id()
+ << " partition_id=" << tablet_idx_pb.partition_id()
+ << " tablet_id=" << tablet_id << " key=" << hex(ver_key);
+ return false;
+ }
+
+ if (version_pb.pending_txn_ids_size() > 0) {
+ DCHECK(version_pb.pending_txn_ids_size() == 1);
+ LOG(WARNING) << "lazy txn not finished, instance_id=" << instance_id
+ << " db_id=" << tablet_idx_pb.db_id()
+ << " table_id=" << tablet_idx_pb.table_id()
+ << " partition_id=" << tablet_idx_pb.partition_id()
+ << " tablet_id=" << tablet_id << " txn_id=" << version_pb.pending_txn_ids(0)
+ << " key=" << hex(ver_key);
+ return false;
+ }
+ return true;
+}
+
int InstanceRecycler::recycle_partitions() {
const std::string task_name = "recycle_partitions";
int num_scanned = 0;
@@ -928,6 +1008,7 @@ int InstanceRecycler::recycle_partitions() {
return -1;
}
}
+
int ret = 0;
for (int64_t index_id : part_pb.index_id()) {
if (recycle_tablets(part_pb.table_id(), index_id, partition_id, is_empty_tablet) != 0) {
@@ -1040,7 +1121,8 @@ int InstanceRecycler::recycle_versions() {
partition_version_key({instance_id_, db_id, table_id, INT64_MAX});
txn->remove(partition_version_key_begin, partition_version_key_end);
LOG(WARNING) << "remove partition version kv, begin=" << hex(partition_version_key_begin)
- << " end=" << hex(partition_version_key_end);
+ << " end=" << hex(partition_version_key_end) << " db_id=" << db_id
+ << " table_id=" << table_id;
// 2. Remove the table version kv of this table
auto tbl_version_key = table_version_key({instance_id_, db_id, table_id});
txn->remove(tbl_version_key);
@@ -1124,6 +1206,12 @@ int InstanceRecycler::recycle_tablets(int64_t table_id, int64_t index_id, int64_
return -1;
}
int64_t tablet_id = tablet_meta_pb.tablet_id();
+
+ if (!check_lazy_txn_finished(txn_kv_, instance_id_, tablet_meta_pb.tablet_id())) {
+ LOG(WARNING) << "lazy txn not finished tablet_id=" << tablet_meta_pb.tablet_id();
+ return -1;
+ }
+
tablet_idx_keys.push_back(meta_tablet_idx_key({instance_id_, tablet_id}));
if (!is_empty_tablet) {
sync_executor.add([this, &num_recycled, tid = tablet_id, range_move = use_range_remove,
@@ -1227,6 +1315,10 @@ int InstanceRecycler::recycle_tablets(int64_t table_id, int64_t index_id, int64_
int ret = scan_and_recycle(tablet_key_begin, tablet_key_end, std::move(recycle_func),
std::move(loop_done));
+ if (ret != 0) {
+ LOG(WARNING) << "failed to scan_and_recycle, instance_id=" << instance_id_;
+ return ret;
+ }
// directly remove tablet stats and tablet jobs of these dropped index or partition
std::unique_ptr<Transaction> txn;
@@ -1716,6 +1808,64 @@ int InstanceRecycler::recycle_rowsets() {
return ret;
}
+bool check_txn_abort(std::shared_ptr<TxnKv> txn_kv, const std::string& instance_id,
+ int64_t txn_id) {
+ std::unique_ptr<Transaction> txn;
+ TxnErrorCode err = txn_kv->create_txn(&txn);
+ if (err != TxnErrorCode::TXN_OK) {
+ LOG(WARNING) << "failed to create txn, txn_id=" << txn_id << " instance_id=" << instance_id;
+ return false;
+ }
+
+ std::string index_val;
+ const std::string index_key = txn_index_key({instance_id, txn_id});
+ err = txn->get(index_key, &index_val);
+ if (err != TxnErrorCode::TXN_OK) {
+ if (TxnErrorCode::TXN_KEY_NOT_FOUND == err) {
+ // txn has been recycled;
+ return true;
+ }
+ LOG(WARNING) << "failed to get txn index key, txn_id=" << txn_id
+ << " instance_id=" << instance_id << " key=" << hex(index_key)
+ << " err=" << err;
+ return false;
+ }
+
+ TxnIndexPB index_pb;
+ if (!index_pb.ParseFromString(index_val)) {
+ LOG(WARNING) << "failed to parse txn_index_pb, txn_id=" << txn_id
+ << " instance_id=" << instance_id;
+ return false;
+ }
+
+ DCHECK(index_pb.has_tablet_index() == true);
+ DCHECK(index_pb.tablet_index().has_db_id() == true);
+ int64_t db_id = index_pb.tablet_index().db_id();
+
+ std::string info_val;
+ const std::string info_key = txn_info_key({instance_id, db_id, txn_id});
+ err = txn->get(info_key, &info_val);
+ if (err != TxnErrorCode::TXN_OK) {
+ DCHECK(err != TxnErrorCode::TXN_KEY_NOT_FOUND);
+ LOG(WARNING) << "failed to get txn info key, txn_id=" << txn_id
+ << " instance_id=" << instance_id << " key=" << hex(info_key)
+ << " err=" << err;
+ return false;
+ }
+
+ TxnInfoPB txn_info;
+ if (!txn_info.ParseFromString(info_val)) {
+ LOG(WARNING) << "failed to parse txn_info, txn_id=" << txn_id
+ << " instance_id=" << instance_id;
+ return false;
+ }
+ DCHECK(txn_info.txn_id() == txn_id);
+ if (TxnStatusPB::TXN_STATUS_ABORTED == txn_info.status()) {
+ return true;
+ }
+ return false;
+}
+
int InstanceRecycler::recycle_tmp_rowsets() {
const std::string task_name = "recycle_tmp_rowsets";
int num_scanned = 0;
@@ -1777,6 +1927,11 @@ int InstanceRecycler::recycle_tmp_rowsets() {
// not expired
return 0;
}
+
+ if (!check_txn_abort(txn_kv_, instance_id_, rowset.txn_id())) {
+ return 0;
+ }
+
++num_expired;
expired_rowset_size += v.size();
if (!rowset.has_resource_id()) {
@@ -1861,6 +2016,7 @@ int InstanceRecycler::abort_timeout_txn() {
int num_scanned = 0;
int num_timeout = 0;
int num_abort = 0;
+ int num_advance = 0;
TxnRunningKeyInfo txn_running_key_info0 {instance_id_, 0, 0};
TxnRunningKeyInfo txn_running_key_info1 {instance_id_, INT64_MAX, INT64_MAX};
@@ -1882,24 +2038,17 @@ int InstanceRecycler::abort_timeout_txn() {
.tag("instance_id", instance_id_)
.tag("num_scanned", num_scanned)
.tag("num_timeout", num_timeout)
- .tag("num_abort", num_abort);
+ .tag("num_abort", num_abort)
+ .tag("num_advance", num_advance);
});
int64_t current_time =
duration_cast<milliseconds>(system_clock::now().time_since_epoch()).count();
- auto handle_txn_running_kv = [&num_scanned, &num_timeout, &num_abort, ¤t_time, this](
- std::string_view k, std::string_view v) -> int {
+ auto handle_txn_running_kv = [&num_scanned, &num_timeout, &num_abort, &num_advance,
+ ¤t_time,
+ this](std::string_view k, std::string_view v) -> int {
++num_scanned;
- TxnRunningPB txn_running_pb;
- if (!txn_running_pb.ParseFromArray(v.data(), v.size())) {
- LOG_WARNING("malformed txn_running_pb").tag("key", hex(k));
- return -1;
- }
- if (txn_running_pb.timeout_time() > current_time) {
- return 0;
- }
- ++num_timeout;
std::unique_ptr<Transaction> txn;
TxnErrorCode err = txn_kv_->create_txn(&txn);
@@ -1931,42 +2080,69 @@ int InstanceRecycler::abort_timeout_txn() {
LOG_WARNING("failed to parse txn info").tag("key", hex(k));
return -1;
}
- txn_info.set_status(TxnStatusPB::TXN_STATUS_ABORTED);
- txn_info.set_finish_time(current_time);
- txn_info.set_reason("timeout");
- VLOG_DEBUG << "txn_info=" << txn_info.DebugString();
- txn_inf_val.clear();
- if (!txn_info.SerializeToString(&txn_inf_val)) {
- LOG_WARNING("failed to serialize txn info").tag("key", hex(k));
- return -1;
- }
- txn->put(txn_inf_key, txn_inf_val);
- VLOG_DEBUG << "txn->put, txn_inf_key=" << hex(txn_inf_key);
- // Put recycle txn key
- std::string recyc_txn_key, recyc_txn_val;
- recycle_txn_key({instance_id_, db_id, txn_id}, &recyc_txn_key);
- RecycleTxnPB recycle_txn_pb;
- recycle_txn_pb.set_creation_time(current_time);
- recycle_txn_pb.set_label(txn_info.label());
- if (!recycle_txn_pb.SerializeToString(&recyc_txn_val)) {
- LOG_WARNING("failed to serialize txn recycle info")
- .tag("key", hex(k))
- .tag("db_id", db_id)
- .tag("txn_id", txn_id);
- return -1;
- }
- txn->put(recyc_txn_key, recyc_txn_val);
- // Remove txn running key
- txn->remove(k);
- err = txn->commit();
- if (err != TxnErrorCode::TXN_OK) {
- LOG_WARNING("failed to commit txn err={}", err)
- .tag("key", hex(k))
- .tag("db_id", db_id)
- .tag("txn_id", txn_id);
- return -1;
+
+ if (TxnStatusPB::TXN_STATUS_COMMITTED == txn_info.status()) {
+ txn.reset();
+ std::shared_ptr<TxnLazyCommitTask> task =
+ txn_lazy_committer_->submit(instance_id_, txn_info.txn_id());
+ std::pair<MetaServiceCode, std::string> ret = task->wait();
+ if (ret.first != MetaServiceCode::OK) {
+ LOG(WARNING) << "lazy commit txn failed txn_id=" << txn_id << " code=" << ret.first
+ << "msg=" << ret.second;
+ return -1;
+ }
+ ++num_advance;
+ return 0;
+ } else {
+ TxnRunningPB txn_running_pb;
+ if (!txn_running_pb.ParseFromArray(v.data(), v.size())) {
+ LOG_WARNING("malformed txn_running_pb").tag("key", hex(k));
+ return -1;
+ }
+ if (txn_running_pb.timeout_time() > current_time) {
+ return 0;
+ }
+ ++num_timeout;
+
+ DCHECK(txn_info.status() != TxnStatusPB::TXN_STATUS_VISIBLE);
+ txn_info.set_status(TxnStatusPB::TXN_STATUS_ABORTED);
+ txn_info.set_finish_time(current_time);
+ txn_info.set_reason("timeout");
+ VLOG_DEBUG << "txn_info=" << txn_info.DebugString();
+ txn_inf_val.clear();
+ if (!txn_info.SerializeToString(&txn_inf_val)) {
+ LOG_WARNING("failed to serialize txn info").tag("key", hex(k));
+ return -1;
+ }
+ txn->put(txn_inf_key, txn_inf_val);
+ VLOG_DEBUG << "txn->put, txn_inf_key=" << hex(txn_inf_key);
+ // Put recycle txn key
+ std::string recyc_txn_key, recyc_txn_val;
+ recycle_txn_key({instance_id_, db_id, txn_id}, &recyc_txn_key);
+ RecycleTxnPB recycle_txn_pb;
+ recycle_txn_pb.set_creation_time(current_time);
+ recycle_txn_pb.set_label(txn_info.label());
+ if (!recycle_txn_pb.SerializeToString(&recyc_txn_val)) {
+ LOG_WARNING("failed to serialize txn recycle info")
+ .tag("key", hex(k))
+ .tag("db_id", db_id)
+ .tag("txn_id", txn_id);
+ return -1;
+ }
+ txn->put(recyc_txn_key, recyc_txn_val);
+ // Remove txn running key
+ txn->remove(k);
+ err = txn->commit();
+ if (err != TxnErrorCode::TXN_OK) {
+ LOG_WARNING("failed to commit txn err={}", err)
+ .tag("key", hex(k))
+ .tag("db_id", db_id)
+ .tag("txn_id", txn_id);
+ return -1;
+ }
+ ++num_abort;
}
- ++num_abort;
+
return 0;
};
diff --git a/cloud/src/recycler/recycler.h b/cloud/src/recycler/recycler.h
index 950c0193f77..91a461f474f 100644
--- a/cloud/src/recycler/recycler.h
+++ b/cloud/src/recycler/recycler.h
@@ -28,6 +28,7 @@
#include <string_view>
#include <thread>
+#include "meta-service/txn_lazy_committer.h"
#include "recycler/storage_vault_accessor.h"
#include "recycler/white_black_list.h"
@@ -101,13 +102,17 @@ private:
WhiteBlackList instance_filter_;
std::unique_ptr<Checker> checker_;
+
RecyclerThreadPoolGroup _thread_pool_group;
+
+ std::shared_ptr<TxnLazyCommitter> txn_lazy_committer_;
};
class InstanceRecycler {
public:
explicit InstanceRecycler(std::shared_ptr<TxnKv> txn_kv, const InstanceInfoPB& instance,
- RecyclerThreadPoolGroup thread_pool_group);
+ RecyclerThreadPoolGroup thread_pool_group,
+ std::shared_ptr<TxnLazyCommitter> txn_lazy_committer);
~InstanceRecycler();
// returns 0 for success otherwise error
@@ -239,7 +244,10 @@ private:
std::mutex recycle_tasks_mutex;
// <task_name, start_time>>
std::map<std::string, int64_t> running_recycle_tasks;
+
RecyclerThreadPoolGroup _thread_pool_group;
+
+ std::shared_ptr<TxnLazyCommitter> txn_lazy_committer_;
};
} // namespace doris::cloud
diff --git a/cloud/src/recycler/recycler_service.cpp b/cloud/src/recycler/recycler_service.cpp
index f9ffe750fdf..08e937a4106 100644
--- a/cloud/src/recycler/recycler_service.cpp
+++ b/cloud/src/recycler/recycler_service.cpp
@@ -42,8 +42,12 @@ extern int reset_s3_rate_limiter(S3RateLimitType type, size_t max_speed, size_t
extern std::tuple<int, std::string_view> convert_ms_code_to_http_code(MetaServiceCode ret);
RecyclerServiceImpl::RecyclerServiceImpl(std::shared_ptr<TxnKv> txn_kv, Recycler* recycler,
- Checker* checker)
- : txn_kv_(std::move(txn_kv)), recycler_(recycler), checker_(checker) {}
+ Checker* checker,
+ std::shared_ptr<TxnLazyCommitter> txn_lazy_committer)
+ : txn_kv_(std::move(txn_kv)),
+ recycler_(recycler),
+ checker_(checker),
+ txn_lazy_committer_(std::move(txn_lazy_committer)) {}
RecyclerServiceImpl::~RecyclerServiceImpl() = default;
@@ -152,7 +156,8 @@ void RecyclerServiceImpl::check_instance(const std::string& instance_id, MetaSer
void recycle_copy_jobs(const std::shared_ptr<TxnKv>& txn_kv, const std::string& instance_id,
MetaServiceCode& code, std::string& msg,
- RecyclerThreadPoolGroup thread_pool_group) {
+ RecyclerThreadPoolGroup thread_pool_group,
+ std::shared_ptr<TxnLazyCommitter> txn_lazy_committer) {
std::unique_ptr<Transaction> txn;
TxnErrorCode err = txn_kv->create_txn(&txn);
if (err != TxnErrorCode::TXN_OK) {
@@ -190,7 +195,8 @@ void recycle_copy_jobs(const std::shared_ptr<TxnKv>& txn_kv, const std::string&
}
}
- auto recycler = std::make_unique<InstanceRecycler>(txn_kv, instance, thread_pool_group);
+ auto recycler = std::make_unique<InstanceRecycler>(txn_kv, instance, thread_pool_group,
+ txn_lazy_committer);
if (recycler->init() != 0) {
LOG(WARNING) << "failed to init InstanceRecycler recycle_copy_jobs on instance "
<< instance_id;
@@ -333,7 +339,9 @@ void RecyclerServiceImpl::http(::google::protobuf::RpcController* controller,
status_code = 400;
return;
}
- recycle_copy_jobs(txn_kv_, *instance_id, code, msg, recycler_->_thread_pool_group);
+ recycle_copy_jobs(txn_kv_, *instance_id, code, msg, recycler_->_thread_pool_group,
+ txn_lazy_committer_);
+
response_body = msg;
return;
}
diff --git a/cloud/src/recycler/recycler_service.h b/cloud/src/recycler/recycler_service.h
index 8f9faec1efe..5ece69c86ba 100644
--- a/cloud/src/recycler/recycler_service.h
+++ b/cloud/src/recycler/recycler_service.h
@@ -20,6 +20,7 @@
#include <gen_cpp/cloud.pb.h>
#include "meta-service/txn_kv.h"
+#include "meta-service/txn_lazy_committer.h"
namespace doris::cloud {
@@ -28,7 +29,8 @@ class Checker;
class RecyclerServiceImpl : public cloud::RecyclerService {
public:
- RecyclerServiceImpl(std::shared_ptr<TxnKv> txn_kv, Recycler* recycler, Checker* checker);
+ RecyclerServiceImpl(std::shared_ptr<TxnKv> txn_kv, Recycler* recycler, Checker* checker,
+ std::shared_ptr<TxnLazyCommitter> txn_lazy_committer);
~RecyclerServiceImpl() override;
void recycle_instance(::google::protobuf::RpcController* controller,
@@ -48,6 +50,7 @@ private:
std::shared_ptr<TxnKv> txn_kv_;
Recycler* recycler_; // Ref
Checker* checker_; // Ref
+ std::shared_ptr<TxnLazyCommitter> txn_lazy_committer_;
};
} // namespace doris::cloud
diff --git a/cloud/test/CMakeLists.txt b/cloud/test/CMakeLists.txt
index ba5e2909918..b504a882534 100644
--- a/cloud/test/CMakeLists.txt
+++ b/cloud/test/CMakeLists.txt
@@ -56,6 +56,8 @@ add_executable(util_test util_test.cpp)
add_executable(network_util_test network_util_test.cpp)
+add_executable(txn_lazy_commit_test txn_lazy_commit_test.cpp)
+
message("Meta-service test dependencies: ${TEST_LINK_LIBS}")
#target_link_libraries(sync_point_test ${TEST_LINK_LIBS})
@@ -91,6 +93,8 @@ target_link_libraries(util_test ${TEST_LINK_LIBS})
target_link_libraries(network_util_test ${TEST_LINK_LIBS})
+target_link_libraries(txn_lazy_commit_test ${TEST_LINK_LIBS})
+
# FDB related tests need to be linked with libfdb_c
set(FDB_LINKER_FLAGS "-lfdb_c -L${THIRDPARTY_DIR}/lib")
diff --git a/cloud/test/meta_service_test.cpp b/cloud/test/meta_service_test.cpp
index 3a04122e487..c7d27713c9b 100644
--- a/cloud/test/meta_service_test.cpp
+++ b/cloud/test/meta_service_test.cpp
@@ -163,7 +163,7 @@ static void commit_txn(MetaServiceProxy* meta_service, int64_t db_id, int64_t tx
}
static doris::RowsetMetaCloudPB create_rowset(int64_t txn_id, int64_t tablet_id,
- int partition_id = 0, int64_t version = -1,
+ int partition_id = 10, int64_t version = -1,
int num_rows = 100) {
doris::RowsetMetaCloudPB rowset;
rowset.set_rowset_id(0); // required
diff --git a/cloud/test/recycler_test.cpp b/cloud/test/recycler_test.cpp
index ae4f35a4605..14687354839 100644
--- a/cloud/test/recycler_test.cpp
+++ b/cloud/test/recycler_test.cpp
@@ -261,17 +261,27 @@ static int create_tablet(TxnKv* txn_kv, int64_t table_id, int64_t index_id, int6
if (txn_kv->create_txn(&txn) != TxnErrorCode::TXN_OK) {
return -1;
}
- auto key = meta_tablet_key({instance_id, table_id, index_id, partition_id, tablet_id});
doris::TabletMetaCloudPB tablet_meta;
tablet_meta.set_tablet_id(tablet_id);
auto val = tablet_meta.SerializeAsString();
+ auto key = meta_tablet_key({instance_id, table_id, index_id, partition_id, tablet_id});
txn->put(key, val);
- key = meta_tablet_idx_key({instance_id, tablet_id});
- txn->put(key, val); // val is not necessary
key = stats_tablet_key({instance_id, table_id, index_id, partition_id, tablet_id});
txn->put(key, val); // val is not necessary
key = job_tablet_key({instance_id, table_id, index_id, partition_id, tablet_id});
txn->put(key, val); // val is not necessary
+
+ // mock tablet index
+ TabletIndexPB tablet_idx_pb;
+ tablet_idx_pb.set_db_id(db_id);
+ tablet_idx_pb.set_table_id(table_id);
+ tablet_idx_pb.set_partition_id(partition_id);
+ tablet_idx_pb.set_tablet_id(tablet_id);
+ auto idx_val = tablet_idx_pb.SerializeAsString();
+ key = meta_tablet_idx_key({instance_id, tablet_id});
+ txn->put(key, idx_val);
+ LOG(INFO) << "tablet_idx_pb:" << tablet_idx_pb.DebugString() << " key=" << hex(key);
+
if (txn->commit() != TxnErrorCode::TXN_OK) {
return -1;
}
@@ -632,7 +642,8 @@ TEST(RecyclerTest, recycle_empty) {
obj_info->set_bucket(config::test_s3_bucket);
obj_info->set_prefix("recycle_empty");
- InstanceRecycler recycler(txn_kv, instance, thread_group);
+ InstanceRecycler recycler(txn_kv, instance, thread_group,
+ std::make_shared<TxnLazyCommitter>(txn_kv));
ASSERT_EQ(recycler.init(), 0);
ASSERT_EQ(recycler.recycle_rowsets(), 0);
@@ -665,7 +676,8 @@ TEST(RecyclerTest, recycle_rowsets) {
sp->set_call_back("InvertedIndexIdCache::insert2", [&](auto&&) { ++insert_inverted_index; });
sp->enable_processing();
- InstanceRecycler recycler(txn_kv, instance, thread_group);
+ InstanceRecycler recycler(txn_kv, instance, thread_group,
+ std::make_shared<TxnLazyCommitter>(txn_kv));
ASSERT_EQ(recycler.init(), 0);
std::vector<doris::TabletSchemaCloudPB> schemas;
@@ -730,7 +742,8 @@ TEST(RecyclerTest, bench_recycle_rowsets) {
config::instance_recycler_worker_pool_size = 10;
config::recycle_task_threshold_seconds = 0;
- InstanceRecycler recycler(txn_kv, instance, thread_group);
+ InstanceRecycler recycler(txn_kv, instance, thread_group,
+ std::make_shared<TxnLazyCommitter>(txn_kv));
ASSERT_EQ(recycler.init(), 0);
auto sp = SyncPoint::get_instance();
@@ -813,7 +826,8 @@ TEST(RecyclerTest, recycle_tmp_rowsets) {
sp->set_call_back("InvertedIndexIdCache::insert2", [&](auto&&) { ++insert_inverted_index; });
sp->enable_processing();
- InstanceRecycler recycler(txn_kv, instance, thread_group);
+ InstanceRecycler recycler(txn_kv, instance, thread_group,
+ std::make_shared<TxnLazyCommitter>(txn_kv));
ASSERT_EQ(recycler.init(), 0);
std::vector<doris::TabletSchemaCloudPB> schemas;
@@ -875,7 +889,8 @@ TEST(RecyclerTest, recycle_tablet) {
obj_info->set_bucket(config::test_s3_bucket);
obj_info->set_prefix("recycle_tablet");
- InstanceRecycler recycler(txn_kv, instance, thread_group);
+ InstanceRecycler recycler(txn_kv, instance, thread_group,
+ std::make_shared<TxnLazyCommitter>(txn_kv));
ASSERT_EQ(recycler.init(), 0);
std::vector<doris::TabletSchemaCloudPB> schemas;
@@ -902,6 +917,8 @@ TEST(RecyclerTest, recycle_tablet) {
create_committed_rowset(txn_kv.get(), accessor.get(), "recycle_tablet", tablet_id, i);
}
+ ASSERT_EQ(create_partition_version_kv(txn_kv.get(), table_id, partition_id), 0);
+
ASSERT_EQ(0, recycler.recycle_tablets(table_id, index_id));
// check rowset does not exist on s3
@@ -950,7 +967,8 @@ TEST(RecyclerTest, recycle_indexes) {
obj_info->set_bucket(config::test_s3_bucket);
obj_info->set_prefix("recycle_indexes");
- InstanceRecycler recycler(txn_kv, instance, thread_group);
+ InstanceRecycler recycler(txn_kv, instance, thread_group,
+ std::make_shared<TxnLazyCommitter>(txn_kv));
ASSERT_EQ(recycler.init(), 0);
std::vector<doris::TabletSchemaCloudPB> schemas;
@@ -985,6 +1003,8 @@ TEST(RecyclerTest, recycle_indexes) {
create_committed_rowset(txn_kv.get(), accessor.get(), "recycle_indexes", tablet_id, j);
}
}
+
+ ASSERT_EQ(create_partition_version_kv(txn_kv.get(), table_id, partition_id), 0);
create_recycle_index(txn_kv.get(), table_id, index_id);
ASSERT_EQ(recycler.recycle_indexes(), 0);
@@ -1062,7 +1082,8 @@ TEST(RecyclerTest, recycle_partitions) {
obj_info->set_bucket(config::test_s3_bucket);
obj_info->set_prefix("recycle_partitions");
- InstanceRecycler recycler(txn_kv, instance, thread_group);
+ InstanceRecycler recycler(txn_kv, instance, thread_group,
+ std::make_shared<TxnLazyCommitter>(txn_kv));
ASSERT_EQ(recycler.init(), 0);
std::vector<doris::TabletSchemaCloudPB> schemas;
@@ -1084,8 +1105,8 @@ TEST(RecyclerTest, recycle_partitions) {
int64_t tablet_id_base = 10100;
for (auto index_id : index_ids) {
for (int i = 0; i < 20; ++i) {
- int64_t tablet_id = tablet_id_base + i;
- create_tablet(txn_kv.get(), table_id, index_id, partition_id, tablet_id);
+ int64_t tablet_id = tablet_id_base++;
+ ASSERT_EQ(create_tablet(txn_kv.get(), table_id, index_id, partition_id, tablet_id), 0);
for (int j = 0; j < 10; ++j) {
auto rowset =
create_rowset("recycle_tablet", tablet_id, index_id, 5, schemas[j % 5]);
@@ -1099,6 +1120,9 @@ TEST(RecyclerTest, recycle_partitions) {
}
}
}
+
+ ASSERT_EQ(create_partition_version_kv(txn_kv.get(), table_id, partition_id), 0);
+
create_recycle_partiton(txn_kv.get(), table_id, partition_id, index_ids);
ASSERT_EQ(recycler.recycle_partitions(), 0);
@@ -1154,7 +1178,7 @@ TEST(RecyclerTest, recycle_versions) {
std::vector<int64_t> index_ids {20001, 20002, 20003, 20004, 20005};
std::vector<int64_t> partition_ids {30001, 30002, 30003, 30004, 30005, 30006};
- constexpr int64_t table_id = 10000;
+ constexpr int table_id = 10000;
int64_t tablet_id = 40000;
for (auto index_id : index_ids) {
@@ -1173,7 +1197,8 @@ TEST(RecyclerTest, recycle_versions) {
InstanceInfoPB instance;
instance.set_instance_id(instance_id);
- InstanceRecycler recycler(txn_kv, instance, thread_group);
+ InstanceRecycler recycler(txn_kv, instance, thread_group,
+ std::make_shared<TxnLazyCommitter>(txn_kv));
ASSERT_EQ(recycler.init(), 0);
// Recycle all partitions in table except 30006
ASSERT_EQ(recycler.recycle_partitions(), 0);
@@ -1211,7 +1236,7 @@ TEST(RecyclerTest, recycle_versions) {
ASSERT_EQ(iter->size(), 0);
}
-TEST(RecyclerTest, abort_timeout_txn) {
+TEST(RecyclerTest, advance_pending_txn) {
auto txn_kv = std::dynamic_pointer_cast<TxnKv>(std::make_shared<MemTxnKv>());
ASSERT_NE(txn_kv.get(), nullptr);
auto rs = std::make_shared<MockResourceManager>(txn_kv);
@@ -1229,7 +1254,7 @@ TEST(RecyclerTest, abort_timeout_txn) {
req.set_cloud_unique_id("test_cloud_unique_id");
TxnInfoPB txn_info_pb;
txn_info_pb.set_db_id(db_id);
- txn_info_pb.set_label("abort_timeout_txn");
+ txn_info_pb.set_label("advance_pending_txn");
txn_info_pb.add_table_ids(table_id);
txn_info_pb.set_timeout_ms(1);
req.mutable_txn_info()->CopyFrom(txn_info_pb);
@@ -1242,7 +1267,8 @@ TEST(RecyclerTest, abort_timeout_txn) {
}
InstanceInfoPB instance;
instance.set_instance_id(mock_instance);
- InstanceRecycler recycler(txn_kv, instance, thread_group);
+ InstanceRecycler recycler(txn_kv, instance, thread_group,
+ std::make_shared<TxnLazyCommitter>(txn_kv));
ASSERT_EQ(recycler.init(), 0);
sleep(1);
ASSERT_EQ(recycler.abort_timeout_txn(), 0);
@@ -1251,7 +1277,7 @@ TEST(RecyclerTest, abort_timeout_txn) {
ASSERT_EQ(txn_info_pb.status(), TxnStatusPB::TXN_STATUS_ABORTED);
}
-TEST(RecyclerTest, abort_timeout_txn_and_rebegin) {
+TEST(RecyclerTest, advance_pending_txn_and_rebegin) {
config::label_keep_max_second = 0;
auto txn_kv = std::dynamic_pointer_cast<TxnKv>(std::make_shared<MemTxnKv>());
ASSERT_NE(txn_kv.get(), nullptr);
@@ -1264,7 +1290,7 @@ TEST(RecyclerTest, abort_timeout_txn_and_rebegin) {
int64_t table_id = 1234;
int64_t txn_id = -1;
std::string cloud_unique_id = "test_cloud_unique_id22131";
- std::string label = "abort_timeout_txn_and_rebegin";
+ std::string label = "advance_pending_txn_and_rebegin";
{
brpc::Controller cntl;
BeginTxnRequest req;
@@ -1285,7 +1311,8 @@ TEST(RecyclerTest, abort_timeout_txn_and_rebegin) {
}
InstanceInfoPB instance;
instance.set_instance_id(mock_instance);
- InstanceRecycler recycler(txn_kv, instance, thread_group);
+ InstanceRecycler recycler(txn_kv, instance, thread_group,
+ std::make_shared<TxnLazyCommitter>(txn_kv));
ASSERT_EQ(recycler.init(), 0);
sleep(1);
ASSERT_EQ(recycler.abort_timeout_txn(), 0);
@@ -1352,7 +1379,8 @@ TEST(RecyclerTest, recycle_expired_txn_label) {
}
InstanceInfoPB instance;
instance.set_instance_id(mock_instance);
- InstanceRecycler recycler(txn_kv, instance, thread_group);
+ InstanceRecycler recycler(txn_kv, instance, thread_group,
+ std::make_shared<TxnLazyCommitter>(txn_kv));
ASSERT_EQ(recycler.init(), 0);
recycler.abort_timeout_txn();
TxnInfoPB txn_info_pb;
@@ -1403,7 +1431,8 @@ TEST(RecyclerTest, recycle_expired_txn_label) {
}
InstanceInfoPB instance;
instance.set_instance_id(mock_instance);
- InstanceRecycler recycler(txn_kv, instance, thread_group);
+ InstanceRecycler recycler(txn_kv, instance, thread_group,
+ std::make_shared<TxnLazyCommitter>(txn_kv));
ASSERT_EQ(recycler.init(), 0);
sleep(1);
recycler.abort_timeout_txn();
@@ -1455,7 +1484,8 @@ TEST(RecyclerTest, recycle_expired_txn_label) {
}
InstanceInfoPB instance;
instance.set_instance_id(mock_instance);
- InstanceRecycler recycler(txn_kv, instance, thread_group);
+ InstanceRecycler recycler(txn_kv, instance, thread_group,
+ std::make_shared<TxnLazyCommitter>(txn_kv));
ASSERT_EQ(recycler.init(), 0);
sleep(1);
recycler.abort_timeout_txn();
@@ -1514,7 +1544,8 @@ TEST(RecyclerTest, recycle_expired_txn_label) {
}
InstanceInfoPB instance;
instance.set_instance_id(mock_instance);
- InstanceRecycler recycler(txn_kv, instance, thread_group);
+ InstanceRecycler recycler(txn_kv, instance, thread_group,
+ std::make_shared<TxnLazyCommitter>(txn_kv));
ASSERT_EQ(recycler.init(), 0);
sleep(1);
recycler.abort_timeout_txn();
@@ -1651,7 +1682,8 @@ TEST(RecyclerTest, recycle_copy_jobs) {
InstanceInfoPB instance_info;
create_instance(internal_stage_id, external_stage_id, instance_info);
- InstanceRecycler recycler(txn_kv, instance_info, thread_group);
+ InstanceRecycler recycler(txn_kv, instance_info, thread_group,
+ std::make_shared<TxnLazyCommitter>(txn_kv));
ASSERT_EQ(recycler.init(), 0);
auto internal_accessor = recycler.accessor_map_.find(internal_stage_id)->second;
@@ -1810,7 +1842,8 @@ TEST(RecyclerTest, recycle_batch_copy_jobs) {
InstanceInfoPB instance_info;
create_instance(internal_stage_id, external_stage_id, instance_info);
- InstanceRecycler recycler(txn_kv, instance_info, thread_group);
+ InstanceRecycler recycler(txn_kv, instance_info, thread_group,
+ std::make_shared<TxnLazyCommitter>(txn_kv));
ASSERT_EQ(recycler.init(), 0);
const auto& internal_accessor = recycler.accessor_map_.find(internal_stage_id)->second;
@@ -1924,7 +1957,8 @@ TEST(RecyclerTest, recycle_stage) {
instance.set_instance_id(mock_instance);
instance.add_obj_info()->CopyFrom(object_info);
- InstanceRecycler recycler(txn_kv, instance, thread_group);
+ InstanceRecycler recycler(txn_kv, instance, thread_group,
+ std::make_shared<TxnLazyCommitter>(txn_kv));
ASSERT_EQ(recycler.init(), 0);
auto accessor = recycler.accessor_map_.begin()->second;
for (int i = 0; i < 10; ++i) {
@@ -1984,7 +2018,8 @@ TEST(RecyclerTest, recycle_deleted_instance) {
InstanceInfoPB instance_info;
create_instance(internal_stage_id, external_stage_id, instance_info);
- InstanceRecycler recycler(txn_kv, instance_info, thread_group);
+ InstanceRecycler recycler(txn_kv, instance_info, thread_group,
+ std::make_shared<TxnLazyCommitter>(txn_kv));
ASSERT_EQ(recycler.init(), 0);
// create txn key
for (size_t i = 0; i < 100; i++) {
@@ -2569,7 +2604,8 @@ TEST(RecyclerTest, delete_rowset_data) {
}
{
- InstanceRecycler recycler(txn_kv, instance, thread_group);
+ InstanceRecycler recycler(txn_kv, instance, thread_group,
+ std::make_shared<TxnLazyCommitter>(txn_kv));
ASSERT_EQ(recycler.init(), 0);
auto accessor = recycler.accessor_map_.begin()->second;
int64_t txn_id_base = 114115;
@@ -2603,7 +2639,8 @@ TEST(RecyclerTest, delete_rowset_data) {
tmp_obj_info->set_bucket(config::test_s3_bucket);
tmp_obj_info->set_prefix(resource_id);
- InstanceRecycler recycler(txn_kv, tmp_instance, thread_group);
+ InstanceRecycler recycler(txn_kv, tmp_instance, thread_group,
+ std::make_shared<TxnLazyCommitter>(txn_kv));
ASSERT_EQ(recycler.init(), 0);
auto accessor = recycler.accessor_map_.begin()->second;
// Delete multiple rowset files using one series of RowsetPB
@@ -2623,7 +2660,8 @@ TEST(RecyclerTest, delete_rowset_data) {
ASSERT_FALSE(list_iter->has_next());
}
{
- InstanceRecycler recycler(txn_kv, instance, thread_group);
+ InstanceRecycler recycler(txn_kv, instance, thread_group,
+ std::make_shared<TxnLazyCommitter>(txn_kv));
ASSERT_EQ(recycler.init(), 0);
auto accessor = recycler.accessor_map_.begin()->second;
// Delete multiple rowset files using one series of RowsetPB
diff --git a/cloud/test/txn_lazy_commit_test.cpp b/cloud/test/txn_lazy_commit_test.cpp
new file mode 100644
index 00000000000..6b4518c09ab
--- /dev/null
+++ b/cloud/test/txn_lazy_commit_test.cpp
@@ -0,0 +1,216 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <brpc/controller.h>
+#include <bvar/window.h>
+#include <fmt/core.h>
+#include <gen_cpp/cloud.pb.h>
+#include <gen_cpp/olap_file.pb.h>
+#include <google/protobuf/repeated_field.h>
+#include <gtest/gtest.h>
+
+#include <atomic>
+#include <condition_variable>
+#include <cstdint>
+#include <memory>
+#include <random>
+#include <string>
+#include <thread>
+
+#include "common/config.h"
+#include "common/logging.h"
+#include "common/util.h"
+#include "cpp/sync_point.h"
+#include "meta-service/keys.h"
+#include "meta-service/mem_txn_kv.h"
+#include "meta-service/meta_service.h"
+#include "meta-service/meta_service_helper.h"
+#include "meta-service/txn_kv_error.h"
+#include "mock_resource_manager.h"
+#include "rate-limiter/rate_limiter.h"
+#include "resource-manager/resource_manager.h"
+
+int main(int argc, char** argv) {
+ const std::string conf_file = "doris_cloud.conf";
+ if (!doris::cloud::config::init(conf_file.c_str(), true)) {
+ std::cerr << "failed to init config file, conf=" << conf_file << std::endl;
+ return -1;
+ }
+
+ config::enable_cloud_txn_lazy_commit = true;
+ config::txn_lazy_commit_rowsets_thresold = 1;
+ config::txn_lazy_max_rowsets_per_batch = 1;
+ config::txn_lazy_commit_num_threads = 2;
+
+ if (!doris::cloud::init_glog("txn_lazy_commit_test")) {
+ std::cerr << "failed to init glog" << std::endl;
+ return -1;
+ }
+ ::testing::InitGoogleTest(&argc, argv);
+ return RUN_ALL_TESTS();
+}
+namespace doris::cloud {
+
+std::unique_ptr<MetaServiceProxy> get_meta_service(std::shared_ptr<TxnKv> txn_kv,
+ bool mock_resource_mgr) {
+ std::unique_ptr<Transaction> txn;
+ EXPECT_EQ(txn_kv->create_txn(&txn), TxnErrorCode::TXN_OK);
+ txn->remove("\x00", "\xfe"); // This is dangerous if the fdb is not correctly set
+ EXPECT_EQ(txn->commit(), TxnErrorCode::TXN_OK);
+
+ auto rs = mock_resource_mgr ? std::make_shared<MockResourceManager>(txn_kv)
+ : std::make_shared<ResourceManager>(txn_kv);
+ auto rl = std::make_shared<RateLimiter>();
+ auto meta_service = std::make_unique<MetaServiceImpl>(txn_kv, rs, rl);
+ return std::make_unique<MetaServiceProxy>(std::move(meta_service));
+}
+
+static std::string next_rowset_id() {
+ static int cnt = 0;
+ return std::to_string(++cnt);
+}
+
+static void add_tablet(CreateTabletsRequest& req, int64_t table_id, int64_t index_id,
+ int64_t partition_id, int64_t tablet_id) {
+ auto tablet = req.add_tablet_metas();
+ tablet->set_table_id(table_id);
+ tablet->set_index_id(index_id);
+ tablet->set_partition_id(partition_id);
+ tablet->set_tablet_id(tablet_id);
+ auto schema = tablet->mutable_schema();
+ schema->set_schema_version(0);
+ auto first_rowset = tablet->add_rs_metas();
+ first_rowset->set_rowset_id(0); // required
+ first_rowset->set_rowset_id_v2(next_rowset_id());
+ first_rowset->set_start_version(0);
+ first_rowset->set_end_version(1);
+ first_rowset->mutable_tablet_schema()->CopyFrom(*schema);
+}
+
+static void create_tablet(MetaServiceProxy* meta_service, int64_t table_id, int64_t index_id,
+ int64_t partition_id, int64_t tablet_id) {
+ brpc::Controller cntl;
+ CreateTabletsRequest req;
+ CreateTabletsResponse res;
+ add_tablet(req, table_id, index_id, partition_id, tablet_id);
+ meta_service->create_tablets(&cntl, &req, &res, nullptr);
+ ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << tablet_id;
+}
+
+static doris::RowsetMetaCloudPB create_rowset(int64_t txn_id, int64_t tablet_id,
+ int partition_id = 10, int64_t version = -1,
+ int num_rows = 100) {
+ doris::RowsetMetaCloudPB rowset;
+ rowset.set_rowset_id(0); // required
+ rowset.set_rowset_id_v2(next_rowset_id());
+ rowset.set_tablet_id(tablet_id);
+ rowset.set_partition_id(partition_id);
+ rowset.set_txn_id(txn_id);
+ if (version > 0) {
+ rowset.set_start_version(version);
+ rowset.set_end_version(version);
+ }
+ rowset.set_num_segments(1);
+ rowset.set_num_rows(num_rows);
+ rowset.set_data_disk_size(num_rows * 100);
+ rowset.mutable_tablet_schema()->set_schema_version(0);
+ rowset.set_txn_expiration(::time(nullptr)); // Required by DCHECK
+ return rowset;
+}
+
+static void commit_rowset(MetaServiceProxy* meta_service, const doris::RowsetMetaCloudPB& rowset,
+ CreateRowsetResponse& res) {
+ brpc::Controller cntl;
+ auto arena = res.GetArena();
+ auto req = google::protobuf::Arena::CreateMessage<CreateRowsetRequest>(arena);
+ req->mutable_rowset_meta()->CopyFrom(rowset);
+ meta_service->commit_rowset(&cntl, req, &res, nullptr);
+ if (!arena) {
+ delete req;
+ }
+}
+
+TEST(TxnLazyCommitTest, CommitTxnEventuallyTest) {
+ int ret = 0;
+ // MemKv
+ auto txn_kv = std::dynamic_pointer_cast<TxnKv>(std::make_shared<MemTxnKv>());
+ if (txn_kv != nullptr) {
+ ret = txn_kv->init();
+ [&] { ASSERT_EQ(ret, 0); }();
+ }
+ [&] { ASSERT_NE(txn_kv.get(), nullptr); }();
+
+ int64_t db_id = 3213235;
+ int64_t table_id = 93213131;
+
+ auto meta_service = get_meta_service(txn_kv, true);
+ brpc::Controller cntl;
+ BeginTxnRequest req;
+ req.set_cloud_unique_id("test_cloud_unique_id");
+ TxnInfoPB txn_info_pb;
+ txn_info_pb.set_db_id(db_id);
+ txn_info_pb.set_label("test_label");
+ txn_info_pb.add_table_ids(table_id);
+ txn_info_pb.set_timeout_ms(36000);
+ req.mutable_txn_info()->CopyFrom(txn_info_pb);
+ BeginTxnResponse res;
+ meta_service->begin_txn(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res,
+ nullptr);
+ ASSERT_EQ(res.status().code(), MetaServiceCode::OK);
+ int64_t txn_id = res.txn_id();
+
+ // mock rowset and tablet
+ int64_t tablet_id_base = 1103;
+ for (int i = 0; i < 5; ++i) {
+ create_tablet(meta_service.get(), table_id, 1235, 1236, tablet_id_base + i);
+ auto tmp_rowset = create_rowset(txn_id, tablet_id_base + i);
+ CreateRowsetResponse res;
+ commit_rowset(meta_service.get(), tmp_rowset, res);
+ ASSERT_EQ(res.status().code(), MetaServiceCode::OK);
+ }
+
+ {
+ brpc::Controller cntl;
+ CommitTxnRequest req;
+ req.set_cloud_unique_id("test_cloud_unique_id");
+ req.set_db_id(db_id);
+ req.set_txn_id(txn_id);
+ req.set_is_2pc(false);
+ req.set_enable_txn_lazy_commit(true);
+ CommitTxnResponse res;
+ meta_service->commit_txn(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req,
+ &res, nullptr);
+ ASSERT_EQ(res.status().code(), MetaServiceCode::OK);
+ }
+
+ {
+ std::unique_ptr<Transaction> txn;
+ ASSERT_EQ(txn_kv->create_txn(&txn), TxnErrorCode::TXN_OK);
+ std::string mock_instance = "test_instance";
+ int64_t tablet_id_base = 1103;
+ for (int i = 0; i < 5; ++i) {
+ int64_t tablet_id = tablet_id_base + i;
+ std::string key = meta_tablet_idx_key({mock_instance, tablet_id});
+ std::string val;
+ ASSERT_EQ(txn->get(key, &val), TxnErrorCode::TXN_OK);
+ TabletIndexPB tablet_idx_pb;
+ tablet_idx_pb.ParseFromString(val);
+ ASSERT_EQ(tablet_idx_pb.db_id(), db_id);
+ }
+ }
+}
+} // namespace doris::cloud
\ No newline at end of file
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index db80a265a3e..eb70a71a9c3 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -3010,6 +3010,10 @@ public class Config extends ConfigBase {
@ConfField(mutable = true, description = {"存算分离模式下commit阶段等锁超时时间，默认5s"})
public static int try_commit_lock_timeout_seconds = 5;
+
+ @ConfField(mutable = true, description = {"存算分离模式下是否开启大事务提交，默认false"})
+ public static boolean enable_cloud_txn_lazy_commit = false;
+
// ATTN: DONOT add any config not related to cloud mode here
// ATTN: DONOT add any config not related to cloud mode here
// ATTN: DONOT add any config not related to cloud mode here
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java
index 9c21bb3a3ad..52846353a8a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java
@@ -215,6 +215,7 @@ public class CloudRollupJobV2 extends RollupJobV2 {
tbl.variantEnableFlattenNested());
requestBuilder.addTabletMetas(builder);
} // end for rollupTablets
+ requestBuilder.setDbId(dbId);
((CloudInternalCatalog) Env.getCurrentInternalCatalog())
.sendCreateTabletsRpc(requestBuilder);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java
index c33e10bb4a5..7061478440f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java
@@ -236,6 +236,7 @@ public class CloudSchemaChangeJobV2 extends SchemaChangeJobV2 {
tbl.variantEnableFlattenNested());
requestBuilder.addTabletMetas(builder);
} // end for rollupTablets
+ requestBuilder.setDbId(dbId);
((CloudInternalCatalog) Env.getCurrentInternalCatalog())
.sendCreateTabletsRpc(requestBuilder);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
index fb520d4295b..ef461c74b21 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
@@ -181,6 +181,7 @@ public class CloudInternalCatalog extends InternalCatalog {
if (!storageVaultIdSet && ((CloudEnv) Env.getCurrentEnv()).getEnableStorageVault()) {
requestBuilder.setStorageVaultName(storageVaultName);
}
+ requestBuilder.setDbId(dbId);
LOG.info("create tablets, dbId: {}, tableId: {}, tableName: {}, partitionId: {}, partitionName: {}, "
+ "indexId: {}, vault name {}",
@@ -682,7 +683,7 @@ public class CloudInternalCatalog extends InternalCatalog {
sendCreateTabletsRpc(Cloud.CreateTabletsRequest.Builder requestBuilder) throws DdlException {
requestBuilder.setCloudUniqueId(Config.cloud_unique_id);
Cloud.CreateTabletsRequest createTabletsReq = requestBuilder.build();
-
+ Preconditions.checkState(createTabletsReq.hasDbId(), "createTabletsReq must set dbId");
if (LOG.isDebugEnabled()) {
LOG.debug("send create tablets rpc, createTabletsReq: {}", createTabletsReq);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java
index e8be1ac133e..f224d2929a6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java
@@ -472,7 +472,8 @@ public class CloudGlobalTransactionMgr implements GlobalTransactionMgrIface {
.setTxnId(transactionId)
.setIs2Pc(is2PC)
.setCloudUniqueId(Config.cloud_unique_id)
- .addAllBaseTabletIds(getBaseTabletsFromTables(tableList, tabletCommitInfos));
+ .addAllBaseTabletIds(getBaseTabletsFromTables(tableList, tabletCommitInfos))
+ .setEnableTxnLazyCommit(Config.enable_cloud_txn_lazy_commit);
// if tablet commit info is empty, no need to pass mowTableList to meta service.
if (tabletCommitInfos != null && !tabletCommitInfos.isEmpty()) {
@@ -933,7 +934,8 @@ public class CloudGlobalTransactionMgr implements GlobalTransactionMgrIface {
.setTxnId(transactionId)
.setIs2Pc(false)
.setCloudUniqueId(Config.cloud_unique_id)
- .setIsTxnLoad(true);
+ .setIsTxnLoad(true)
+ .setEnableTxnLazyCommit(Config.enable_cloud_txn_lazy_commit);
// add sub txn infos
for (SubTransactionState subTransactionState : subTransactionStates) {
builder.addSubTxnInfos(SubTxnInfo.newBuilder().setSubTxnId(subTransactionState.getSubTransactionId())
diff --git a/gensrc/proto/cloud.proto b/gensrc/proto/cloud.proto
index 9c81c33c575..87948fdeea9 100644
--- a/gensrc/proto/cloud.proto
+++ b/gensrc/proto/cloud.proto
@@ -404,6 +404,8 @@ message TxnRunningPB {
message VersionPB {
optional int64 version = 1;
optional int64 update_time_ms = 2;
+ // now only allow one element
+ repeated int64 pending_txn_ids = 3;
}
message RecycleTxnPB {
@@ -664,6 +666,7 @@ message CommitTxnRequest {
// for transaction load
optional bool is_txn_load = 9;
repeated SubTxnInfo sub_txn_infos = 10;
+ optional bool enable_txn_lazy_commit = 11;
}
message SubTxnInfo {
@@ -891,6 +894,7 @@ message CreateTabletsRequest {
optional string cloud_unique_id = 1; // For auth
repeated doris.TabletMetaCloudPB tablet_metas = 2;
optional string storage_vault_name = 3;
+ optional int64 db_id = 4;
}
message CreateTabletsResponse {
diff --git a/regression-test/data/large_txn_load_p1/test_insert_with_many_buckets.out b/regression-test/data/large_txn_load_p1/test_insert_with_many_buckets.out
new file mode 100644
index 00000000000..a507ec71566
--- /dev/null
+++ b/regression-test/data/large_txn_load_p1/test_insert_with_many_buckets.out
@@ -0,0 +1,4 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !sql --
+999
+
diff --git a/regression-test/data/large_txn_load_p1/test_insert_with_many_partitions.out b/regression-test/data/large_txn_load_p1/test_insert_with_many_partitions.out
new file mode 100644
index 00000000000..ce114af904d
--- /dev/null
+++ b/regression-test/data/large_txn_load_p1/test_insert_with_many_partitions.out
@@ -0,0 +1,4 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !sql --
+1000
+
diff --git a/regression-test/data/large_txn_load_p1/test_insert_with_many_partitions_and_buckets.out b/regression-test/data/large_txn_load_p1/test_insert_with_many_partitions_and_buckets.out
new file mode 100644
index 00000000000..475b9ff01e9
--- /dev/null
+++ b/regression-test/data/large_txn_load_p1/test_insert_with_many_partitions_and_buckets.out
@@ -0,0 +1,4 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !sql --
+400
+
diff --git a/regression-test/suites/large_txn_load_p1/test_insert_with_many_buckets.groovy b/regression-test/suites/large_txn_load_p1/test_insert_with_many_buckets.groovy
new file mode 100644
index 00000000000..ddfabe468e4
--- /dev/null
+++ b/regression-test/suites/large_txn_load_p1/test_insert_with_many_buckets.groovy
@@ -0,0 +1,57 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_insert_with_many_buckets") {
+ def tableName = 'test_insert_with_many_buckets'
+
+ // [start, end)
+ def makeInsertStmt = { start, end ->
+ def stmt = ""
+ for (int i = start; i < end; i++) {
+ if (i == (end - 1)) {
+ stmt += """(${i}, false, 1, null, "c5", "2023-09-01", null, "c8", 1.1234, "c10")\n"""
+ } else {
+ stmt += """(${i}, false, 1, null, "c5", "2023-09-01", null, "c8", 1.1234, "c10"),\n"""
+ }
+ }
+ return stmt
+ }
+
+ sql """ DROP TABLE IF EXISTS ${tableName} FORCE"""
+ sql """ CREATE TABLE IF NOT EXISTS ${tableName} (
+ `c1` bigint(20) not null,
+ `c2` boolean REPLACE_IF_NOT_NULL null,
+ `c3` tinyint(4) REPLACE_IF_NOT_NULL null,
+ `c4` decimalv3(9, 3) REPLACE_IF_NOT_NULL null,
+ `c5` char(36) REPLACE_IF_NOT_NULL null,
+ `c6` date REPLACE_IF_NOT_NULL null,
+ `c7` datetime REPLACE_IF_NOT_NULL null,
+ `c8` varchar(64) REPLACE_IF_NOT_NULL null,
+ `c9` double REPLACE_IF_NOT_NULL null,
+ `c10` string REPLACE_IF_NOT_NULL null
+ ) engine=olap
+ AGGREGATE KEY(`c1`)
+ DISTRIBUTED BY HASH(`c1`) BUCKETS 1000
+ PROPERTIES (
+ "replication_num" = "1"
+ );
+ """
+
+ sql """ insert into ${tableName} values ${makeInsertStmt(1 , 1000)};"""
+ qt_sql """ select count(*) from ${tableName};"""
+ sql """ DROP TABLE IF EXISTS ${tableName} FORCE"""
+}
diff --git a/regression-test/suites/large_txn_load_p1/test_insert_with_many_partitions.groovy b/regression-test/suites/large_txn_load_p1/test_insert_with_many_partitions.groovy
new file mode 100644
index 00000000000..a530f797a0c
--- /dev/null
+++ b/regression-test/suites/large_txn_load_p1/test_insert_with_many_partitions.groovy
@@ -0,0 +1,73 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_insert_with_many_parititions") {
+ def tableName = 'test_insert_with_many_parititions'
+
+ def makePartitionsStmt = { partitionNum ->
+ def stmt = ""
+ for (int i = 0; i < partitionNum; i++) {
+ if (i == (partitionNum - 1)) {
+ stmt += """partition `p${i}` values [("${i}"), ("${i + 1}"))"""
+ } else {
+ stmt += """partition `p${i}` values [("${i}"), ("${i + 1}")),\n\t\t\t"""
+ }
+ }
+ return stmt
+ }
+
+ // [start, end)
+ def makeInsertStmt = { start, end ->
+ def stmt = ""
+ for (int i = start; i < end; i++) {
+ if (i == (end - 1)) {
+ stmt += """(${i}, false, 1, null, "c5", "2023-09-01", null, "c8", 1.1234, "c10")\n"""
+ } else {
+ stmt += """(${i}, false, 1, null, "c5", "2023-09-01", null, "c8", 1.1234, "c10"),\n"""
+ }
+ }
+ return stmt
+ }
+
+ sql """ DROP TABLE IF EXISTS ${tableName} FORCE"""
+ sql """ CREATE TABLE IF NOT EXISTS ${tableName} (
+ `c1` bigint(20) not null,
+ `c2` boolean REPLACE_IF_NOT_NULL null,
+ `c3` tinyint(4) REPLACE_IF_NOT_NULL null,
+ `c4` decimalv3(9, 3) REPLACE_IF_NOT_NULL null,
+ `c5` char(36) REPLACE_IF_NOT_NULL null,
+ `c6` date REPLACE_IF_NOT_NULL null,
+ `c7` datetime REPLACE_IF_NOT_NULL null,
+ `c8` varchar(64) REPLACE_IF_NOT_NULL null,
+ `c9` double REPLACE_IF_NOT_NULL null,
+ `c10` string REPLACE_IF_NOT_NULL null
+ ) engine=olap
+ AGGREGATE KEY(`c1`)
+ PARTITION BY RANGE(`c1`)
+ (
+ ${makePartitionsStmt(1000)}
+ )
+ DISTRIBUTED BY HASH(`c1`) BUCKETS 1
+ PROPERTIES (
+ "replication_num" = "1"
+ );
+ """
+
+ sql """ insert into ${tableName} values ${makeInsertStmt(0, 1000)};"""
+ qt_sql """ select count(*) from ${tableName};"""
+ sql """ DROP TABLE IF EXISTS ${tableName} FORCE"""
+}
diff --git a/regression-test/suites/large_txn_load_p1/test_insert_with_many_partitions_and_buckets.groovy b/regression-test/suites/large_txn_load_p1/test_insert_with_many_partitions_and_buckets.groovy
new file mode 100644
index 00000000000..b303ce13027
--- /dev/null
+++ b/regression-test/suites/large_txn_load_p1/test_insert_with_many_partitions_and_buckets.groovy
@@ -0,0 +1,79 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_insert_with_many_parititions_and_buckets") {
+ def tableName = 'test_insert_with_many_parititions_and_buckets'
+
+ def makePartitionsStmt = { partitionNum ->
+ def stmt = ""
+ for (int i = 0; i < partitionNum; i++) {
+ if (i == (partitionNum - 1)) {
+ stmt += """partition `p${i}` values [("${i}"), ("${i + 1}"))"""
+ } else {
+ stmt += """partition `p${i}` values [("${i}"), ("${i + 1}")),\n\t\t\t"""
+ }
+ }
+ return stmt
+ }
+
+ // [start, end)
+ def makeInsertStmt = { start, end ->
+ def stmt = ""
+ for (int i = start; i < end; i++) {
+ if (i == (end - 1)) {
+ stmt += """(${i}, 1, false,null, "c5", "2023-09-01", null, "c8", 1.1234, "c10"),\n"""
+ stmt += """(${i}, 2, false, null, "c5", "2023-09-01", null, "c8", 1.1234, "c10"),\n"""
+ stmt += """(${i}, 3, false, null, "c5", "2023-09-01", null, "c8", 1.1234, "c10"),\n"""
+ stmt += """(${i}, 4, false, null, "c5", "2023-09-01", null, "c8", 1.1234, "c10")\n"""
+ } else {
+ stmt += """(${i}, 1, false, null, "c5", "2023-09-01", null, "c8", 1.1234, "c10"),\n"""
+ stmt += """(${i}, 2, false, null, "c5", "2023-09-01", null, "c8", 1.1234, "c10"),\n"""
+ stmt += """(${i}, 3, false, null, "c5", "2023-09-01", null, "c8", 1.1234, "c10"),\n"""
+ stmt += """(${i}, 4, false, null, "c5", "2023-09-01", null, "c8", 1.1234, "c10"),\n"""
+ }
+ }
+ return stmt
+ }
+
+ sql """ DROP TABLE IF EXISTS ${tableName} FORCE"""
+ sql """ CREATE TABLE IF NOT EXISTS ${tableName} (
+ `c1` bigint(20) not null,
+ `c2` int(6) not null,
+ `c3` boolean REPLACE_IF_NOT_NULL null,
+ `c4` decimalv3(9, 3) REPLACE_IF_NOT_NULL null,
+ `c5` char(36) REPLACE_IF_NOT_NULL null,
+ `c6` date REPLACE_IF_NOT_NULL null,
+ `c7` datetime REPLACE_IF_NOT_NULL null,
+ `c8` varchar(64) REPLACE_IF_NOT_NULL null,
+ `c9` double REPLACE_IF_NOT_NULL null,
+ `c10` string REPLACE_IF_NOT_NULL null
+ ) engine=olap
+ AGGREGATE KEY(`c1`, `c2`)
+ PARTITION BY RANGE(`c1`)
+ (
+ ${makePartitionsStmt(100)}
+ )
+ DISTRIBUTED BY HASH(`c2`) BUCKETS 16
+ PROPERTIES (
+ "replication_num" = "1"
+ );
+ """
+
+ sql """ insert into ${tableName} values ${makeInsertStmt(0, 100)};"""
+ qt_sql """ select count(*) from ${tableName};"""
+ sql """ DROP TABLE IF EXISTS ${tableName} FORCE;"""
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 03/11: [feature](vault) Support alter storage vault stmt in FE (#37606)
Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gavinchou pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 057c2521bd9fd31e52a445cc9635ef20a09f75f4
Author: AlexYue <yj...@gmail.com>
AuthorDate: Mon Jul 29 22:07:20 2024 +0800
[feature](vault) Support alter storage vault stmt in FE (#37606)
Support alter storage vault stmt in FE. Currently only support alter s3
vault's ak and sk.
For example
```sql
alter storage vault s3_vault properties(
"type" = "s3",
"ak" = "new_ak"
);
```
---
cloud/src/common/bvars.cpp | 1 +
cloud/src/common/bvars.h | 1 +
cloud/src/meta-service/meta_service.h | 12 +
cloud/src/meta-service/meta_service_http.cpp | 34 +-
cloud/src/meta-service/meta_service_resource.cpp | 569 ++++++++++++++++-----
cloud/test/meta_service_test.cpp | 140 ++++-
.../antlr4/org/apache/doris/nereids/DorisParser.g4 | 1 +
.../org/apache/doris/catalog/S3StorageVault.java | 11 +
.../org/apache/doris/catalog/StorageVaultMgr.java | 56 +-
.../apache/doris/cloud/rpc/MetaServiceClient.java | 9 +
.../apache/doris/cloud/rpc/MetaServiceProxy.java | 10 +
.../property/constants/S3Properties.java | 32 +-
.../doris/nereids/parser/LogicalPlanBuilder.java | 10 +
.../apache/doris/nereids/trees/plans/PlanType.java | 1 +
.../plans/commands/AlterStorageVaultCommand.java | 58 +++
.../doris/cloud/catalog/HdfsStorageVaultTest.java | 10 +-
gensrc/proto/cloud.proto | 2 +
17 files changed, 779 insertions(+), 178 deletions(-)
diff --git a/cloud/src/common/bvars.cpp b/cloud/src/common/bvars.cpp
index 6bf81d264f3..ad7d81c021f 100644
--- a/cloud/src/common/bvars.cpp
+++ b/cloud/src/common/bvars.cpp
@@ -51,6 +51,7 @@ BvarLatencyRecorderWithTag g_bvar_ms_drop_partition("ms", "drop_partition");
BvarLatencyRecorderWithTag g_bvar_ms_get_tablet_stats("ms", "get_tablet_stats");
BvarLatencyRecorderWithTag g_bvar_ms_get_obj_store_info("ms", "get_obj_store_info");
BvarLatencyRecorderWithTag g_bvar_ms_alter_obj_store_info("ms", "alter_obj_store_info");
+BvarLatencyRecorderWithTag g_bvar_ms_alter_storage_vault("ms", "alter_storage_vault");
BvarLatencyRecorderWithTag g_bvar_ms_create_instance("ms", "create_instance");
BvarLatencyRecorderWithTag g_bvar_ms_alter_instance("ms", "alter_instance");
BvarLatencyRecorderWithTag g_bvar_ms_alter_cluster("ms", "alter_cluster");
diff --git a/cloud/src/common/bvars.h b/cloud/src/common/bvars.h
index 954d21fab1b..44a2632c6af 100644
--- a/cloud/src/common/bvars.h
+++ b/cloud/src/common/bvars.h
@@ -149,6 +149,7 @@ extern BvarLatencyRecorderWithTag g_bvar_ms_drop_partition;
extern BvarLatencyRecorderWithTag g_bvar_ms_get_tablet_stats;
extern BvarLatencyRecorderWithTag g_bvar_ms_get_obj_store_info;
extern BvarLatencyRecorderWithTag g_bvar_ms_alter_obj_store_info;
+extern BvarLatencyRecorderWithTag g_bvar_ms_alter_storage_vault;
extern BvarLatencyRecorderWithTag g_bvar_ms_create_instance;
extern BvarLatencyRecorderWithTag g_bvar_ms_alter_instance;
extern BvarLatencyRecorderWithTag g_bvar_ms_alter_cluster;
diff --git a/cloud/src/meta-service/meta_service.h b/cloud/src/meta-service/meta_service.h
index 5d2ae55ffeb..5415e7ce1e6 100644
--- a/cloud/src/meta-service/meta_service.h
+++ b/cloud/src/meta-service/meta_service.h
@@ -187,6 +187,11 @@ public:
AlterObjStoreInfoResponse* response,
::google::protobuf::Closure* done) override;
+ void alter_storage_vault(google::protobuf::RpcController* controller,
+ const AlterObjStoreInfoRequest* request,
+ AlterObjStoreInfoResponse* response,
+ ::google::protobuf::Closure* done) override;
+
void update_ak_sk(google::protobuf::RpcController* controller, const UpdateAkSkRequest* request,
UpdateAkSkResponse* response, ::google::protobuf::Closure* done) override;
@@ -504,6 +509,13 @@ public:
call_impl(&cloud::MetaService::alter_obj_store_info, controller, request, response, done);
}
+ void alter_storage_vault(google::protobuf::RpcController* controller,
+ const AlterObjStoreInfoRequest* request,
+ AlterObjStoreInfoResponse* response,
+ ::google::protobuf::Closure* done) override {
+ call_impl(&cloud::MetaService::alter_storage_vault, controller, request, response, done);
+ }
+
void update_ak_sk(google::protobuf::RpcController* controller, const UpdateAkSkRequest* request,
UpdateAkSkResponse* response, ::google::protobuf::Closure* done) override {
call_impl(&cloud::MetaService::update_ak_sk, controller, request, response, done);
diff --git a/cloud/src/meta-service/meta_service_http.cpp b/cloud/src/meta-service/meta_service_http.cpp
index 5225bf83f77..d19ef224c84 100644
--- a/cloud/src/meta-service/meta_service_http.cpp
+++ b/cloud/src/meta-service/meta_service_http.cpp
@@ -213,7 +213,26 @@ static HttpResponse process_get_obj_store_info(MetaServiceImpl* service, brpc::C
static HttpResponse process_alter_obj_store_info(MetaServiceImpl* service, brpc::Controller* ctrl) {
static std::unordered_map<std::string_view, AlterObjStoreInfoRequest::Operation> operations {
{"add_obj_info", AlterObjStoreInfoRequest::ADD_OBJ_INFO},
- {"legacy_update_ak_sk", AlterObjStoreInfoRequest::LEGACY_UPDATE_AK_SK},
+ {"legacy_update_ak_sk", AlterObjStoreInfoRequest::LEGACY_UPDATE_AK_SK}};
+
+ auto& path = ctrl->http_request().unresolved_path();
+ auto it = operations.find(remove_version_prefix(path));
+ if (it == operations.end()) {
+ std::string msg = "not supportted alter obj store info operation: " + path;
+ return http_json_reply(MetaServiceCode::INVALID_ARGUMENT, msg);
+ }
+
+ AlterObjStoreInfoRequest req;
+ PARSE_MESSAGE_OR_RETURN(ctrl, req);
+ req.set_op(it->second);
+
+ AlterObjStoreInfoResponse resp;
+ service->alter_obj_store_info(ctrl, &req, &resp, nullptr);
+ return http_json_reply(resp.status());
+}
+
+static HttpResponse process_alter_storage_vault(MetaServiceImpl* service, brpc::Controller* ctrl) {
+ static std::unordered_map<std::string_view, AlterObjStoreInfoRequest::Operation> operations {
{"drop_s3_vault", AlterObjStoreInfoRequest::DROP_S3_VAULT},
{"add_s3_vault", AlterObjStoreInfoRequest::ADD_S3_VAULT},
{"drop_hdfs_vault", AlterObjStoreInfoRequest::DROP_HDFS_INFO},
@@ -222,7 +241,7 @@ static HttpResponse process_alter_obj_store_info(MetaServiceImpl* service, brpc:
auto& path = ctrl->http_request().unresolved_path();
auto it = operations.find(remove_version_prefix(path));
if (it == operations.end()) {
- std::string msg = "not supportted alter obj store info operation: " + path;
+ std::string msg = "not supportted alter storage vault operation: " + path;
return http_json_reply(MetaServiceCode::INVALID_ARGUMENT, msg);
}
@@ -231,7 +250,7 @@ static HttpResponse process_alter_obj_store_info(MetaServiceImpl* service, brpc:
req.set_op(it->second);
AlterObjStoreInfoResponse resp;
- service->alter_obj_store_info(ctrl, &req, &resp, nullptr);
+ service->alter_storage_vault(ctrl, &req, &resp, nullptr);
return http_json_reply(resp.status());
}
@@ -516,10 +535,11 @@ void MetaServiceImpl::http(::google::protobuf::RpcController* controller,
{"v1/legacy_update_ak_sk", process_alter_obj_store_info},
{"v1/update_ak_sk", process_update_ak_sk},
{"show_storage_vaults", process_get_obj_store_info},
- {"add_hdfs_vault", process_alter_obj_store_info},
- {"add_s3_vault", process_alter_obj_store_info},
- {"drop_s3_vault", process_alter_obj_store_info},
- {"drop_hdfs_vault", process_alter_obj_store_info},
+ {"add_hdfs_vault", process_alter_storage_vault},
+ {"add_s3_vault", process_alter_storage_vault},
+ {"alter_s3_vault", process_alter_storage_vault},
+ {"drop_s3_vault", process_alter_storage_vault},
+ {"drop_hdfs_vault", process_alter_storage_vault},
// for tools
{"decode_key", process_decode_key},
{"encode_key", process_encode_key},
diff --git a/cloud/src/meta-service/meta_service_resource.cpp b/cloud/src/meta-service/meta_service_resource.cpp
index 6175df6c40d..a73301205dc 100644
--- a/cloud/src/meta-service/meta_service_resource.cpp
+++ b/cloud/src/meta-service/meta_service_resource.cpp
@@ -71,6 +71,7 @@ static int encrypt_ak_sk_helper(const std::string plain_ak, const std::string pl
MetaServiceCode& code, std::string& msg) {
std::string key;
int64_t key_id;
+ LOG_INFO("enter encrypt_ak_sk_helper, plain_ak {}", plain_ak);
int ret = get_newest_encryption_key_for_ak_sk(&key_id, &key);
TEST_SYNC_POINT_CALLBACK("encrypt_ak_sk:get_encryption_key", &ret, &key, &key_id);
if (ret != 0) {
@@ -509,56 +510,216 @@ static void set_default_vault_log_helper(const InstanceInfoPB& instance,
LOG(INFO) << vault_msg;
}
-void MetaServiceImpl::alter_obj_store_info(google::protobuf::RpcController* controller,
- const AlterObjStoreInfoRequest* request,
- AlterObjStoreInfoResponse* response,
- ::google::protobuf::Closure* done) {
+static int alter_s3_storage_vault(InstanceInfoPB& instance, std::unique_ptr<Transaction> txn,
+ const StorageVaultPB& vault, MetaServiceCode& code,
+ std::string& msg) {
+ if (!vault.has_obj_info()) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ std::stringstream ss;
+ ss << "Only s3 vault can be altered";
+ msg = ss.str();
+ return -1;
+ }
+ const auto& obj_info = vault.obj_info();
+ if (obj_info.has_bucket() || obj_info.has_endpoint() || obj_info.has_prefix() ||
+ obj_info.has_provider()) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ std::stringstream ss;
+ ss << "Only ak, sk can be altered";
+ msg = ss.str();
+ return -1;
+ }
+ const auto& name = vault.name();
+ auto name_itr = std::find_if(instance.storage_vault_names().begin(),
+ instance.storage_vault_names().end(),
+ [&](const auto& vault_name) { return name == vault_name; });
+ if (name_itr == instance.storage_vault_names().end()) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ std::stringstream ss;
+ ss << "invalid storage vault name, not found, name =" << name;
+ msg = ss.str();
+ return -1;
+ }
+ auto pos = name_itr - instance.storage_vault_names().begin();
+ auto id_itr = instance.resource_ids().begin() + pos;
+ auto vault_key = storage_vault_key({instance.instance_id(), *id_itr});
+ std::string val;
+
+ auto err = txn->get(vault_key, &val);
+ LOG(INFO) << "get instance_key=" << hex(vault_key);
+
+ if (err != TxnErrorCode::TXN_OK) {
+ code = cast_as<ErrCategory::READ>(err);
+ std::stringstream ss;
+ ss << "failed to get storage vault, vault_id=" << *name_itr << ", vault_name="
+ << "" << name << " err=" << err;
+ msg = ss.str();
+ return -1;
+ }
+ StorageVaultPB alter;
+ alter.ParseFromString(val);
+ AkSkPair pre {alter.obj_info().ak(), alter.obj_info().sk()};
+ const auto& plain_ak = obj_info.has_ak() ? obj_info.ak() : alter.obj_info().ak();
+ const auto& plain_sk = obj_info.has_ak() ? obj_info.sk() : alter.obj_info().sk();
+ auto obfuscating_sk = [](const auto& sk) -> std::string {
+ if (sk.empty()) {
+ return "";
+ }
+ std::string result(sk.length(), '*');
+ result.replace(0, 2, sk, 0, 2);
+ result.replace(result.length() - 2, 2, sk, sk.length() - 2, 2);
+ return result;
+ };
+ AkSkPair plain_ak_sk_pair {plain_ak, plain_sk};
+ AkSkPair cipher_ak_sk_pair;
+ EncryptionInfoPB encryption_info;
+ auto ret = encrypt_ak_sk_helper(plain_ak, plain_sk, &encryption_info, &cipher_ak_sk_pair, code,
+ msg);
+ if (ret != 0) {
+ msg = "failed to encrypt";
+ code = MetaServiceCode::ERR_ENCRYPT;
+ LOG(WARNING) << msg;
+ return -1;
+ }
+ alter.mutable_obj_info()->set_ak(cipher_ak_sk_pair.first);
+ alter.mutable_obj_info()->set_sk(cipher_ak_sk_pair.second);
+ alter.mutable_obj_info()->mutable_encryption_info()->CopyFrom(encryption_info);
+
+ val = alter.SerializeAsString();
+ if (val.empty()) {
+ msg = "failed to serialize";
+ code = MetaServiceCode::PROTOBUF_SERIALIZE_ERR;
+ return -1;
+ }
+
+ txn->put(vault_key, val);
+ LOG(INFO) << "put vault_id=" << *id_itr << ", instance_key=" << hex(vault_key)
+ << ", previous ak=" << pre.first << ", previous sk=" << obfuscating_sk(pre.second)
+ << ", new ak=" << cipher_ak_sk_pair.first
+ << ", new sk=" << obfuscating_sk(cipher_ak_sk_pair.second);
+ err = txn->commit();
+ if (err != TxnErrorCode::TXN_OK) {
+ code = cast_as<ErrCategory::COMMIT>(err);
+ msg = fmt::format("failed to commit kv txn, err={}", err);
+ LOG(WARNING) << msg;
+ }
+
+ return 0;
+}
+
+struct ObjectStorageDesc {
+ std::string& ak;
+ std::string& sk;
+ std::string& bucket;
+ std::string& prefix;
+ std::string& endpoint;
+ std::string& external_endpoint;
+ std::string& region;
+};
+
+static int extract_object_storage_info(const AlterObjStoreInfoRequest* request,
+ MetaServiceCode& code, std::string& msg,
+ ObjectStorageDesc& obj_desc,
+ EncryptionInfoPB& encryption_info,
+ AkSkPair& cipher_ak_sk_pair) {
+ if (!request->has_obj() && (!request->has_vault() || !request->vault().has_obj_info())) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ msg = "s3 obj info err " + proto_to_json(*request);
+ return -1;
+ }
+ auto& [ak, sk, bucket, prefix, endpoint, external_endpoint, region] = obj_desc;
+ const auto& obj = request->has_obj() ? request->obj() : request->vault().obj_info();
+ // Prepare data
+ if (!obj.has_ak() || !obj.has_sk()) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ msg = "s3 obj info err " + proto_to_json(*request);
+ return -1;
+ }
+
+ std::string plain_ak = obj.has_ak() ? obj.ak() : "";
+ std::string plain_sk = obj.has_sk() ? obj.sk() : "";
+
+ auto ret = encrypt_ak_sk_helper(plain_ak, plain_sk, &encryption_info, &cipher_ak_sk_pair, code,
+ msg);
+ if (ret != 0) {
+ return -1;
+ }
+ TEST_SYNC_POINT_CALLBACK("extract_object_storage_info:get_aksk_pair", &cipher_ak_sk_pair);
+
+ ak = cipher_ak_sk_pair.first;
+ sk = cipher_ak_sk_pair.second;
+ bucket = obj.has_bucket() ? obj.bucket() : "";
+ prefix = obj.has_prefix() ? obj.prefix() : "";
+ endpoint = obj.has_endpoint() ? obj.endpoint() : "";
+ external_endpoint = obj.has_external_endpoint() ? obj.external_endpoint() : "";
+ region = obj.has_region() ? obj.region() : "";
+ // obj size > 1k, refuse
+ if (obj.ByteSizeLong() > 1024) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ msg = "s3 obj info greater than 1k " + proto_to_json(*request);
+ return -1;
+ };
+ return 0;
+}
+
+static ObjectStoreInfoPB object_info_pb_factory(ObjectStorageDesc& obj_info,
+ const ObjectStoreInfoPB& obj,
+ InstanceInfoPB& instance,
+ EncryptionInfoPB& encryption_info,
+ AkSkPair& cipher_ak_sk_pair) {
+ ObjectStoreInfoPB last_item;
+ auto& [ak, sk, bucket, prefix, endpoint, external_endpoint, region] = obj_info;
+ auto now_time = std::chrono::system_clock::now();
+ uint64_t time =
+ std::chrono::duration_cast<std::chrono::seconds>(now_time.time_since_epoch()).count();
+ last_item.set_ctime(time);
+ last_item.set_mtime(time);
+ last_item.set_id(next_available_vault_id(instance));
+ if (obj.has_user_id()) {
+ last_item.set_user_id(obj.user_id());
+ }
+ last_item.set_ak(std::move(cipher_ak_sk_pair.first));
+ last_item.set_sk(std::move(cipher_ak_sk_pair.second));
+ last_item.mutable_encryption_info()->CopyFrom(encryption_info);
+ last_item.set_bucket(bucket);
+ // format prefix, such as `/aa/bb/`, `aa/bb//`, `//aa/bb`, ` /aa/bb` -> `aa/bb`
+ trim(prefix);
+ last_item.set_prefix(prefix);
+ last_item.set_endpoint(endpoint);
+ last_item.set_external_endpoint(external_endpoint);
+ last_item.set_region(region);
+ last_item.set_provider(obj.provider());
+ last_item.set_sse_enabled(instance.sse_enabled());
+ return last_item;
+}
+
+void MetaServiceImpl::alter_storage_vault(google::protobuf::RpcController* controller,
+ const AlterObjStoreInfoRequest* request,
+ AlterObjStoreInfoResponse* response,
+ ::google::protobuf::Closure* done) {
std::string ak, sk, bucket, prefix, endpoint, external_endpoint, region;
EncryptionInfoPB encryption_info;
AkSkPair cipher_ak_sk_pair;
- RPC_PREPROCESS(alter_obj_store_info);
+ RPC_PREPROCESS(alter_storage_vault);
switch (request->op()) {
- case AlterObjStoreInfoRequest::ADD_OBJ_INFO:
case AlterObjStoreInfoRequest::ADD_S3_VAULT:
- case AlterObjStoreInfoRequest::DROP_S3_VAULT:
- case AlterObjStoreInfoRequest::LEGACY_UPDATE_AK_SK:
- case AlterObjStoreInfoRequest::UPDATE_AK_SK: {
- if (!request->has_obj() && (!request->has_vault() || !request->vault().has_obj_info())) {
- code = MetaServiceCode::INVALID_ARGUMENT;
- msg = "s3 obj info err " + proto_to_json(*request);
+ case AlterObjStoreInfoRequest::DROP_S3_VAULT: {
+ auto tmp_desc =
+ ObjectStorageDesc {ak, sk, bucket, prefix, endpoint, external_endpoint, region};
+ if (0 != extract_object_storage_info(request, code, msg, tmp_desc, encryption_info,
+ cipher_ak_sk_pair)) {
return;
}
- auto& obj = request->has_obj() ? request->obj() : request->vault().obj_info();
- // Prepare data
- if (!obj.has_ak() || !obj.has_sk()) {
+ } break;
+ case AlterObjStoreInfoRequest::ADD_BUILT_IN_VAULT: {
+ // It should at least has one hdfs info or obj info inside storage vault
+ if ((!request->has_vault())) {
code = MetaServiceCode::INVALID_ARGUMENT;
- msg = "s3 obj info err " + proto_to_json(*request);
- return;
- }
-
- std::string plain_ak = obj.has_ak() ? obj.ak() : "";
- std::string plain_sk = obj.has_sk() ? obj.sk() : "";
-
- auto ret = encrypt_ak_sk_helper(plain_ak, plain_sk, &encryption_info, &cipher_ak_sk_pair,
- code, msg);
- if (ret != 0) {
+ msg = "storage vault is set " + proto_to_json(*request);
return;
}
- ak = cipher_ak_sk_pair.first;
- sk = cipher_ak_sk_pair.second;
- bucket = obj.has_bucket() ? obj.bucket() : "";
- prefix = obj.has_prefix() ? obj.prefix() : "";
- endpoint = obj.has_endpoint() ? obj.endpoint() : "";
- external_endpoint = obj.has_external_endpoint() ? obj.external_endpoint() : "";
- region = obj.has_region() ? obj.region() : "";
-
- // obj size > 1k, refuse
- if (obj.ByteSizeLong() > 1024) {
- code = MetaServiceCode::INVALID_ARGUMENT;
- msg = "s3 obj info greater than 1k " + proto_to_json(*request);
- return;
- };
- } break;
+ break;
+ }
case AlterObjStoreInfoRequest::ADD_HDFS_INFO:
case AlterObjStoreInfoRequest::DROP_HDFS_INFO: {
if (!request->has_vault() || !request->vault().has_name()) {
@@ -575,22 +736,20 @@ void MetaServiceImpl::alter_obj_store_info(google::protobuf::RpcController* cont
}
break;
}
- case AlterObjStoreInfoRequest::ADD_BUILT_IN_VAULT: {
- // It should at least has one hdfs info or obj info inside storage vault
- if ((!request->has_vault())) {
- code = MetaServiceCode::INVALID_ARGUMENT;
- msg = "hdfs info is not found " + proto_to_json(*request);
- return;
- }
+ case AlterObjStoreInfoRequest::ALTER_S3_VAULT:
+ break;
+ case AlterObjStoreInfoRequest::UNSET_DEFAULT_VAULT:
break;
- }
case AlterObjStoreInfoRequest::UNKNOWN: {
code = MetaServiceCode::INVALID_ARGUMENT;
msg = "Unknown alter info " + proto_to_json(*request);
return;
} break;
- case AlterObjStoreInfoRequest::UNSET_DEFAULT_VAULT:
- break;
+ default:
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ msg = "Unknown alter obj store info, request info " + proto_to_json(*request);
+ LOG_WARNING("Unknown alter obj store info, request info {}", request->DebugString());
+ return;
}
// TODO(dx): check s3 info right
@@ -646,45 +805,13 @@ void MetaServiceImpl::alter_obj_store_info(google::protobuf::RpcController* cont
return;
}
- auto now_time = std::chrono::system_clock::now();
- uint64_t time =
- std::chrono::duration_cast<std::chrono::seconds>(now_time.time_since_epoch()).count();
-
switch (request->op()) {
- case AlterObjStoreInfoRequest::LEGACY_UPDATE_AK_SK: {
- // get id
- std::string id = request->obj().has_id() ? request->obj().id() : "0";
- int idx = std::stoi(id);
- if (idx < 1 || idx > instance.obj_info().size()) {
- // err
- code = MetaServiceCode::INVALID_ARGUMENT;
- msg = "id invalid, please check it";
- return;
- }
- auto& obj_info =
- const_cast<std::decay_t<decltype(instance.obj_info())>&>(instance.obj_info());
- for (auto& it : obj_info) {
- if (std::stoi(it.id()) == idx) {
- if (it.ak() == ak && it.sk() == sk) {
- // not change, just return ok
- code = MetaServiceCode::OK;
- msg = "";
- return;
- }
- it.set_mtime(time);
- it.set_ak(ak);
- it.set_sk(sk);
- it.mutable_encryption_info()->CopyFrom(encryption_info);
- }
- }
- } break;
- case AlterObjStoreInfoRequest::ADD_OBJ_INFO:
- if (instance.enable_storage_vault()) {
+ case AlterObjStoreInfoRequest::ADD_S3_VAULT: {
+ if (!instance.enable_storage_vault()) {
code = MetaServiceCode::INVALID_ARGUMENT;
- msg = "Storage vault doesn't support add obj info";
+ msg = "Storage vault doesn't support storage vault";
return;
}
- case AlterObjStoreInfoRequest::ADD_S3_VAULT: {
auto& obj = request->has_obj() ? request->obj() : request->vault().obj_info();
if (!obj.has_provider()) {
code = MetaServiceCode::INVALID_ARGUMENT;
@@ -715,58 +842,37 @@ void MetaServiceImpl::alter_obj_store_info(google::protobuf::RpcController* cont
}
}
// calc id
- ObjectStoreInfoPB last_item;
- last_item.set_ctime(time);
- last_item.set_mtime(time);
- last_item.set_id(next_available_vault_id(instance));
- if (obj.has_user_id()) {
- last_item.set_user_id(obj.user_id());
- }
- last_item.set_ak(std::move(cipher_ak_sk_pair.first));
- last_item.set_sk(std::move(cipher_ak_sk_pair.second));
- last_item.mutable_encryption_info()->CopyFrom(encryption_info);
- last_item.set_bucket(bucket);
- // format prefix, such as `/aa/bb/`, `aa/bb//`, `//aa/bb`, ` /aa/bb` -> `aa/bb`
- trim(prefix);
- last_item.set_prefix(prefix);
- last_item.set_endpoint(endpoint);
- last_item.set_external_endpoint(external_endpoint);
- last_item.set_region(region);
- last_item.set_provider(obj.provider());
- last_item.set_sse_enabled(instance.sse_enabled());
- if (request->op() == AlterObjStoreInfoRequest::ADD_OBJ_INFO) {
- instance.add_obj_info()->CopyFrom(last_item);
- LOG_INFO("Instance {} tries to put obj info", instance.instance_id());
- } else if (request->op() == AlterObjStoreInfoRequest::ADD_S3_VAULT) {
- if (instance.storage_vault_names().end() !=
- std::find_if(instance.storage_vault_names().begin(),
- instance.storage_vault_names().end(),
- [&](const std::string& candidate_name) {
- return candidate_name == request->vault().name();
- })) {
- code = MetaServiceCode::ALREADY_EXISTED;
- msg = fmt::format("vault_name={} already created", request->vault().name());
- return;
- }
- StorageVaultPB vault;
- vault.set_id(last_item.id());
- vault.set_name(request->vault().name());
- *instance.mutable_resource_ids()->Add() = vault.id();
- *instance.mutable_storage_vault_names()->Add() = vault.name();
- vault.mutable_obj_info()->MergeFrom(last_item);
- auto vault_key = storage_vault_key({instance.instance_id(), last_item.id()});
- txn->put(vault_key, vault.SerializeAsString());
- if (request->has_set_as_default_storage_vault() &&
- request->set_as_default_storage_vault()) {
- response->set_default_storage_vault_replaced(
- instance.has_default_storage_vault_id());
- set_default_vault_log_helper(instance, vault.name(), vault.id());
- instance.set_default_storage_vault_id(vault.id());
- instance.set_default_storage_vault_name(vault.name());
- }
- LOG_INFO("try to put storage vault_id={}, vault_name={}, vault_key={}", vault.id(),
- vault.name(), hex(vault_key));
+ auto tmp_tuple =
+ ObjectStorageDesc {ak, sk, bucket, prefix, endpoint, external_endpoint, region};
+ ObjectStoreInfoPB last_item = object_info_pb_factory(tmp_tuple, obj, instance,
+ encryption_info, cipher_ak_sk_pair);
+ if (instance.storage_vault_names().end() !=
+ std::find_if(instance.storage_vault_names().begin(),
+ instance.storage_vault_names().end(),
+ [&](const std::string& candidate_name) {
+ return candidate_name == request->vault().name();
+ })) {
+ code = MetaServiceCode::ALREADY_EXISTED;
+ msg = fmt::format("vault_name={} already created", request->vault().name());
+ return;
+ }
+ StorageVaultPB vault;
+ vault.set_id(last_item.id());
+ vault.set_name(request->vault().name());
+ *instance.mutable_resource_ids()->Add() = vault.id();
+ *instance.mutable_storage_vault_names()->Add() = vault.name();
+ vault.mutable_obj_info()->MergeFrom(last_item);
+ auto vault_key = storage_vault_key({instance.instance_id(), last_item.id()});
+ txn->put(vault_key, vault.SerializeAsString());
+ if (request->has_set_as_default_storage_vault() &&
+ request->set_as_default_storage_vault()) {
+ response->set_default_storage_vault_replaced(instance.has_default_storage_vault_id());
+ set_default_vault_log_helper(instance, vault.name(), vault.id());
+ instance.set_default_storage_vault_id(vault.id());
+ instance.set_default_storage_vault_name(vault.name());
}
+ LOG_INFO("try to put storage vault_id={}, vault_name={}, vault_key={}", vault.id(),
+ vault.name(), hex(vault_key));
} break;
case AlterObjStoreInfoRequest::ADD_HDFS_INFO: {
if (auto ret = add_vault_into_instance(
@@ -835,6 +941,10 @@ void MetaServiceImpl::alter_obj_store_info(google::protobuf::RpcController* cont
instance.clear_default_storage_vault_name();
break;
}
+ case AlterObjStoreInfoRequest::ALTER_S3_VAULT: {
+ alter_s3_storage_vault(instance, std::move(txn), request->vault(), code, msg);
+ return;
+ }
case AlterObjStoreInfoRequest::DROP_S3_VAULT:
[[fallthrough]];
default: {
@@ -865,6 +975,193 @@ void MetaServiceImpl::alter_obj_store_info(google::protobuf::RpcController* cont
}
}
+void MetaServiceImpl::alter_obj_store_info(google::protobuf::RpcController* controller,
+ const AlterObjStoreInfoRequest* request,
+ AlterObjStoreInfoResponse* response,
+ ::google::protobuf::Closure* done) {
+ std::string ak, sk, bucket, prefix, endpoint, external_endpoint, region;
+ EncryptionInfoPB encryption_info;
+ AkSkPair cipher_ak_sk_pair;
+ RPC_PREPROCESS(alter_obj_store_info);
+ switch (request->op()) {
+ case AlterObjStoreInfoRequest::ADD_OBJ_INFO:
+ case AlterObjStoreInfoRequest::LEGACY_UPDATE_AK_SK:
+ case AlterObjStoreInfoRequest::UPDATE_AK_SK: {
+ auto tmp_desc =
+ ObjectStorageDesc {ak, sk, bucket, prefix, endpoint, external_endpoint, region};
+ if (0 != extract_object_storage_info(request, code, msg, tmp_desc, encryption_info,
+ cipher_ak_sk_pair)) {
+ return;
+ }
+ } break;
+ case AlterObjStoreInfoRequest::UNKNOWN: {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ msg = "Unknown alter info " + proto_to_json(*request);
+ return;
+ } break;
+ default:
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ msg = "Unknown alter obj store info, request info " + proto_to_json(*request);
+ LOG_WARNING("Unknown alter obj store info, request info {}", request->DebugString());
+ return;
+ }
+
+ // TODO(dx): check s3 info right
+
+ std::string cloud_unique_id = request->has_cloud_unique_id() ? request->cloud_unique_id() : "";
+ if (cloud_unique_id.empty()) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ msg = "cloud unique id not set";
+ return;
+ }
+
+ instance_id = get_instance_id(resource_mgr_, cloud_unique_id);
+ if (instance_id.empty()) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ msg = "empty instance_id";
+ LOG(INFO) << msg << ", cloud_unique_id=" << cloud_unique_id;
+ return;
+ }
+ RPC_RATE_LIMIT(alter_obj_store_info)
+ InstanceKeyInfo key_info {instance_id};
+ std::string key;
+ std::string val;
+ instance_key(key_info, &key);
+
+ std::unique_ptr<Transaction> txn;
+ TxnErrorCode err = txn_kv_->create_txn(&txn);
+ if (err != TxnErrorCode::TXN_OK) {
+ code = cast_as<ErrCategory::CREATE>(err);
+ msg = "failed to create txn";
+ LOG(WARNING) << msg << " err=" << err;
+ return;
+ }
+ err = txn->get(key, &val);
+ LOG(INFO) << "get instance_key=" << hex(key);
+
+ if (err != TxnErrorCode::TXN_OK) {
+ code = cast_as<ErrCategory::READ>(err);
+ ss << "failed to get instance, instance_id=" << instance_id << " err=" << err;
+ msg = ss.str();
+ return;
+ }
+
+ InstanceInfoPB instance;
+ if (!instance.ParseFromString(val)) {
+ code = MetaServiceCode::PROTOBUF_PARSE_ERR;
+ msg = "failed to parse InstanceInfoPB";
+ return;
+ }
+
+ if (instance.status() == InstanceInfoPB::DELETED) {
+ code = MetaServiceCode::CLUSTER_NOT_FOUND;
+ msg = "instance status has been set delete, plz check it";
+ return;
+ }
+
+ switch (request->op()) {
+ case AlterObjStoreInfoRequest::LEGACY_UPDATE_AK_SK: {
+ // get id
+ std::string id = request->obj().has_id() ? request->obj().id() : "0";
+ int idx = std::stoi(id);
+ if (idx < 1 || idx > instance.obj_info().size()) {
+ // err
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ msg = "id invalid, please check it";
+ return;
+ }
+ auto& obj_info =
+ const_cast<std::decay_t<decltype(instance.obj_info())>&>(instance.obj_info());
+ for (auto& it : obj_info) {
+ if (std::stoi(it.id()) == idx) {
+ if (it.ak() == ak && it.sk() == sk) {
+ // not change, just return ok
+ code = MetaServiceCode::OK;
+ msg = "";
+ return;
+ }
+ auto now_time = std::chrono::system_clock::now();
+ uint64_t time = std::chrono::duration_cast<std::chrono::seconds>(
+ now_time.time_since_epoch())
+ .count();
+ it.set_mtime(time);
+ it.set_ak(ak);
+ it.set_sk(sk);
+ it.mutable_encryption_info()->CopyFrom(encryption_info);
+ }
+ }
+ } break;
+ case AlterObjStoreInfoRequest::ADD_OBJ_INFO: {
+ if (instance.enable_storage_vault()) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ msg = "Storage vault doesn't support add obj info";
+ return;
+ }
+ auto& obj = request->has_obj() ? request->obj() : request->vault().obj_info();
+ if (!obj.has_provider()) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ msg = "s3 conf lease provider info";
+ return;
+ }
+ if (instance.obj_info().size() >= 10) {
+ code = MetaServiceCode::UNDEFINED_ERR;
+ msg = "this instance history has greater than 10 objs, please new another instance";
+ return;
+ }
+ // ATTN: prefix may be empty
+ if (ak.empty() || sk.empty() || bucket.empty() || endpoint.empty() || region.empty()) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ msg = "s3 conf info err, please check it";
+ return;
+ }
+
+ auto& objs = instance.obj_info();
+ for (auto& it : objs) {
+ if (bucket == it.bucket() && prefix == it.prefix() && endpoint == it.endpoint() &&
+ region == it.region() && ak == it.ak() && sk == it.sk() &&
+ obj.provider() == it.provider() && external_endpoint == it.external_endpoint()) {
+ // err, anything not changed
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ msg = "original obj infos has a same conf, please check it";
+ return;
+ }
+ }
+ // calc id
+ auto tmp_tuple =
+ ObjectStorageDesc {ak, sk, bucket, prefix, endpoint, external_endpoint, region};
+ ObjectStoreInfoPB last_item = object_info_pb_factory(tmp_tuple, obj, instance,
+ encryption_info, cipher_ak_sk_pair);
+ instance.add_obj_info()->CopyFrom(last_item);
+ LOG_INFO("Instance {} tries to put obj info", instance.instance_id());
+ } break;
+ default: {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ ss << "invalid request op, op=" << request->op();
+ msg = ss.str();
+ return;
+ }
+ }
+
+ LOG(INFO) << "instance " << instance_id << " has " << instance.obj_info().size()
+ << " s3 history info, and instance = " << proto_to_json(instance);
+
+ val = instance.SerializeAsString();
+ if (val.empty()) {
+ msg = "failed to serialize";
+ code = MetaServiceCode::PROTOBUF_SERIALIZE_ERR;
+ return;
+ }
+
+ txn->put(key, val);
+ LOG(INFO) << "put instance_id=" << instance_id << " instance_key=" << hex(key);
+ err = txn->commit();
+ if (err != TxnErrorCode::TXN_OK) {
+ code = cast_as<ErrCategory::COMMIT>(err);
+ msg = fmt::format("failed to commit kv txn, err={}", err);
+ LOG(WARNING) << msg;
+ }
+}
+
void MetaServiceImpl::update_ak_sk(google::protobuf::RpcController* controller,
const UpdateAkSkRequest* request, UpdateAkSkResponse* response,
::google::protobuf::Closure* done) {
diff --git a/cloud/test/meta_service_test.cpp b/cloud/test/meta_service_test.cpp
index c7d27713c9b..5a7ab21103f 100644
--- a/cloud/test/meta_service_test.cpp
+++ b/cloud/test/meta_service_test.cpp
@@ -428,6 +428,110 @@ TEST(MetaServiceTest, CreateInstanceTest) {
}
}
+TEST(MetaServiceTest, AlterS3StorageVaultTest) {
+ auto meta_service = get_meta_service();
+
+ auto sp = SyncPoint::get_instance();
+ sp->enable_processing();
+ sp->set_call_back("encrypt_ak_sk:get_encryption_key", [](auto&& args) {
+ auto* ret = try_any_cast<int*>(args[0]);
+ *ret = 0;
+ auto* key = try_any_cast<std::string*>(args[1]);
+ *key = "selectdbselectdbselectdbselectdb";
+ auto* key_id = try_any_cast<int64_t*>(args[2]);
+ *key_id = 1;
+ });
+ std::pair<std::string, std::string> pair;
+ sp->set_call_back("extract_object_storage_info:get_aksk_pair", [&](auto&& args) {
+ auto* ret = try_any_cast<std::pair<std::string, std::string>*>(args[0]);
+ pair = *ret;
+ });
+
+ std::unique_ptr<Transaction> txn;
+ ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK);
+ std::string key;
+ std::string val;
+ InstanceKeyInfo key_info {"test_instance"};
+ instance_key(key_info, &key);
+
+ ObjectStoreInfoPB obj_info;
+ obj_info.set_id("1");
+ obj_info.set_ak("ak");
+ obj_info.set_sk("sk");
+ StorageVaultPB vault;
+ vault.mutable_obj_info()->MergeFrom(obj_info);
+ vault.set_name("test_alter_s3_vault");
+ vault.set_id("2");
+ InstanceInfoPB instance;
+ instance.add_storage_vault_names(vault.name());
+ instance.add_resource_ids(vault.id());
+ instance.set_instance_id("GetObjStoreInfoTestInstance");
+ val = instance.SerializeAsString();
+ txn->put(key, val);
+ txn->put(storage_vault_key({instance.instance_id(), "2"}), vault.SerializeAsString());
+ ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK);
+ txn = nullptr;
+
+ auto get_test_instance = [&](InstanceInfoPB& i) {
+ std::string key;
+ std::string val;
+ std::unique_ptr<Transaction> txn;
+ ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK);
+ InstanceKeyInfo key_info {"test_instance"};
+ instance_key(key_info, &key);
+ ASSERT_EQ(txn->get(key, &val), TxnErrorCode::TXN_OK);
+ i.ParseFromString(val);
+ };
+
+ {
+ AlterObjStoreInfoRequest req;
+ req.set_cloud_unique_id("test_cloud_unique_id");
+ req.set_op(AlterObjStoreInfoRequest::ALTER_S3_VAULT);
+ StorageVaultPB vault;
+ vault.mutable_obj_info()->set_ak("new_ak");
+ vault.set_name("test_alter_s3_vault");
+ req.mutable_vault()->CopyFrom(vault);
+
+ brpc::Controller cntl;
+ AlterObjStoreInfoResponse res;
+ meta_service->alter_storage_vault(
+ reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, nullptr);
+ ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << res.status().msg();
+ InstanceInfoPB instance;
+ get_test_instance(instance);
+
+ std::unique_ptr<Transaction> txn;
+ ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK);
+ std::string val;
+ ASSERT_EQ(txn->get(storage_vault_key({instance.instance_id(), "2"}), &val),
+ TxnErrorCode::TXN_OK);
+ StorageVaultPB get_obj;
+ get_obj.ParseFromString(val);
+ ASSERT_EQ(get_obj.obj_info().ak(), "new_ak") << get_obj.obj_info().ak();
+ }
+
+ {
+ AlterObjStoreInfoRequest req;
+ req.set_cloud_unique_id("test_cloud_unique_id");
+ req.set_op(AlterObjStoreInfoRequest::ALTER_S3_VAULT);
+ StorageVaultPB vault;
+ ObjectStoreInfoPB obj;
+ obj_info.set_ak("new_ak");
+ vault.mutable_obj_info()->MergeFrom(obj);
+ vault.set_name("test_alter_s3_vault_non_exist");
+ req.mutable_vault()->CopyFrom(vault);
+
+ brpc::Controller cntl;
+ AlterObjStoreInfoResponse res;
+ meta_service->alter_storage_vault(
+ reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, nullptr);
+ ASSERT_NE(res.status().code(), MetaServiceCode::OK) << res.status().msg();
+ }
+
+ SyncPoint::get_instance()->disable_processing();
+ SyncPoint::get_instance()->clear_all_call_backs();
+}
+
TEST(MetaServiceTest, AlterClusterTest) {
auto meta_service = get_meta_service();
ASSERT_NE(meta_service, nullptr);
@@ -6126,13 +6230,13 @@ TEST(MetaServiceTest, AddHdfsInfoTest) {
brpc::Controller cntl;
AlterObjStoreInfoResponse res;
- meta_service->alter_obj_store_info(
+ meta_service->alter_storage_vault(
reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, nullptr);
// Invalid fs name
ASSERT_EQ(res.status().code(), MetaServiceCode::INVALID_ARGUMENT) << res.status().msg();
req.mutable_vault()->mutable_hdfs_info()->mutable_build_conf()->set_fs_name(
"hdfs://ip:port");
- meta_service->alter_obj_store_info(
+ meta_service->alter_storage_vault(
reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, nullptr);
ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << res.status().msg();
@@ -6157,7 +6261,7 @@ TEST(MetaServiceTest, AddHdfsInfoTest) {
brpc::Controller cntl;
AlterObjStoreInfoResponse res;
- meta_service->alter_obj_store_info(
+ meta_service->alter_storage_vault(
reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, nullptr);
ASSERT_EQ(res.status().code(), MetaServiceCode::ALREADY_EXISTED) << res.status().msg();
}
@@ -6177,7 +6281,7 @@ TEST(MetaServiceTest, AddHdfsInfoTest) {
brpc::Controller cntl;
AlterObjStoreInfoResponse res;
- meta_service->alter_obj_store_info(
+ meta_service->alter_storage_vault(
reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, nullptr);
ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << res.status().msg();
InstanceInfoPB instance;
@@ -6247,7 +6351,7 @@ TEST(MetaServiceTest, DropHdfsInfoTest) {
brpc::Controller cntl;
AlterObjStoreInfoResponse res;
- meta_service->alter_obj_store_info(
+ meta_service->alter_storage_vault(
reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, nullptr);
ASSERT_EQ(res.status().code(), MetaServiceCode::INVALID_ARGUMENT) << res.status().msg();
}
@@ -6266,7 +6370,7 @@ TEST(MetaServiceTest, DropHdfsInfoTest) {
brpc::Controller cntl;
AlterObjStoreInfoResponse res;
- meta_service->alter_obj_store_info(
+ meta_service->alter_storage_vault(
reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, nullptr);
ASSERT_EQ(res.status().code(), MetaServiceCode::STORAGE_VAULT_NOT_FOUND)
<< res.status().msg();
@@ -6286,7 +6390,7 @@ TEST(MetaServiceTest, DropHdfsInfoTest) {
brpc::Controller cntl;
AlterObjStoreInfoResponse res;
- meta_service->alter_obj_store_info(
+ meta_service->alter_storage_vault(
reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, nullptr);
ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << res.status().msg();
InstanceInfoPB instance;
@@ -6318,7 +6422,7 @@ TEST(MetaServiceTest, DropHdfsInfoTest) {
brpc::Controller cntl;
AlterObjStoreInfoResponse res;
- meta_service->alter_obj_store_info(
+ meta_service->alter_storage_vault(
reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, nullptr);
ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << res.status().msg();
InstanceInfoPB instance;
@@ -6345,7 +6449,7 @@ TEST(MetaServiceTest, DropHdfsInfoTest) {
brpc::Controller cntl;
AlterObjStoreInfoResponse res;
- meta_service->alter_obj_store_info(
+ meta_service->alter_storage_vault(
reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, nullptr);
ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << res.status().msg();
InstanceInfoPB instance;
@@ -6371,7 +6475,7 @@ TEST(MetaServiceTest, DropHdfsInfoTest) {
brpc::Controller cntl;
AlterObjStoreInfoResponse res;
- meta_service->alter_obj_store_info(
+ meta_service->alter_storage_vault(
reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, nullptr);
ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << res.status().msg();
InstanceInfoPB instance;
@@ -6394,7 +6498,7 @@ TEST(MetaServiceTest, DropHdfsInfoTest) {
brpc::Controller cntl;
AlterObjStoreInfoResponse res;
- meta_service->alter_obj_store_info(
+ meta_service->alter_storage_vault(
reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, nullptr);
ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << res.status().msg();
InstanceInfoPB instance;
@@ -6432,7 +6536,7 @@ TEST(MetaServiceTest, DropHdfsInfoTest) {
brpc::Controller cntl;
AlterObjStoreInfoResponse res;
- meta_service->alter_obj_store_info(
+ meta_service->alter_storage_vault(
reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, nullptr);
ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << res.status().msg();
InstanceInfoPB instance;
@@ -6615,7 +6719,7 @@ TEST(MetaServiceTest, SetDefaultVaultTest) {
brpc::Controller cntl;
AlterObjStoreInfoResponse res;
- meta_service->alter_obj_store_info(
+ meta_service->alter_storage_vault(
reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, nullptr);
ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << res.status().msg();
@@ -6624,7 +6728,7 @@ TEST(MetaServiceTest, SetDefaultVaultTest) {
set_default_req.set_op(AlterObjStoreInfoRequest::SET_DEFAULT_VAULT);
set_default_req.mutable_vault()->CopyFrom(hdfs);
AlterObjStoreInfoResponse set_default_res;
- meta_service->alter_obj_store_info(
+ meta_service->alter_storage_vault(
reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &set_default_req,
&set_default_res, nullptr);
ASSERT_EQ(set_default_res.status().code(), MetaServiceCode::OK)
@@ -6648,7 +6752,7 @@ TEST(MetaServiceTest, SetDefaultVaultTest) {
set_default_req.set_op(AlterObjStoreInfoRequest::SET_DEFAULT_VAULT);
set_default_req.mutable_vault()->CopyFrom(hdfs);
AlterObjStoreInfoResponse set_default_res;
- meta_service->alter_obj_store_info(
+ meta_service->alter_storage_vault(
reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &set_default_req,
&set_default_res, nullptr);
ASSERT_NE(set_default_res.status().code(), MetaServiceCode::OK)
@@ -6727,7 +6831,7 @@ TEST(MetaServiceTest, GetObjStoreInfoTest) {
brpc::Controller cntl;
AlterObjStoreInfoResponse res;
- meta_service->alter_obj_store_info(
+ meta_service->alter_storage_vault(
reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, nullptr);
ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << res.status().msg();
InstanceInfoPB instance;
@@ -6848,7 +6952,7 @@ TEST(MetaServiceTest, CreateTabletsVaultsTest) {
brpc::Controller cntl;
AlterObjStoreInfoResponse res;
- meta_service->alter_obj_store_info(
+ meta_service->alter_storage_vault(
reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, nullptr);
ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << res.status().msg();
@@ -6874,7 +6978,7 @@ TEST(MetaServiceTest, CreateTabletsVaultsTest) {
set_default_req.mutable_vault()->CopyFrom(hdfs);
AlterObjStoreInfoResponse set_default_res;
brpc::Controller cntl;
- meta_service->alter_obj_store_info(
+ meta_service->alter_storage_vault(
reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &set_default_req,
&set_default_res, nullptr);
ASSERT_EQ(set_default_res.status().code(), MetaServiceCode::OK)
diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index 473f7dafe7a..2dd10b990ef 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -156,6 +156,7 @@ supportedCreateStatement
supportedAlterStatement
: ALTER VIEW name=multipartIdentifier (LEFT_PAREN cols=simpleColumnDefs RIGHT_PAREN)?
AS query #alterView
+ | ALTER STORAGE VAULT name=multipartIdentifier properties=propertyClause #alterStorageVault
;
supportedDropStatement
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3StorageVault.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3StorageVault.java
index 3f06286f47d..710354b0092 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3StorageVault.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3StorageVault.java
@@ -19,9 +19,12 @@ package org.apache.doris.catalog;
import org.apache.doris.analysis.CreateResourceStmt;
import org.apache.doris.common.DdlException;
+import org.apache.doris.datasource.property.constants.S3Properties;
import com.google.gson.annotations.SerializedName;
+import java.util.Arrays;
+import java.util.HashSet;
import java.util.Map;
/**
@@ -53,6 +56,14 @@ public class S3StorageVault extends StorageVault {
// Reuse all the code from S3Resource
private Resource resource;
+ private static final String TYPE = "type";
+
+ public static final HashSet<String> ALTER_CHECK_PROPERTIES = new HashSet<>(Arrays.asList(
+ TYPE,
+ S3Properties.ACCESS_KEY,
+ S3Properties.SECRET_KEY
+ ));
+
@SerializedName(value = "properties")
private Map<String, String> properties;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVaultMgr.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVaultMgr.java
index 1b9426e4570..ee5356eeb2a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVaultMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVaultMgr.java
@@ -19,6 +19,7 @@ package org.apache.doris.catalog;
import org.apache.doris.analysis.CreateStorageVaultStmt;
import org.apache.doris.analysis.SetDefaultStorageVaultStmt;
+import org.apache.doris.catalog.StorageVault.StorageVaultType;
import org.apache.doris.cloud.proto.Cloud;
import org.apache.doris.cloud.proto.Cloud.AlterObjStoreInfoRequest.Operation;
import org.apache.doris.cloud.rpc.MetaServiceProxy;
@@ -78,6 +79,53 @@ public class StorageVaultMgr {
ALTER_BE_SYNC_THREAD_POOL.execute(() -> alterSyncVaultTask());
}
+ public void refreshVaultMap(Map<String, String> vaultMap) {
+ rwLock.writeLock().lock();
+ vaultNameToVaultId = vaultMap;
+ rwLock.writeLock().unlock();
+ }
+
+ public String getVaultIdByName(String name) {
+ String vaultId;
+ rwLock.readLock().lock();
+ vaultId = vaultNameToVaultId.getOrDefault(name, "");
+ rwLock.readLock().unlock();
+ return vaultId;
+ }
+
+ public void alterStorageVault(StorageVaultType type, Map<String, String> properties, String name) throws Exception {
+ if (type != StorageVaultType.S3) {
+ throw new DdlException("Only support alter s3 storage vault");
+ }
+ properties.keySet().stream()
+ .filter(key -> !S3StorageVault.ALTER_CHECK_PROPERTIES.contains(key))
+ .findAny()
+ .ifPresent(key -> {
+ throw new IllegalArgumentException("Alter property " + key + " is not allowed.");
+ });
+ Cloud.AlterObjStoreInfoRequest.Builder requestBuilder
+ = Cloud.AlterObjStoreInfoRequest.newBuilder();
+ requestBuilder.setOp(Cloud.AlterObjStoreInfoRequest.Operation.ALTER_S3_VAULT);
+ Cloud.ObjectStoreInfoPB.Builder objBuilder = S3Properties.getObjStoreInfoPB(properties);
+ Cloud.StorageVaultPB.Builder alterObjVaultBuilder = Cloud.StorageVaultPB.newBuilder();
+ alterObjVaultBuilder.setName(name);
+ alterObjVaultBuilder.setObjInfo(objBuilder.build());
+ requestBuilder.setVault(alterObjVaultBuilder.build());
+ try {
+ Cloud.AlterObjStoreInfoResponse response =
+ MetaServiceProxy.getInstance().alterStorageVault(requestBuilder.build());
+ if (response.getStatus().getCode() != Cloud.MetaServiceCode.OK) {
+ LOG.warn("failed to alter storage vault response: {} ", response);
+ throw new DdlException(response.getStatus().getMsg());
+ }
+ LOG.info("Succeed to alter s3 vault {}, id {}, origin default vault replaced {}",
+ name, response.getStorageVaultId(), response.getDefaultStorageVaultReplaced());
+ } catch (RpcException e) {
+ LOG.warn("failed to alter storage vault due to RpcException: {}", e);
+ throw new DdlException(e.getMessage());
+ }
+ }
+
@VisibleForTesting
public void setDefaultStorageVault(SetDefaultStorageVaultStmt stmt) throws DdlException {
Cloud.AlterObjStoreInfoRequest.Builder builder = Cloud.AlterObjStoreInfoRequest.newBuilder();
@@ -89,7 +137,7 @@ public class StorageVaultMgr {
LOG.info("try to set vault {} as default vault", stmt.getStorageVaultName());
try {
Cloud.AlterObjStoreInfoResponse resp =
- MetaServiceProxy.getInstance().alterObjStoreInfo(builder.build());
+ MetaServiceProxy.getInstance().alterStorageVault(builder.build());
if (resp.getStatus().getCode() != Cloud.MetaServiceCode.OK) {
LOG.warn("failed to set default storage vault response: {}, vault name {}",
resp, stmt.getStorageVaultName());
@@ -110,7 +158,7 @@ public class StorageVaultMgr {
builder.setOp(Operation.UNSET_DEFAULT_VAULT);
try {
Cloud.AlterObjStoreInfoResponse resp =
- MetaServiceProxy.getInstance().alterObjStoreInfo(builder.build());
+ MetaServiceProxy.getInstance().alterStorageVault(builder.build());
if (resp.getStatus().getCode() != Cloud.MetaServiceCode.OK) {
LOG.warn("failed to unset default storage vault");
throw new DdlException(resp.getStatus().getMsg());
@@ -158,7 +206,7 @@ public class StorageVaultMgr {
requestBuilder.setSetAsDefaultStorageVault(vault.setAsDefault());
try {
Cloud.AlterObjStoreInfoResponse response =
- MetaServiceProxy.getInstance().alterObjStoreInfo(requestBuilder.build());
+ MetaServiceProxy.getInstance().alterStorageVault(requestBuilder.build());
if (response.getStatus().getCode() == Cloud.MetaServiceCode.ALREADY_EXISTED
&& hdfsStorageVault.ifNotExists()) {
LOG.info("Hdfs vault {} already existed", hdfsStorageVault.getName());
@@ -210,7 +258,7 @@ public class StorageVaultMgr {
requestBuilder.setSetAsDefaultStorageVault(vault.setAsDefault());
try {
Cloud.AlterObjStoreInfoResponse response =
- MetaServiceProxy.getInstance().alterObjStoreInfo(requestBuilder.build());
+ MetaServiceProxy.getInstance().alterStorageVault(requestBuilder.build());
if (response.getStatus().getCode() == Cloud.MetaServiceCode.ALREADY_EXISTED
&& s3StorageVault.ifNotExists()) {
LOG.info("S3 vault {} already existed", s3StorageVault.getName());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceClient.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceClient.java
index ecf5b1c43ba..aaa28e5eeaa 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceClient.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceClient.java
@@ -321,6 +321,15 @@ public class MetaServiceClient {
return blockingStub.alterObjStoreInfo(request);
}
+ public Cloud.AlterObjStoreInfoResponse alterStorageVault(Cloud.AlterObjStoreInfoRequest request) {
+ if (!request.hasCloudUniqueId()) {
+ Cloud.AlterObjStoreInfoRequest.Builder builder = Cloud.AlterObjStoreInfoRequest.newBuilder();
+ builder.mergeFrom(request);
+ return blockingStub.alterStorageVault(builder.setCloudUniqueId(Config.cloud_unique_id).build());
+ }
+ return blockingStub.alterStorageVault(request);
+ }
+
public Cloud.GetDeleteBitmapUpdateLockResponse getDeleteBitmapUpdateLock(
Cloud.GetDeleteBitmapUpdateLockRequest request) {
if (!request.hasCloudUniqueId()) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceProxy.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceProxy.java
index c3853b45554..7f466374cf6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceProxy.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceProxy.java
@@ -487,6 +487,16 @@ public class MetaServiceProxy {
}
}
+ public Cloud.AlterObjStoreInfoResponse alterStorageVault(Cloud.AlterObjStoreInfoRequest request)
+ throws RpcException {
+ try {
+ final MetaServiceClient client = getProxy();
+ return client.alterStorageVault(request);
+ } catch (Exception e) {
+ throw new RpcException("", e.getMessage(), e);
+ }
+ }
+
public Cloud.GetRLTaskCommitAttachResponse
getRLTaskCommitAttach(Cloud.GetRLTaskCommitAttachRequest request)
throws RpcException {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/S3Properties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/S3Properties.java
index 521918de93b..fa7604dc56b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/S3Properties.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/S3Properties.java
@@ -308,14 +308,30 @@ public class S3Properties extends BaseProperties {
public static Cloud.ObjectStoreInfoPB.Builder getObjStoreInfoPB(Map<String, String> properties) {
Cloud.ObjectStoreInfoPB.Builder builder = Cloud.ObjectStoreInfoPB.newBuilder();
- builder.setEndpoint(properties.get(S3Properties.ENDPOINT));
- builder.setRegion(properties.get(S3Properties.REGION));
- builder.setAk(properties.get(S3Properties.ACCESS_KEY));
- builder.setSk(properties.get(S3Properties.SECRET_KEY));
- builder.setPrefix(properties.get(S3Properties.ROOT_PATH));
- builder.setBucket(properties.get(S3Properties.BUCKET));
- builder.setExternalEndpoint(properties.get(S3Properties.EXTERNAL_ENDPOINT));
- builder.setProvider(Provider.valueOf(properties.get(S3Properties.PROVIDER)));
+ if (properties.containsKey(S3Properties.ENDPOINT)) {
+ builder.setEndpoint(properties.get(S3Properties.ENDPOINT));
+ }
+ if (properties.containsKey(S3Properties.REGION)) {
+ builder.setRegion(properties.get(S3Properties.REGION));
+ }
+ if (properties.containsKey(S3Properties.ACCESS_KEY)) {
+ builder.setAk(properties.get(S3Properties.ACCESS_KEY));
+ }
+ if (properties.containsKey(S3Properties.SECRET_KEY)) {
+ builder.setSk(properties.get(S3Properties.SECRET_KEY));
+ }
+ if (properties.containsKey(S3Properties.ROOT_PATH)) {
+ builder.setPrefix(properties.get(S3Properties.ROOT_PATH));
+ }
+ if (properties.containsKey(S3Properties.BUCKET)) {
+ builder.setBucket(properties.get(S3Properties.BUCKET));
+ }
+ if (properties.containsKey(S3Properties.EXTERNAL_ENDPOINT)) {
+ builder.setExternalEndpoint(properties.get(S3Properties.EXTERNAL_ENDPOINT));
+ }
+ if (properties.containsKey(S3Properties.PROVIDER)) {
+ builder.setProvider(Provider.valueOf(properties.get(S3Properties.PROVIDER)));
+ }
return builder;
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index 2ff714d0a6a..82466223ede 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -49,6 +49,7 @@ import org.apache.doris.nereids.DorisParser.AggStateDataTypeContext;
import org.apache.doris.nereids.DorisParser.AliasQueryContext;
import org.apache.doris.nereids.DorisParser.AliasedQueryContext;
import org.apache.doris.nereids.DorisParser.AlterMTMVContext;
+import org.apache.doris.nereids.DorisParser.AlterStorageVaultContext;
import org.apache.doris.nereids.DorisParser.AlterViewContext;
import org.apache.doris.nereids.DorisParser.ArithmeticBinaryContext;
import org.apache.doris.nereids.DorisParser.ArithmeticUnaryContext;
@@ -363,6 +364,7 @@ import org.apache.doris.nereids.trees.plans.algebra.Aggregate;
import org.apache.doris.nereids.trees.plans.algebra.SetOperation.Qualifier;
import org.apache.doris.nereids.trees.plans.commands.AddConstraintCommand;
import org.apache.doris.nereids.trees.plans.commands.AlterMTMVCommand;
+import org.apache.doris.nereids.trees.plans.commands.AlterStorageVaultCommand;
import org.apache.doris.nereids.trees.plans.commands.AlterViewCommand;
import org.apache.doris.nereids.trees.plans.commands.CallCommand;
import org.apache.doris.nereids.trees.plans.commands.CancelMTMVTaskCommand;
@@ -868,6 +870,14 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
return new AlterViewCommand(info);
}
+ @Override
+ public LogicalPlan visitAlterStorageVault(AlterStorageVaultContext ctx) {
+ List<String> nameParts = this.visitMultipartIdentifier(ctx.name);
+ String vaultName = nameParts.get(0);
+ Map<String, String> properties = this.visitPropertyClause(ctx.properties);
+ return new AlterStorageVaultCommand(vaultName, properties);
+ }
+
@Override
public LogicalPlan visitShowConstraint(ShowConstraintContext ctx) {
List<String> parts = visitMultipartIdentifier(ctx.table);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
index 20d4aa9fb66..e4f0f4c102e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
@@ -157,6 +157,7 @@ public enum PlanType {
SHOW_CREATE_PROCEDURE_COMMAND,
CREATE_VIEW_COMMAND,
ALTER_VIEW_COMMAND,
+ ALTER_STORAGE_VAULT,
DROP_CATALOG_RECYCLE_BIN_COMMAND,
UNSUPPORTED_COMMAND,
CREATE_TABLE_LIKE_COMMAND,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterStorageVaultCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterStorageVaultCommand.java
new file mode 100644
index 00000000000..cbdc5765839
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterStorageVaultCommand.java
@@ -0,0 +1,58 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands;
+
+import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.StorageVault;
+import org.apache.doris.catalog.StorageVault.StorageVaultType;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.StmtExecutor;
+
+import java.util.Map;
+
+/**
+ * Alter Storage Vault command
+ */
+public class AlterStorageVaultCommand extends Command implements ForwardWithSync {
+ private static final String TYPE = "type";
+ private final Map<String, String> properties;
+ private final String name;
+
+ public AlterStorageVaultCommand(String name, final Map<String, String> properties) {
+ super(PlanType.ALTER_STORAGE_VAULT);
+ this.name = name;
+ this.properties = properties;
+ }
+
+ @Override
+ public void run(ConnectContext ctx, StmtExecutor executor) throws Exception {
+ StorageVault.StorageVaultType vaultType = StorageVaultType.fromString(properties.get(TYPE));
+ if (vaultType == StorageVault.StorageVaultType.UNKNOWN) {
+ throw new AnalysisException("Unsupported Storage Vault type: " + type);
+ }
+ Env.getCurrentEnv().getStorageVaultMgr().alterStorageVault(vaultType, properties, name);
+ }
+
+ @Override
+ public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+ return visitor.visitCommand(this, context);
+ }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/HdfsStorageVaultTest.java b/fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/HdfsStorageVaultTest.java
index 5f92b9665b5..0f34b322b05 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/HdfsStorageVaultTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/HdfsStorageVaultTest.java
@@ -65,7 +65,7 @@ public class HdfsStorageVaultTest {
new MockUp<MetaServiceProxy>(MetaServiceProxy.class) {
@Mock
public Cloud.AlterObjStoreInfoResponse
- alterObjStoreInfo(Cloud.AlterObjStoreInfoRequest request) throws RpcException {
+ alterStorageVault(Cloud.AlterObjStoreInfoRequest request) throws RpcException {
Cloud.AlterObjStoreInfoResponse.Builder resp = Cloud.AlterObjStoreInfoResponse.newBuilder();
resp.setStatus(MetaServiceResponseStatus.newBuilder().build());
resp.setStorageVaultId("1");
@@ -88,7 +88,7 @@ public class HdfsStorageVaultTest {
private Set<String> existed = new HashSet<>();
@Mock
public Cloud.AlterObjStoreInfoResponse
- alterObjStoreInfo(Cloud.AlterObjStoreInfoRequest request) throws RpcException {
+ alterStorageVault(Cloud.AlterObjStoreInfoRequest request) throws RpcException {
Cloud.AlterObjStoreInfoResponse.Builder resp = Cloud.AlterObjStoreInfoResponse.newBuilder();
MetaServiceResponseStatus.Builder status = MetaServiceResponseStatus.newBuilder();
if (existed.contains(request.getVault().getName())) {
@@ -117,7 +117,7 @@ public class HdfsStorageVaultTest {
new MockUp<MetaServiceProxy>(MetaServiceProxy.class) {
@Mock
public Cloud.AlterObjStoreInfoResponse
- alterObjStoreInfo(Cloud.AlterObjStoreInfoRequest request) throws RpcException {
+ alterStorageVault(Cloud.AlterObjStoreInfoRequest request) throws RpcException {
Cloud.AlterObjStoreInfoResponse.Builder resp = Cloud.AlterObjStoreInfoResponse.newBuilder();
if (!request.getVault().hasName() || request.getVault().getName().isEmpty()) {
resp.setStatus(MetaServiceResponseStatus.newBuilder()
@@ -144,7 +144,7 @@ public class HdfsStorageVaultTest {
private Set<String> existed = new HashSet<>();
@Mock
public Cloud.AlterObjStoreInfoResponse
- alterObjStoreInfo(Cloud.AlterObjStoreInfoRequest request) throws RpcException {
+ alterStorageVault(Cloud.AlterObjStoreInfoRequest request) throws RpcException {
Cloud.AlterObjStoreInfoResponse.Builder resp = Cloud.AlterObjStoreInfoResponse.newBuilder();
MetaServiceResponseStatus.Builder status = MetaServiceResponseStatus.newBuilder();
if (existed.contains(request.getVault().getName())) {
@@ -178,7 +178,7 @@ public class HdfsStorageVaultTest {
@Mock
public Cloud.AlterObjStoreInfoResponse
- alterObjStoreInfo(Cloud.AlterObjStoreInfoRequest request) throws RpcException {
+ alterStorageVault(Cloud.AlterObjStoreInfoRequest request) throws RpcException {
Cloud.AlterObjStoreInfoResponse.Builder resp = Cloud.AlterObjStoreInfoResponse.newBuilder();
MetaServiceResponseStatus.Builder status = MetaServiceResponseStatus.newBuilder();
if (request.getOp() == Operation.ADD_HDFS_INFO) {
diff --git a/gensrc/proto/cloud.proto b/gensrc/proto/cloud.proto
index cad16c24910..b113debae64 100644
--- a/gensrc/proto/cloud.proto
+++ b/gensrc/proto/cloud.proto
@@ -857,6 +857,7 @@ message AlterObjStoreInfoRequest {
ADD_BUILT_IN_VAULT = 102;
ADD_S3_VAULT = 103;
DROP_S3_VAULT = 104;
+ ALTER_S3_VAULT = 105;
SET_DEFAULT_VAULT = 200;
UNSET_DEFAULT_VAULT = 201;
@@ -1512,6 +1513,7 @@ service MetaService {
rpc get_obj_store_info(GetObjStoreInfoRequest) returns (GetObjStoreInfoResponse);
rpc alter_obj_store_info(AlterObjStoreInfoRequest) returns (AlterObjStoreInfoResponse);
+ rpc alter_storage_vault(AlterObjStoreInfoRequest) returns (AlterObjStoreInfoResponse);
rpc update_ak_sk(UpdateAkSkRequest) returns (UpdateAkSkResponse);
rpc create_instance(CreateInstanceRequest) returns (CreateInstanceResponse);
rpc alter_instance(AlterInstanceRequest) returns (AlterInstanceResponse);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 07/11: [feature](cloud) export fdb client busyness (#40369)
Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gavinchou pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit c0eae4da1766ad05eb3f351e12652e515fe32f68
Author: walter <w4...@gmail.com>
AuthorDate: Thu Sep 5 14:55:27 2024 +0800
[feature](cloud) export fdb client busyness (#40369)
a new bvar `fdb_client_thread_busyness_percent` (integer between 0 and
100) will be present when
```
curl ms_ip:ms_port/vars
```
---------
Co-authored-by: Gavin Chou <ga...@gmail.com>
---
cloud/src/common/bvars.cpp | 2 ++
cloud/src/common/bvars.h | 1 +
cloud/src/common/metric.cpp | 8 +++++++-
cloud/src/meta-service/txn_kv.cpp | 4 ++++
cloud/src/meta-service/txn_kv.h | 4 ++++
5 files changed, 18 insertions(+), 1 deletion(-)
diff --git a/cloud/src/common/bvars.cpp b/cloud/src/common/bvars.cpp
index ad7d81c021f..21b91416a65 100644
--- a/cloud/src/common/bvars.cpp
+++ b/cloud/src/common/bvars.cpp
@@ -177,6 +177,8 @@ bvar::Status<int64_t> g_bvar_fdb_workload_transactions_committed_hz(
"fdb_workload_transactions_committed_hz", BVAR_FDB_INVALID_VALUE);
bvar::Status<int64_t> g_bvar_fdb_workload_transactions_rejected_hz(
"fdb_workload_transactions_rejected_hz", BVAR_FDB_INVALID_VALUE);
+bvar::Status<int64_t> g_bvar_fdb_client_thread_busyness_percent(
+ "fdb_client_thread_busyness_percent", BVAR_FDB_INVALID_VALUE);
// checker's bvars
BvarStatusWithTag<long> g_bvar_checker_num_scanned("checker", "num_scanned");
diff --git a/cloud/src/common/bvars.h b/cloud/src/common/bvars.h
index 44a2632c6af..c5067b053b3 100644
--- a/cloud/src/common/bvars.h
+++ b/cloud/src/common/bvars.h
@@ -234,6 +234,7 @@ extern bvar::Status<int64_t> g_bvar_fdb_workload_written_bytes_hz;
extern bvar::Status<int64_t> g_bvar_fdb_workload_transactions_started_hz;
extern bvar::Status<int64_t> g_bvar_fdb_workload_transactions_committed_hz;
extern bvar::Status<int64_t> g_bvar_fdb_workload_transactions_rejected_hz;
+extern bvar::Status<int64_t> g_bvar_fdb_client_thread_busyness_percent;
// checker
extern BvarStatusWithTag<long> g_bvar_checker_num_scanned;
diff --git a/cloud/src/common/metric.cpp b/cloud/src/common/metric.cpp
index ee3013b1a23..9f796cd3f49 100644
--- a/cloud/src/common/metric.cpp
+++ b/cloud/src/common/metric.cpp
@@ -28,6 +28,7 @@
#include <vector>
#include "common/bvars.h"
+#include "meta-service/txn_kv.h"
#include "meta-service/txn_kv_error.h"
namespace doris::cloud {
@@ -228,8 +229,14 @@ static void export_fdb_status_details(const std::string& status_str) {
}
void FdbMetricExporter::export_fdb_metrics(TxnKv* txn_kv) {
+ int64_t busyness = 0;
std::string fdb_status = get_fdb_status(txn_kv);
export_fdb_status_details(fdb_status);
+ if (auto* kv = dynamic_cast<FdbTxnKv*>(txn_kv); kv != nullptr) {
+ busyness = static_cast<int64_t>(kv->get_client_thread_busyness() * 100);
+ g_bvar_fdb_client_thread_busyness_percent.set_value(busyness);
+ }
+ LOG(INFO) << "finish to collect fdb metric, client busyness: " << busyness << "%";
}
FdbMetricExporter::~FdbMetricExporter() {
@@ -250,7 +257,6 @@ int FdbMetricExporter::start() {
std::unique_lock l(running_mtx_);
running_cond_.wait_for(l, std::chrono::milliseconds(sleep_interval_ms_),
[this]() { return !running_.load(std::memory_order_acquire); });
- LOG(INFO) << "finish to collect fdb metric";
}
});
return 0;
diff --git a/cloud/src/meta-service/txn_kv.cpp b/cloud/src/meta-service/txn_kv.cpp
index 8cb2e906ba0..f40bee76f7a 100644
--- a/cloud/src/meta-service/txn_kv.cpp
+++ b/cloud/src/meta-service/txn_kv.cpp
@@ -140,6 +140,10 @@ TxnErrorCode FdbTxnKv::get_partition_boundaries(std::vector<std::string>* bounda
return TxnErrorCode::TXN_OK;
}
+double FdbTxnKv::get_client_thread_busyness() const {
+ return fdb_database_get_main_thread_busyness(database_->db());
+}
+
} // namespace doris::cloud
namespace doris::cloud::fdb {
diff --git a/cloud/src/meta-service/txn_kv.h b/cloud/src/meta-service/txn_kv.h
index 5da5d4e1d63..645982f7616 100644
--- a/cloud/src/meta-service/txn_kv.h
+++ b/cloud/src/meta-service/txn_kv.h
@@ -333,6 +333,10 @@ public:
// Return the partition boundaries of the database.
TxnErrorCode get_partition_boundaries(std::vector<std::string>* boundaries);
+ // Returns a value where 0 indicates that the client is idle and 1 (or larger) indicates that
+ // the client is saturated. This value is updated every second.
+ double get_client_thread_busyness() const;
+
static std::string_view fdb_partition_key_prefix() { return "\xff/keyServers/"; }
static std::string_view fdb_partition_key_end() {
// '0' is the next byte after '/' in the ASCII table
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 04/11: [feature](Vault) Support alter hdfs storage vault and alter vault's name (#38685)
Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gavinchou pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 5e4f83a91dd800461897552bf65a397011097239
Author: AlexYue <yj...@gmail.com>
AuthorDate: Tue Aug 13 22:39:10 2024 +0800
[feature](Vault) Support alter hdfs storage vault and alter vault's name (#38685)
Previously only s3 vault can be altered. This pr adds support of hdfs
storage vault. And this pr also supports to alter vault's name.
usage example
```
ALTER STORAGE VAULT alter_s3_vault
PROPERTIES (
"type"="S3",
"s3.access_key" = "new_ak", -- change ak
"s3.secrete_key" = "new_ak", -- change sk
);
ALTER STORAGE VAULT alter_hdfs_vault
PROPERTIES (
"type"="hdfs",
"VAULT_NAME" = "alter_hdfs_vault_new_name", -- change vault name
"hadoop.username" = "hdfs" -- change user name
);
```
---
cloud/src/meta-service/meta_service_resource.cpp | 186 ++++++++++++++----
cloud/test/meta_service_test.cpp | 214 ++++++++++++++++++++-
.../doris/analysis/CreateStorageVaultStmt.java | 25 +++
.../org/apache/doris/catalog/HdfsStorageVault.java | 9 +
.../org/apache/doris/catalog/S3StorageVault.java | 5 +-
.../org/apache/doris/catalog/StorageVault.java | 12 ++
.../org/apache/doris/catalog/StorageVaultMgr.java | 139 ++++++++-----
gensrc/proto/cloud.proto | 3 +
.../suites/vaults/alter/alter_hdfs.groovy | 119 ++++++++++++
.../suites/vaults/alter/alter_s3.groovy | 124 ++++++++++++
10 files changed, 754 insertions(+), 82 deletions(-)
diff --git a/cloud/src/meta-service/meta_service_resource.cpp b/cloud/src/meta-service/meta_service_resource.cpp
index a73301205dc..90a88f86006 100644
--- a/cloud/src/meta-service/meta_service_resource.cpp
+++ b/cloud/src/meta-service/meta_service_resource.cpp
@@ -24,6 +24,7 @@
#include <charconv>
#include <chrono>
#include <numeric>
+#include <regex>
#include <string>
#include <tuple>
@@ -40,6 +41,14 @@
using namespace std::chrono;
+namespace {
+constexpr char pattern_str[] = "^[a-zA-Z][0-9a-zA-Z_]*$";
+bool is_valid_storage_vault_name(const std::string& str) {
+ const std::regex pattern(pattern_str);
+ return std::regex_match(str, pattern);
+}
+} // namespace
+
namespace doris::cloud {
static void* run_bthread_work(void* arg) {
@@ -510,13 +519,114 @@ static void set_default_vault_log_helper(const InstanceInfoPB& instance,
LOG(INFO) << vault_msg;
}
+static int alter_hdfs_storage_vault(InstanceInfoPB& instance, std::unique_ptr<Transaction> txn,
+ const StorageVaultPB& vault, MetaServiceCode& code,
+ std::string& msg) {
+ if (!vault.has_hdfs_info()) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ std::stringstream ss;
+ ss << "There is no hdfs vault provided";
+ msg = ss.str();
+ return -1;
+ }
+ const auto& hdfs_info = vault.hdfs_info();
+ if (hdfs_info.has_prefix() || !hdfs_info.has_build_conf() ||
+ hdfs_info.build_conf().has_fs_name()) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ std::stringstream ss;
+ ss << "You can not alter prefix or fs name because it might lose previoud written data";
+ msg = ss.str();
+ return -1;
+ }
+ const auto& name = vault.name();
+ // Here we try to get mutable iter since we might need to alter the vault name
+ auto name_itr = std::find_if(instance.mutable_storage_vault_names()->begin(),
+ instance.mutable_storage_vault_names()->end(),
+ [&](const auto& vault_name) { return name == vault_name; });
+ if (name_itr == instance.storage_vault_names().end()) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ std::stringstream ss;
+ ss << "invalid storage vault name, not found, name =" << name;
+ msg = ss.str();
+ return -1;
+ }
+ auto pos = name_itr - instance.storage_vault_names().begin();
+ std::string vault_id = instance.resource_ids().begin()[pos];
+ auto vault_key = storage_vault_key({instance.instance_id(), vault_id});
+ std::string val;
+
+ auto err = txn->get(vault_key, &val);
+ LOG(INFO) << "get vault_key=" << hex(vault_key);
+
+ if (err != TxnErrorCode::TXN_OK) {
+ code = cast_as<ErrCategory::READ>(err);
+ std::stringstream ss;
+ ss << "failed to get storage vault, vault_id=" << vault_id << ", vault_name="
+ << "" << name << " err=" << err;
+ msg = ss.str();
+ return -1;
+ }
+ StorageVaultPB new_vault;
+ new_vault.ParseFromString(val);
+ auto origin_vault_info = new_vault.DebugString();
+ if (vault.has_alter_name()) {
+ if (!is_valid_storage_vault_name(vault.alter_name())) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ std::stringstream ss;
+ ss << "invalid storage vault name =" << vault.alter_name() << " the name must satisfy "
+ << pattern_str;
+ msg = ss.str();
+ return -1;
+ }
+ new_vault.set_name(vault.alter_name());
+ *name_itr = vault.alter_name();
+ }
+ auto* alter_hdfs_info = new_vault.mutable_hdfs_info();
+ if (hdfs_info.build_conf().has_hdfs_kerberos_keytab()) {
+ alter_hdfs_info->mutable_build_conf()->set_hdfs_kerberos_keytab(
+ hdfs_info.build_conf().hdfs_kerberos_keytab());
+ }
+ if (hdfs_info.build_conf().has_hdfs_kerberos_principal()) {
+ alter_hdfs_info->mutable_build_conf()->set_hdfs_kerberos_principal(
+ hdfs_info.build_conf().hdfs_kerberos_principal());
+ }
+ if (hdfs_info.build_conf().has_user()) {
+ alter_hdfs_info->mutable_build_conf()->set_user(hdfs_info.build_conf().user());
+ }
+ if (0 != hdfs_info.build_conf().hdfs_confs_size()) {
+ alter_hdfs_info->mutable_build_conf()->mutable_hdfs_confs()->Add(
+ hdfs_info.build_conf().hdfs_confs().begin(),
+ hdfs_info.build_conf().hdfs_confs().end());
+ }
+ auto new_vault_info = new_vault.DebugString();
+
+ val = new_vault.SerializeAsString();
+ if (val.empty()) {
+ msg = "failed to serialize";
+ code = MetaServiceCode::PROTOBUF_SERIALIZE_ERR;
+ return -1;
+ }
+
+ txn->put(vault_key, val);
+ LOG(INFO) << "put vault_id=" << vault_id << ", vault_key=" << hex(vault_key)
+ << ", origin vault=" << origin_vault_info << ", new_vault=" << new_vault_info;
+ err = txn->commit();
+ if (err != TxnErrorCode::TXN_OK) {
+ code = cast_as<ErrCategory::COMMIT>(err);
+ msg = fmt::format("failed to commit kv txn, err={}", err);
+ LOG(WARNING) << msg;
+ }
+
+ return 0;
+}
+
static int alter_s3_storage_vault(InstanceInfoPB& instance, std::unique_ptr<Transaction> txn,
const StorageVaultPB& vault, MetaServiceCode& code,
std::string& msg) {
if (!vault.has_obj_info()) {
code = MetaServiceCode::INVALID_ARGUMENT;
std::stringstream ss;
- ss << "Only s3 vault can be altered";
+ ss << "There is no s3 vault provided";
msg = ss.str();
return -1;
}
@@ -530,8 +640,9 @@ static int alter_s3_storage_vault(InstanceInfoPB& instance, std::unique_ptr<Tran
return -1;
}
const auto& name = vault.name();
- auto name_itr = std::find_if(instance.storage_vault_names().begin(),
- instance.storage_vault_names().end(),
+ // Here we try to get mutable iter since we might need to alter the vault name
+ auto name_itr = std::find_if(instance.mutable_storage_vault_names()->begin(),
+ instance.mutable_storage_vault_names()->end(),
[&](const auto& vault_name) { return name == vault_name; });
if (name_itr == instance.storage_vault_names().end()) {
code = MetaServiceCode::INVALID_ARGUMENT;
@@ -541,35 +652,39 @@ static int alter_s3_storage_vault(InstanceInfoPB& instance, std::unique_ptr<Tran
return -1;
}
auto pos = name_itr - instance.storage_vault_names().begin();
- auto id_itr = instance.resource_ids().begin() + pos;
- auto vault_key = storage_vault_key({instance.instance_id(), *id_itr});
+ std::string vault_id = instance.resource_ids().begin()[pos];
+ auto vault_key = storage_vault_key({instance.instance_id(), vault_id});
std::string val;
auto err = txn->get(vault_key, &val);
- LOG(INFO) << "get instance_key=" << hex(vault_key);
+ LOG(INFO) << "get vault_key=" << hex(vault_key);
if (err != TxnErrorCode::TXN_OK) {
code = cast_as<ErrCategory::READ>(err);
std::stringstream ss;
- ss << "failed to get storage vault, vault_id=" << *name_itr << ", vault_name="
+ ss << "failed to get storage vault, vault_id=" << vault_id << ", vault_name="
<< "" << name << " err=" << err;
msg = ss.str();
return -1;
}
- StorageVaultPB alter;
- alter.ParseFromString(val);
- AkSkPair pre {alter.obj_info().ak(), alter.obj_info().sk()};
- const auto& plain_ak = obj_info.has_ak() ? obj_info.ak() : alter.obj_info().ak();
- const auto& plain_sk = obj_info.has_ak() ? obj_info.sk() : alter.obj_info().sk();
- auto obfuscating_sk = [](const auto& sk) -> std::string {
- if (sk.empty()) {
- return "";
- }
- std::string result(sk.length(), '*');
- result.replace(0, 2, sk, 0, 2);
- result.replace(result.length() - 2, 2, sk, sk.length() - 2, 2);
- return result;
- };
+ StorageVaultPB new_vault;
+ new_vault.ParseFromString(val);
+ if (vault.has_alter_name()) {
+ if (!is_valid_storage_vault_name(vault.alter_name())) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ std::stringstream ss;
+ ss << "invalid storage vault name =" << vault.alter_name() << " the name must satisfy "
+ << pattern_str;
+ msg = ss.str();
+ return -1;
+ }
+ new_vault.set_name(vault.alter_name());
+ *name_itr = vault.alter_name();
+ }
+ auto origin_vault_info = new_vault.DebugString();
+ AkSkPair pre {new_vault.obj_info().ak(), new_vault.obj_info().sk()};
+ const auto& plain_ak = obj_info.has_ak() ? obj_info.ak() : new_vault.obj_info().ak();
+ const auto& plain_sk = obj_info.has_ak() ? obj_info.sk() : new_vault.obj_info().sk();
AkSkPair plain_ak_sk_pair {plain_ak, plain_sk};
AkSkPair cipher_ak_sk_pair;
EncryptionInfoPB encryption_info;
@@ -581,11 +696,12 @@ static int alter_s3_storage_vault(InstanceInfoPB& instance, std::unique_ptr<Tran
LOG(WARNING) << msg;
return -1;
}
- alter.mutable_obj_info()->set_ak(cipher_ak_sk_pair.first);
- alter.mutable_obj_info()->set_sk(cipher_ak_sk_pair.second);
- alter.mutable_obj_info()->mutable_encryption_info()->CopyFrom(encryption_info);
+ new_vault.mutable_obj_info()->set_ak(cipher_ak_sk_pair.first);
+ new_vault.mutable_obj_info()->set_sk(cipher_ak_sk_pair.second);
+ new_vault.mutable_obj_info()->mutable_encryption_info()->CopyFrom(encryption_info);
- val = alter.SerializeAsString();
+ auto new_vault_info = new_vault.DebugString();
+ val = new_vault.SerializeAsString();
if (val.empty()) {
msg = "failed to serialize";
code = MetaServiceCode::PROTOBUF_SERIALIZE_ERR;
@@ -593,10 +709,8 @@ static int alter_s3_storage_vault(InstanceInfoPB& instance, std::unique_ptr<Tran
}
txn->put(vault_key, val);
- LOG(INFO) << "put vault_id=" << *id_itr << ", instance_key=" << hex(vault_key)
- << ", previous ak=" << pre.first << ", previous sk=" << obfuscating_sk(pre.second)
- << ", new ak=" << cipher_ak_sk_pair.first
- << ", new sk=" << obfuscating_sk(cipher_ak_sk_pair.second);
+ LOG(INFO) << "put vault_id=" << vault_id << ", vault_key=" << hex(vault_key)
+ << ", origin vault=" << origin_vault_info << ", new vault=" << new_vault_info;
err = txn->commit();
if (err != TxnErrorCode::TXN_OK) {
code = cast_as<ErrCategory::COMMIT>(err);
@@ -738,6 +852,8 @@ void MetaServiceImpl::alter_storage_vault(google::protobuf::RpcController* contr
}
case AlterObjStoreInfoRequest::ALTER_S3_VAULT:
break;
+ case AlterObjStoreInfoRequest::ALTER_HDFS_VAULT:
+ break;
case AlterObjStoreInfoRequest::UNSET_DEFAULT_VAULT:
break;
case AlterObjStoreInfoRequest::UNKNOWN: {
@@ -925,12 +1041,12 @@ void MetaServiceImpl::alter_storage_vault(google::protobuf::RpcController* contr
return;
}
auto pos = name_itr - instance.storage_vault_names().begin();
- auto id_itr = instance.resource_ids().begin() + pos;
+ std::string vault_id = instance.resource_ids().begin()[pos];
response->set_default_storage_vault_replaced(instance.has_default_storage_vault_id());
- set_default_vault_log_helper(instance, name, *id_itr);
- instance.set_default_storage_vault_id(*id_itr);
+ set_default_vault_log_helper(instance, name, vault_id);
+ instance.set_default_storage_vault_id(vault_id);
instance.set_default_storage_vault_name(name);
- response->set_storage_vault_id(*id_itr);
+ response->set_storage_vault_id(vault_id);
break;
}
case AlterObjStoreInfoRequest::UNSET_DEFAULT_VAULT: {
@@ -945,6 +1061,10 @@ void MetaServiceImpl::alter_storage_vault(google::protobuf::RpcController* contr
alter_s3_storage_vault(instance, std::move(txn), request->vault(), code, msg);
return;
}
+ case AlterObjStoreInfoRequest::ALTER_HDFS_VAULT: {
+ alter_hdfs_storage_vault(instance, std::move(txn), request->vault(), code, msg);
+ return;
+ }
case AlterObjStoreInfoRequest::DROP_S3_VAULT:
[[fallthrough]];
default: {
diff --git a/cloud/test/meta_service_test.cpp b/cloud/test/meta_service_test.cpp
index 5a7ab21103f..3baec482710 100644
--- a/cloud/test/meta_service_test.cpp
+++ b/cloud/test/meta_service_test.cpp
@@ -459,8 +459,9 @@ TEST(MetaServiceTest, AlterS3StorageVaultTest) {
obj_info.set_ak("ak");
obj_info.set_sk("sk");
StorageVaultPB vault;
+ constexpr char vault_name[] = "test_alter_s3_vault";
vault.mutable_obj_info()->MergeFrom(obj_info);
- vault.set_name("test_alter_s3_vault");
+ vault.set_name(vault_name);
vault.set_id("2");
InstanceInfoPB instance;
instance.add_storage_vault_names(vault.name());
@@ -489,7 +490,7 @@ TEST(MetaServiceTest, AlterS3StorageVaultTest) {
req.set_op(AlterObjStoreInfoRequest::ALTER_S3_VAULT);
StorageVaultPB vault;
vault.mutable_obj_info()->set_ak("new_ak");
- vault.set_name("test_alter_s3_vault");
+ vault.set_name(vault_name);
req.mutable_vault()->CopyFrom(vault);
brpc::Controller cntl;
@@ -528,6 +529,215 @@ TEST(MetaServiceTest, AlterS3StorageVaultTest) {
ASSERT_NE(res.status().code(), MetaServiceCode::OK) << res.status().msg();
}
+ {
+ AlterObjStoreInfoRequest req;
+ constexpr char new_vault_name[] = "@!#vault_name";
+ req.set_cloud_unique_id("test_cloud_unique_id");
+ req.set_op(AlterObjStoreInfoRequest::ALTER_S3_VAULT);
+ StorageVaultPB vault;
+ vault.set_alter_name(new_vault_name);
+ ObjectStoreInfoPB obj;
+ obj_info.set_ak("new_ak");
+ vault.mutable_obj_info()->MergeFrom(obj);
+ vault.set_name(vault_name);
+ req.mutable_vault()->CopyFrom(vault);
+
+ brpc::Controller cntl;
+ AlterObjStoreInfoResponse res;
+ meta_service->alter_storage_vault(
+ reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, nullptr);
+ ASSERT_NE(res.status().code(), MetaServiceCode::OK) << res.status().msg();
+ ASSERT_TRUE(res.status().msg().find("invalid storage vault name") != std::string::npos)
+ << res.status().msg();
+ }
+
+ {
+ AlterObjStoreInfoRequest req;
+ constexpr char new_vault_name[] = "new_test_alter_s3_vault";
+ req.set_cloud_unique_id("test_cloud_unique_id");
+ req.set_op(AlterObjStoreInfoRequest::ALTER_S3_VAULT);
+ StorageVaultPB vault;
+ vault.set_alter_name(new_vault_name);
+ ObjectStoreInfoPB obj;
+ obj_info.set_ak("new_ak");
+ vault.mutable_obj_info()->MergeFrom(obj);
+ vault.set_name(vault_name);
+ req.mutable_vault()->CopyFrom(vault);
+
+ brpc::Controller cntl;
+ AlterObjStoreInfoResponse res;
+ meta_service->alter_storage_vault(
+ reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, nullptr);
+ ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << res.status().msg();
+ InstanceInfoPB instance;
+ get_test_instance(instance);
+
+ std::unique_ptr<Transaction> txn;
+ ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK);
+ std::string val;
+ ASSERT_EQ(txn->get(storage_vault_key({instance.instance_id(), "2"}), &val),
+ TxnErrorCode::TXN_OK);
+ StorageVaultPB get_obj;
+ get_obj.ParseFromString(val);
+ ASSERT_EQ(get_obj.name(), new_vault_name) << get_obj.obj_info().ak();
+ }
+
+ SyncPoint::get_instance()->disable_processing();
+ SyncPoint::get_instance()->clear_all_call_backs();
+}
+
+TEST(MetaServiceTest, AlterHdfsStorageVaultTest) {
+ auto meta_service = get_meta_service();
+
+ auto sp = SyncPoint::get_instance();
+ sp->enable_processing();
+ sp->set_call_back("encrypt_ak_sk:get_encryption_key", [](auto&& args) {
+ auto* ret = try_any_cast<int*>(args[0]);
+ *ret = 0;
+ auto* key = try_any_cast<std::string*>(args[1]);
+ *key = "selectdbselectdbselectdbselectdb";
+ auto* key_id = try_any_cast<int64_t*>(args[2]);
+ *key_id = 1;
+ });
+ std::pair<std::string, std::string> pair;
+ sp->set_call_back("extract_object_storage_info:get_aksk_pair", [&](auto&& args) {
+ auto* ret = try_any_cast<std::pair<std::string, std::string>*>(args[0]);
+ pair = *ret;
+ });
+
+ std::unique_ptr<Transaction> txn;
+ ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK);
+ std::string key;
+ std::string val;
+ InstanceKeyInfo key_info {"test_instance"};
+ instance_key(key_info, &key);
+
+ HdfsBuildConf hdfs_build_conf;
+ hdfs_build_conf.set_fs_name("fs_name");
+ hdfs_build_conf.set_user("root");
+ HdfsVaultInfo hdfs_info;
+ hdfs_info.set_prefix("root_path");
+ hdfs_info.mutable_build_conf()->MergeFrom(hdfs_build_conf);
+ StorageVaultPB vault;
+ constexpr char vault_name[] = "test_alter_hdfs_vault";
+ vault.mutable_hdfs_info()->MergeFrom(hdfs_info);
+ vault.set_name(vault_name);
+ vault.set_id("2");
+ InstanceInfoPB instance;
+ instance.add_storage_vault_names(vault.name());
+ instance.add_resource_ids(vault.id());
+ instance.set_instance_id("GetObjStoreInfoTestInstance");
+ val = instance.SerializeAsString();
+ txn->put(key, val);
+ txn->put(storage_vault_key({instance.instance_id(), "2"}), vault.SerializeAsString());
+ ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK);
+ txn = nullptr;
+
+ auto get_test_instance = [&](InstanceInfoPB& i) {
+ std::string key;
+ std::string val;
+ std::unique_ptr<Transaction> txn;
+ ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK);
+ InstanceKeyInfo key_info {"test_instance"};
+ instance_key(key_info, &key);
+ ASSERT_EQ(txn->get(key, &val), TxnErrorCode::TXN_OK);
+ i.ParseFromString(val);
+ };
+
+ {
+ AlterObjStoreInfoRequest req;
+ req.set_cloud_unique_id("test_cloud_unique_id");
+ req.set_op(AlterObjStoreInfoRequest::ALTER_HDFS_VAULT);
+ StorageVaultPB vault;
+ vault.mutable_hdfs_info()->mutable_build_conf()->set_user("hadoop");
+ vault.set_name(vault_name);
+ req.mutable_vault()->CopyFrom(vault);
+
+ brpc::Controller cntl;
+ AlterObjStoreInfoResponse res;
+ meta_service->alter_storage_vault(
+ reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, nullptr);
+ ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << res.status().msg();
+ InstanceInfoPB instance;
+ get_test_instance(instance);
+
+ std::unique_ptr<Transaction> txn;
+ ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK);
+ std::string val;
+ ASSERT_EQ(txn->get(storage_vault_key({instance.instance_id(), "2"}), &val),
+ TxnErrorCode::TXN_OK);
+ StorageVaultPB get_obj;
+ get_obj.ParseFromString(val);
+ ASSERT_EQ(get_obj.hdfs_info().build_conf().user(), "hadoop")
+ << get_obj.hdfs_info().build_conf().fs_name();
+ }
+
+ {
+ AlterObjStoreInfoRequest req;
+ req.set_cloud_unique_id("test_cloud_unique_id");
+ req.set_op(AlterObjStoreInfoRequest::ALTER_HDFS_VAULT);
+ StorageVaultPB vault;
+ auto* hdfs = vault.mutable_hdfs_info();
+ hdfs->set_prefix("fake_one");
+ vault.set_name("test_alter_hdfs_vault_non_exist");
+ req.mutable_vault()->CopyFrom(vault);
+
+ brpc::Controller cntl;
+ AlterObjStoreInfoResponse res;
+ meta_service->alter_storage_vault(
+ reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, nullptr);
+ ASSERT_NE(res.status().code(), MetaServiceCode::OK) << res.status().msg();
+ }
+
+ {
+ AlterObjStoreInfoRequest req;
+ constexpr char new_vault_name[] = "Thi213***@fakeVault";
+ req.set_cloud_unique_id("test_cloud_unique_id");
+ req.set_op(AlterObjStoreInfoRequest::ALTER_HDFS_VAULT);
+ StorageVaultPB vault;
+ vault.mutable_hdfs_info()->mutable_build_conf()->set_user("hadoop");
+ vault.set_name(vault_name);
+ vault.set_alter_name(new_vault_name);
+ req.mutable_vault()->CopyFrom(vault);
+
+ brpc::Controller cntl;
+ AlterObjStoreInfoResponse res;
+ meta_service->alter_storage_vault(
+ reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, nullptr);
+ ASSERT_NE(res.status().code(), MetaServiceCode::OK) << res.status().msg();
+ ASSERT_TRUE(res.status().msg().find("invalid storage vault name") != std::string::npos)
+ << res.status().msg();
+ }
+
+ {
+ AlterObjStoreInfoRequest req;
+ constexpr char new_vault_name[] = "new_test_alter_hdfs_vault";
+ req.set_cloud_unique_id("test_cloud_unique_id");
+ req.set_op(AlterObjStoreInfoRequest::ALTER_HDFS_VAULT);
+ StorageVaultPB vault;
+ vault.mutable_hdfs_info()->mutable_build_conf()->set_user("hadoop");
+ vault.set_name(vault_name);
+ vault.set_alter_name(new_vault_name);
+ req.mutable_vault()->CopyFrom(vault);
+
+ brpc::Controller cntl;
+ AlterObjStoreInfoResponse res;
+ meta_service->alter_storage_vault(
+ reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, nullptr);
+ ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << res.status().msg();
+ InstanceInfoPB instance;
+ get_test_instance(instance);
+
+ std::unique_ptr<Transaction> txn;
+ ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK);
+ std::string val;
+ ASSERT_EQ(txn->get(storage_vault_key({instance.instance_id(), "2"}), &val),
+ TxnErrorCode::TXN_OK);
+ StorageVaultPB get_obj;
+ get_obj.ParseFromString(val);
+ ASSERT_EQ(get_obj.name(), new_vault_name) << get_obj.obj_info().ak();
+ }
+
SyncPoint::get_instance()->disable_processing();
SyncPoint::get_instance()->clear_all_call_backs();
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateStorageVaultStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateStorageVaultStmt.java
index 78ba855ea4a..f1aff6b9ab6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateStorageVaultStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateStorageVaultStmt.java
@@ -37,12 +37,19 @@ import java.util.Map;
// PROPERTIES (key1 = value1, ...)
public class CreateStorageVaultStmt extends DdlStmt implements NotFallbackInParser {
private static final String TYPE = "type";
+
+ private static final String PATH_VERSION = "path_version";
+
+ private static final String SHARD_NUM = "shard_num";
+
private static final String SET_AS_DEFAULT = "set_as_default";
private final boolean ifNotExists;
private final String vaultName;
private final Map<String, String> properties;
private boolean setAsDefault;
+ private int pathVersion = 0;
+ private int numShard = 0;
private StorageVault.StorageVaultType vaultType;
public CreateStorageVaultStmt(boolean ifNotExists, String vaultName, Map<String, String> properties) {
@@ -64,6 +71,14 @@ public class CreateStorageVaultStmt extends DdlStmt implements NotFallbackInPars
return vaultName;
}
+ public int getNumShard() {
+ return numShard;
+ }
+
+ public int getPathVersion() {
+ return pathVersion;
+ }
+
public Map<String, String> getProperties() {
return properties;
}
@@ -108,6 +123,16 @@ public class CreateStorageVaultStmt extends DdlStmt implements NotFallbackInPars
if (type == null) {
throw new AnalysisException("Storage Vault type can't be null");
}
+ final String pathVersionString = properties.get(PATH_VERSION);
+ if (pathVersionString != null) {
+ this.pathVersion = Integer.parseInt(pathVersionString);
+ properties.remove(PATH_VERSION);
+ }
+ final String numShardString = properties.get(SHARD_NUM);
+ if (numShardString != null) {
+ this.numShard = Integer.parseInt(numShardString);
+ properties.remove(SHARD_NUM);
+ }
setAsDefault = Boolean.parseBoolean(properties.getOrDefault(SET_AS_DEFAULT, "false"));
setStorageVaultType(StorageVault.StorageVaultType.fromString(type));
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/HdfsStorageVault.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/HdfsStorageVault.java
index 4614beef828..9be463ee3a1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/HdfsStorageVault.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/HdfsStorageVault.java
@@ -27,6 +27,8 @@ import com.google.gson.annotations.SerializedName;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
+import java.util.Arrays;
+import java.util.HashSet;
import java.util.Map;
import java.util.Set;
@@ -56,6 +58,11 @@ public class HdfsStorageVault extends StorageVault {
public static final String HDFS_PREFIX = "hdfs:";
public static final String HDFS_FILE_PREFIX = "hdfs://";
+ public static final HashSet<String> FORBID_CHECK_PROPERTIES = new HashSet<>(Arrays.asList(
+ VAULT_PATH_PREFIX,
+ HADOOP_FS_NAME
+ ));
+
/**
* Property keys used by Doris, and should not be put in HDFS client configs,
* such as `type`, `path_prefix`, etc.
@@ -99,6 +106,8 @@ public class HdfsStorageVault extends StorageVault {
hdfsConfBuilder.setHdfsKerberosPrincipal(property.getValue());
} else if (property.getKey().equalsIgnoreCase(AuthenticationConfig.HADOOP_KERBEROS_KEYTAB)) {
hdfsConfBuilder.setHdfsKerberosKeytab(property.getValue());
+ } else if (property.getKey().equalsIgnoreCase(VAULT_NAME)) {
+ continue;
} else {
if (!nonHdfsConfPropertyKeys.contains(property.getKey().toLowerCase())) {
Cloud.HdfsBuildConf.HdfsConfKVPair.Builder conf = Cloud.HdfsBuildConf.HdfsConfKVPair.newBuilder();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3StorageVault.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3StorageVault.java
index 710354b0092..67cc351cdb3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3StorageVault.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3StorageVault.java
@@ -58,10 +58,11 @@ public class S3StorageVault extends StorageVault {
private static final String TYPE = "type";
- public static final HashSet<String> ALTER_CHECK_PROPERTIES = new HashSet<>(Arrays.asList(
+ public static final HashSet<String> ALLOW_ALTER_PROPERTIES = new HashSet<>(Arrays.asList(
TYPE,
S3Properties.ACCESS_KEY,
- S3Properties.SECRET_KEY
+ S3Properties.SECRET_KEY,
+ VAULT_NAME
));
@SerializedName(value = "properties")
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVault.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVault.java
index fb68581a00e..fe285b6919d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVault.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVault.java
@@ -59,11 +59,14 @@ public abstract class StorageVault {
}
}
+ protected static final String VAULT_NAME = "VAULT_NAME";
protected String name;
protected StorageVaultType type;
protected String id;
private boolean ifNotExists;
private boolean setAsDefault;
+ private int pathVersion = 0;
+ private int numShard = 0;
private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(true);
@@ -105,6 +108,13 @@ public abstract class StorageVault {
return this.setAsDefault;
}
+ public int getPathVersion() {
+ return pathVersion;
+ }
+
+ public int getNumShard() {
+ return numShard;
+ }
public String getId() {
return this.id;
@@ -142,6 +152,8 @@ public abstract class StorageVault {
default:
throw new DdlException("Unknown StorageVault type: " + type);
}
+ vault.pathVersion = stmt.getPathVersion();
+ vault.numShard = stmt.getNumShard();
return vault;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVaultMgr.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVaultMgr.java
index ee5356eeb2a..c0942d916d1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVaultMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVaultMgr.java
@@ -38,23 +38,21 @@ import com.google.common.annotations.VisibleForTesting;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
-import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
public class StorageVaultMgr {
private static final Logger LOG = LogManager.getLogger(StorageVaultMgr.class);
-
- // <VaultName, VaultId>
- private Pair<String, String> defaultVaultInfo;
-
- private ReadWriteLock rwLock = new ReentrantReadWriteLock();
-
private static final ExecutorService ALTER_BE_SYNC_THREAD_POOL = Executors.newFixedThreadPool(1);
-
private final SystemInfoService systemInfoService;
+ // <VaultName, VaultId>
+ private Pair<String, String> defaultVaultInfo;
+ private Map<String, String> vaultNameToVaultId = new HashMap<>();
+ private ReentrantReadWriteLock rwLock = new ReentrantReadWriteLock();
public StorageVaultMgr(SystemInfoService systemInfoService) {
this.systemInfoService = systemInfoService;
@@ -93,32 +91,85 @@ public class StorageVaultMgr {
return vaultId;
}
- public void alterStorageVault(StorageVaultType type, Map<String, String> properties, String name) throws Exception {
- if (type != StorageVaultType.S3) {
- throw new DdlException("Only support alter s3 storage vault");
- }
- properties.keySet().stream()
- .filter(key -> !S3StorageVault.ALTER_CHECK_PROPERTIES.contains(key))
- .findAny()
- .ifPresent(key -> {
- throw new IllegalArgumentException("Alter property " + key + " is not allowed.");
- });
- Cloud.AlterObjStoreInfoRequest.Builder requestBuilder
- = Cloud.AlterObjStoreInfoRequest.newBuilder();
- requestBuilder.setOp(Cloud.AlterObjStoreInfoRequest.Operation.ALTER_S3_VAULT);
+ private Cloud.StorageVaultPB.Builder buildAlterS3VaultRequest(Map<String, String> properties, String name)
+ throws Exception {
Cloud.ObjectStoreInfoPB.Builder objBuilder = S3Properties.getObjStoreInfoPB(properties);
Cloud.StorageVaultPB.Builder alterObjVaultBuilder = Cloud.StorageVaultPB.newBuilder();
alterObjVaultBuilder.setName(name);
alterObjVaultBuilder.setObjInfo(objBuilder.build());
- requestBuilder.setVault(alterObjVaultBuilder.build());
+ if (properties.containsKey(StorageVault.VAULT_NAME)) {
+ alterObjVaultBuilder.setAlterName(properties.get(StorageVault.VAULT_NAME));
+ }
+ return alterObjVaultBuilder;
+ }
+
+ private Cloud.StorageVaultPB.Builder buildAlterHdfsVaultRequest(Map<String, String> properties, String name)
+ throws Exception {
+ Cloud.HdfsVaultInfo hdfsInfos = HdfsStorageVault.generateHdfsParam(properties);
+ Cloud.StorageVaultPB.Builder alterHdfsInfoBuilder = Cloud.StorageVaultPB.newBuilder();
+ alterHdfsInfoBuilder.setName(name);
+ alterHdfsInfoBuilder.setHdfsInfo(hdfsInfos);
+ if (properties.containsKey(StorageVault.VAULT_NAME)) {
+ alterHdfsInfoBuilder.setAlterName(properties.get(StorageVault.VAULT_NAME));
+ }
+ return alterHdfsInfoBuilder;
+ }
+
+ private Cloud.StorageVaultPB.Builder buildAlterStorageVaultRequest(StorageVaultType type,
+ Map<String, String> properties, String name) throws Exception {
+ Cloud.StorageVaultPB.Builder builder;
+ if (type == StorageVaultType.S3) {
+ builder = buildAlterS3VaultRequest(properties, name);
+ } else if (type == StorageVaultType.HDFS) {
+ builder = buildAlterHdfsVaultRequest(properties, name);
+ } else {
+ throw new DdlException("Unknown storage vault type");
+ }
+ return builder;
+ }
+
+ private Cloud.StorageVaultPB.Builder buildAlterStorageVaultRequest(StorageVault vault) throws Exception {
+ Cloud.StorageVaultPB.Builder builder = buildAlterStorageVaultRequest(vault.getType(),
+ vault.getCopiedProperties(), vault.getName());
+ Cloud.StorageVaultPB.PathFormat.Builder pathBuilder = Cloud.StorageVaultPB.PathFormat.newBuilder();
+ pathBuilder.setShardNum(vault.getNumShard());
+ pathBuilder.setPathVersion(vault.getPathVersion());
+ builder.setPathFormat(pathBuilder);
+ return builder;
+ }
+
+ public void alterStorageVault(StorageVaultType type, Map<String, String> properties, String name) throws Exception {
+ if (type == StorageVaultType.UNKNOWN) {
+ throw new DdlException("Unknown storage vault type");
+ }
try {
+ Cloud.AlterObjStoreInfoRequest.Builder request = Cloud.AlterObjStoreInfoRequest.newBuilder();
+ if (type == StorageVaultType.S3) {
+ properties.keySet().stream()
+ .filter(key -> !S3StorageVault.ALLOW_ALTER_PROPERTIES.contains(key))
+ .findAny()
+ .ifPresent(key -> {
+ throw new IllegalArgumentException("Alter property " + key + " is not allowed.");
+ });
+ request.setOp(Operation.ALTER_S3_VAULT);
+ } else if (type == StorageVaultType.HDFS) {
+ properties.keySet().stream()
+ .filter(HdfsStorageVault.FORBID_CHECK_PROPERTIES::contains)
+ .findAny()
+ .ifPresent(key -> {
+ throw new IllegalArgumentException("Alter property " + key + " is not allowed.");
+ });
+ request.setOp(Operation.ALTER_HDFS_VAULT);
+ }
+ Cloud.StorageVaultPB.Builder vaultBuilder = buildAlterStorageVaultRequest(type, properties, name);
+ request.setVault(vaultBuilder);
Cloud.AlterObjStoreInfoResponse response =
- MetaServiceProxy.getInstance().alterStorageVault(requestBuilder.build());
+ MetaServiceProxy.getInstance().alterStorageVault(request.build());
if (response.getStatus().getCode() != Cloud.MetaServiceCode.OK) {
LOG.warn("failed to alter storage vault response: {} ", response);
throw new DdlException(response.getStatus().getMsg());
}
- LOG.info("Succeed to alter s3 vault {}, id {}, origin default vault replaced {}",
+ LOG.info("Succeed to alter storage vault {}, id {}, origin default vault replaced {}",
name, response.getStorageVaultId(), response.getDefaultStorageVaultReplaced());
} catch (RpcException e) {
LOG.warn("failed to alter storage vault due to RpcException: {}", e);
@@ -193,12 +244,8 @@ public class StorageVaultMgr {
}
@VisibleForTesting
- public void createHdfsVault(StorageVault vault) throws DdlException {
- HdfsStorageVault hdfsStorageVault = (HdfsStorageVault) vault;
- Cloud.HdfsVaultInfo hdfsInfos = HdfsStorageVault.generateHdfsParam(hdfsStorageVault.getCopiedProperties());
- Cloud.StorageVaultPB.Builder alterHdfsInfoBuilder = Cloud.StorageVaultPB.newBuilder();
- alterHdfsInfoBuilder.setName(hdfsStorageVault.getName());
- alterHdfsInfoBuilder.setHdfsInfo(hdfsInfos);
+ public void createHdfsVault(StorageVault vault) throws Exception {
+ Cloud.StorageVaultPB.Builder alterHdfsInfoBuilder = buildAlterStorageVaultRequest(vault);
Cloud.AlterObjStoreInfoRequest.Builder requestBuilder
= Cloud.AlterObjStoreInfoRequest.newBuilder();
requestBuilder.setOp(Cloud.AlterObjStoreInfoRequest.Operation.ADD_HDFS_INFO);
@@ -208,18 +255,21 @@ public class StorageVaultMgr {
Cloud.AlterObjStoreInfoResponse response =
MetaServiceProxy.getInstance().alterStorageVault(requestBuilder.build());
if (response.getStatus().getCode() == Cloud.MetaServiceCode.ALREADY_EXISTED
- && hdfsStorageVault.ifNotExists()) {
- LOG.info("Hdfs vault {} already existed", hdfsStorageVault.getName());
+ && vault.ifNotExists()) {
+ LOG.info("Hdfs vault {} already existed", vault.getName());
return;
}
if (response.getStatus().getCode() != Cloud.MetaServiceCode.OK) {
LOG.warn("failed to create hdfs storage vault, vault name {}, response: {} ",
- hdfsStorageVault.getName(), response);
+ vault.getName(), response);
throw new DdlException(response.getStatus().getMsg());
}
+ rwLock.writeLock().lock();
+ vaultNameToVaultId.put(vault.getName(), response.getStorageVaultId());
+ rwLock.writeLock().unlock();
LOG.info("Succeed to create hdfs vault {}, id {}, origin default vault replaced {}",
- hdfsStorageVault.getName(), response.getStorageVaultId(),
- response.getDefaultStorageVaultReplaced());
+ vault.getName(), response.getStorageVaultId(),
+ response.getDefaultStorageVaultReplaced());
} catch (RpcException e) {
LOG.warn("failed to alter storage vault due to RpcException: {}", e);
throw new DdlException(e.getMessage());
@@ -245,31 +295,30 @@ public class StorageVaultMgr {
});
}
- public void createS3Vault(StorageVault vault) throws DdlException {
- S3StorageVault s3StorageVault = (S3StorageVault) vault;
+ public void createS3Vault(StorageVault vault) throws Exception {
+ Cloud.StorageVaultPB.Builder s3StorageVaultBuilder = buildAlterStorageVaultRequest(vault);
Cloud.AlterObjStoreInfoRequest.Builder requestBuilder
= Cloud.AlterObjStoreInfoRequest.newBuilder();
requestBuilder.setOp(Cloud.AlterObjStoreInfoRequest.Operation.ADD_S3_VAULT);
- Cloud.ObjectStoreInfoPB.Builder objBuilder = S3Properties.getObjStoreInfoPB(vault.getCopiedProperties());
- Cloud.StorageVaultPB.Builder alterObjVaultBuilder = Cloud.StorageVaultPB.newBuilder();
- alterObjVaultBuilder.setName(s3StorageVault.getName());
- alterObjVaultBuilder.setObjInfo(objBuilder.build());
- requestBuilder.setVault(alterObjVaultBuilder.build());
+ requestBuilder.setVault(s3StorageVaultBuilder);
requestBuilder.setSetAsDefaultStorageVault(vault.setAsDefault());
try {
Cloud.AlterObjStoreInfoResponse response =
MetaServiceProxy.getInstance().alterStorageVault(requestBuilder.build());
if (response.getStatus().getCode() == Cloud.MetaServiceCode.ALREADY_EXISTED
- && s3StorageVault.ifNotExists()) {
- LOG.info("S3 vault {} already existed", s3StorageVault.getName());
+ && vault.ifNotExists()) {
+ LOG.info("S3 vault {} already existed", vault.getName());
return;
}
if (response.getStatus().getCode() != Cloud.MetaServiceCode.OK) {
LOG.warn("failed to alter storage vault response: {} ", response);
throw new DdlException(response.getStatus().getMsg());
}
+ rwLock.writeLock().lock();
+ vaultNameToVaultId.put(vault.getName(), response.getStorageVaultId());
+ rwLock.writeLock().unlock();
LOG.info("Succeed to create s3 vault {}, id {}, origin default vault replaced {}",
- s3StorageVault.getName(), response.getStorageVaultId(), response.getDefaultStorageVaultReplaced());
+ vault.getName(), response.getStorageVaultId(), response.getDefaultStorageVaultReplaced());
} catch (RpcException e) {
LOG.warn("failed to alter storage vault due to RpcException: {}", e);
throw new DdlException(e.getMessage());
diff --git a/gensrc/proto/cloud.proto b/gensrc/proto/cloud.proto
index b113debae64..cb07ea1ec07 100644
--- a/gensrc/proto/cloud.proto
+++ b/gensrc/proto/cloud.proto
@@ -212,6 +212,8 @@ message StorageVaultPB {
}
optional PathFormat path_format = 5;
+ // Set this filed when the user tries to alter name to alter_name
+ optional string alter_name = 6;
}
message HdfsBuildConf {
@@ -858,6 +860,7 @@ message AlterObjStoreInfoRequest {
ADD_S3_VAULT = 103;
DROP_S3_VAULT = 104;
ALTER_S3_VAULT = 105;
+ ALTER_HDFS_VAULT = 106;
SET_DEFAULT_VAULT = 200;
UNSET_DEFAULT_VAULT = 201;
diff --git a/regression-test/suites/vaults/alter/alter_hdfs.groovy b/regression-test/suites/vaults/alter/alter_hdfs.groovy
new file mode 100644
index 00000000000..1a1299a93cc
--- /dev/null
+++ b/regression-test/suites/vaults/alter/alter_hdfs.groovy
@@ -0,0 +1,119 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("alter_hdfs_vault", "nonConcurrent") {
+ if (!enableStoragevault()) {
+ logger.info("skip alter hdfs storgage vault case")
+ return
+ }
+
+ sql """
+ CREATE STORAGE VAULT IF NOT EXISTS alter_hdfs_vault
+ PROPERTIES (
+ "type"="HDFS",
+ "fs.defaultFS"="${getHmsHdfsFs()}",
+ "path_prefix" = "ssb_sf1_p2",
+ "hadoop.username" = "hadoop"
+ );
+ """
+
+ sql """
+ CREATE TABLE IF NOT EXISTS alter_hdfs_vault_tbl (
+ C_CUSTKEY INTEGER NOT NULL,
+ C_NAME INTEGER NOT NULL
+ )
+ DUPLICATE KEY(C_CUSTKEY, C_NAME)
+ DISTRIBUTED BY HASH(C_CUSTKEY) BUCKETS 1
+ PROPERTIES (
+ "replication_num" = "1",
+ "storage_vault_name" = "alter_hdfs_vault"
+ )
+ """
+
+ sql """
+ insert into alter_hdfs_vault_tbl values("1", "1");
+ """
+
+ expectExceptionLike({
+ sql """
+ ALTER STORAGE VAULT alter_hdfs_vault
+ PROPERTIES (
+ "type"="hdfs",
+ "path_prefix" = "ssb_sf1_p3"
+ );
+ """
+ }, "Alter property")
+
+ expectExceptionLike({
+ sql """
+ ALTER STORAGE VAULT alter_hdfs_vault
+ PROPERTIES (
+ "type"="hdfs",
+ "fs.defaultFS" = "ssb_sf1_p3"
+ );
+ """
+ }, "Alter property")
+
+ def vault_name = "alter_hdfs_vault"
+ String properties;
+
+ def vaults_info = try_sql """
+ show storage vault
+ """
+
+ for (int i = 0; i < vaults_info.size(); i++) {
+ def name = vaults_info[i][0]
+ if (name.equals(vault_name)) {
+ properties = vaults_info[i][2]
+ }
+ }
+
+ sql """
+ ALTER STORAGE VAULT alter_hdfs_vault
+ PROPERTIES (
+ "type"="hdfs",
+ "VAULT_NAME" = "alter_hdfs_vault_new_name",
+ "hadoop.username" = "hdfs"
+ );
+ """
+
+ def new_vault_name = "alter_hdfs_vault_new_name"
+
+ vaults_info = sql """
+ SHOW STORAGE VAULT;
+ """
+ boolean exist = false
+
+ for (int i = 0; i < vaults_info.size(); i++) {
+ def name = vaults_info[i][0]
+ logger.info("name is ${name}, info ${vaults_info[i]}")
+ if (name.equals(vault_name)) {
+ exist = true
+ }
+ if (name.equals(new_vault_name)) {
+ assertTrue(vaults_info[i][2].contains(""""hadoop.username" = "hdfs"""""))
+ }
+ }
+ assertFalse(exist)
+
+ // failed to insert due to the wrong ak
+ expectExceptionLike({
+ sql """
+ insert into alter_hdfs_vault_tbl values("2", "2");
+ """
+ }, "")
+}
diff --git a/regression-test/suites/vaults/alter/alter_s3.groovy b/regression-test/suites/vaults/alter/alter_s3.groovy
new file mode 100644
index 00000000000..37f9edd0415
--- /dev/null
+++ b/regression-test/suites/vaults/alter/alter_s3.groovy
@@ -0,0 +1,124 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("alter_s3_vault", "nonConcurrent") {
+ if (!enableStoragevault()) {
+ logger.info("skip alter s3 storgage vault case")
+ return
+ }
+
+ sql """
+ CREATE STORAGE VAULT IF NOT EXISTS alter_s3_vault
+ PROPERTIES (
+ "type"="S3",
+ "s3.endpoint"="${getS3Endpoint()}",
+ "s3.region" = "${getS3Region()}",
+ "s3.access_key" = "${getS3AK()}",
+ "s3.secret_key" = "${getS3SK()}",
+ "s3.root.path" = "ssb_sf1_p2_s3",
+ "s3.bucket" = "${getS3BucketName()}",
+ "s3.external_endpoint" = "",
+ "provider" = "${getS3Provider()}"
+ );
+ """
+
+ sql """
+ CREATE TABLE IF NOT EXISTS alter_s3_vault_tbl (
+ C_CUSTKEY INTEGER NOT NULL,
+ C_NAME INTEGER NOT NULL
+ )
+ DUPLICATE KEY(C_CUSTKEY, C_NAME)
+ DISTRIBUTED BY HASH(C_CUSTKEY) BUCKETS 1
+ PROPERTIES (
+ "replication_num" = "1",
+ "storage_vault_name" = "alter_s3_vault"
+ )
+ """
+
+ sql """
+ insert into alter_s3_vault_tbl values("1", "1");
+ """
+
+ expectExceptionLike({
+ sql """
+ ALTER STORAGE VAULT alter_s3_vault
+ PROPERTIES (
+ "type"="S3",
+ "s3.bucket" = "error_bucket"
+ );
+ """
+ }, "Alter property")
+ expectExceptionLike({
+ sql """
+ ALTER STORAGE VAULT alter_s3_vault
+ PROPERTIES (
+ "type"="S3",
+ "provider" = "${getS3Provider()}"
+ );
+ """
+ }, "Alter property")
+
+ def vault_name = "alter_s3_vault"
+ String properties;
+
+ def vaults_info = try_sql """
+ show storage vault
+ """
+
+ for (int i = 0; i < vaults_info.size(); i++) {
+ def name = vaults_info[i][0]
+ if (name.equals(vault_name)) {
+ properties = vaults_info[i][2]
+ }
+ }
+
+ sql """
+ ALTER STORAGE VAULT alter_s3_vault
+ PROPERTIES (
+ "type"="S3",
+ "VAULT_NAME" = "alter_s3_vault",
+ "s3.access_key" = "new_ak"
+ );
+ """
+
+ def new_vault_name = "alter_s3_vault_new"
+
+ vaults_info = sql """
+ SHOW STORAGE VAULT;
+ """
+ boolean exist = false
+
+ for (int i = 0; i < vaults_info.size(); i++) {
+ def name = vaults_info[i][0]
+ logger.info("name is ${name}, info ${vaults_info[i]}")
+ if (name.equals(vault_name)) {
+ exist = true
+ }
+ if (name.equals(new_vault_name)) {
+ assertTrue(vaults_info[i][2].contains(""""s3.access_key" = "new_ak"""""))
+ }
+ }
+ assertFalse(exist)
+
+ // failed to insert due to the wrong ak
+ expectExceptionLike({
+ sql """
+ insert into alter_s3_vault_tbl values("2", "2");
+ """
+ }, "")
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 08/11: [enhance](cloud) Add a policy when be abnormal, tablet delay switch be (#40371)
Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gavinchou pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 2f3df2d0ef3b1af73957c78ba8030bf1af7880e3
Author: deardeng <56...@qq.com>
AuthorDate: Fri Sep 6 19:10:17 2024 +0800
[enhance](cloud) Add a policy when be abnormal, tablet delay switch be (#40371)
---
.../main/java/org/apache/doris/common/Config.java | 3 +
.../apache/doris/cloud/catalog/CloudReplica.java | 79 +++++++++++++++-------
.../doris/cloud/catalog/CloudTabletRebalancer.java | 21 +++---
.../cloud/datasource/CloudInternalCatalog.java | 4 +-
.../cloud_p0/multi_cluster/test_rebalance.groovy | 2 +-
5 files changed, 71 insertions(+), 38 deletions(-)
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index eb70a71a9c3..42affa36daf 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -3014,6 +3014,9 @@ public class Config extends ConfigBase {
@ConfField(mutable = true, description = {"存算分离模式下是否开启大事务提交，默认false"})
public static boolean enable_cloud_txn_lazy_commit = false;
+ @ConfField(mutable = true, description = {"存算分离模式下，当tablet分布的be异常，是否立即映射tablet到新的be上，默认true"})
+ public static boolean enable_immediate_be_assign = true;
+
// ATTN: DONOT add any config not related to cloud mode here
// ATTN: DONOT add any config not related to cloud mode here
// ATTN: DONOT add any config not related to cloud mode here
diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudReplica.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudReplica.java
index be0c510559e..43f7dcbc687 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudReplica.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudReplica.java
@@ -20,7 +20,6 @@ package org.apache.doris.cloud.catalog;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.Partition;
import org.apache.doris.catalog.Replica;
-import org.apache.doris.catalog.Replica.ReplicaContext;
import org.apache.doris.cloud.system.CloudSystemInfoService;
import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
@@ -49,7 +48,7 @@ public class CloudReplica extends Replica {
// In the future, a replica may be mapped to multiple BEs in a cluster,
// so this value is be list
- private Map<String, List<Long>> clusterToBackends = new ConcurrentHashMap<String, List<Long>>();
+ private Map<String, List<Long>> primaryClusterToBackends = new ConcurrentHashMap<String, List<Long>>();
@SerializedName(value = "dbId")
private long dbId = -1;
@SerializedName(value = "tableId")
@@ -65,6 +64,9 @@ public class CloudReplica extends Replica {
private Map<String, List<Long>> memClusterToBackends = new ConcurrentHashMap<String, List<Long>>();
+ // clusterId, secondaryBe, changeTimestamp
+ private Map<String, List<Long>> secondaryClusterToBackends = new ConcurrentHashMap<String, List<Long>>();
+
public CloudReplica() {
}
@@ -186,8 +188,8 @@ public class CloudReplica extends Replica {
backendId = memClusterToBackends.get(clusterId).get(indexRand);
}
- if (!replicaEnough && !allowColdRead && clusterToBackends.containsKey(clusterId)) {
- backendId = clusterToBackends.get(clusterId).get(0);
+ if (!replicaEnough && !allowColdRead && primaryClusterToBackends.containsKey(clusterId)) {
+ backendId = primaryClusterToBackends.get(clusterId).get(0);
}
if (backendId > 0) {
@@ -212,21 +214,47 @@ public class CloudReplica extends Replica {
}
}
- if (clusterToBackends.containsKey(clusterId)) {
- long backendId = clusterToBackends.get(clusterId).get(0);
- Backend be = Env.getCurrentSystemInfo().getBackend(backendId);
- if (be != null && be.isQueryAvailable()) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("backendId={} ", backendId);
- }
- return backendId;
+ // use primaryClusterToBackends, if find be normal
+ long pickBeId = getAvaliableBeId(clusterId, primaryClusterToBackends);
+ if (pickBeId != -1) {
+ return pickBeId;
+ }
+
+ if (!Config.enable_immediate_be_assign) {
+ // use secondaryClusterToBackends, if find be normal
+ pickBeId = getAvaliableBeId(clusterId, secondaryClusterToBackends);
+ if (pickBeId != -1) {
+ return pickBeId;
}
}
- if (DebugPointUtil.isEnable("CloudReplica.getBackendIdImpl.clusterToBackends")) {
- LOG.info("Debug Point enable CloudReplica.getBackendIdImpl.clusterToBackends");
+
+ if (DebugPointUtil.isEnable("CloudReplica.getBackendIdImpl.primaryClusterToBackends")) {
+ LOG.info("Debug Point enable CloudReplica.getBackendIdImpl.primaryClusterToBackends");
+ return -1;
+ }
+
+ // be abnormal, rehash it. configure settings to different maps
+ pickBeId = hashReplicaToBe(clusterId, false);
+ updateClusterToBe(clusterId, pickBeId, Config.enable_immediate_be_assign);
+ return pickBeId;
+ }
+
+ private long getAvaliableBeId(String clusterId, Map<String, List<Long>> clusterToBackends) {
+ List<Long> backendIds = clusterToBackends.get(clusterId);
+ if (backendIds == null || backendIds.isEmpty()) {
return -1;
}
- return hashReplicaToBe(clusterId, false);
+
+ long backendId = backendIds.get(0);
+ Backend be = Env.getCurrentSystemInfo().getBackend(backendId);
+ if (be != null && be.isQueryAvailable()) {
+ // be normal
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("backendId={} ", backendId);
+ }
+ return backendId;
+ }
+ return -1;
}
public long hashReplicaToBe(String clusterId, boolean isBackGround) {
@@ -270,14 +298,10 @@ public class CloudReplica extends Replica {
pickedBeId, getId(), partitionId, availableBes.size(), idx, index,
hashCode == null ? -1 : hashCode.asLong());
- // save to clusterToBackends map
- List<Long> bes = new ArrayList<Long>();
- bes.add(pickedBeId);
- clusterToBackends.put(clusterId, bes);
-
return pickedBeId;
}
+
public List<Long> hashReplicaToBes(String clusterId, boolean isBackGround, int replicaNum) {
// TODO(luwei) list should be sorted
List<Backend> clusterBes = ((CloudSystemInfoService) Env.getCurrentSystemInfo())
@@ -375,7 +399,7 @@ public class CloudReplica extends Replica {
long beId = in.readLong();
List<Long> bes = new ArrayList<Long>();
bes.add(beId);
- clusterToBackends.put(clusterId, bes);
+ primaryClusterToBackends.put(clusterId, bes);
}
}
@@ -399,14 +423,19 @@ public class CloudReplica extends Replica {
return idx;
}
- public Map<String, List<Long>> getClusterToBackends() {
- return clusterToBackends;
+ public Map<String, List<Long>> getprimaryClusterToBackends() {
+ return primaryClusterToBackends;
}
- public void updateClusterToBe(String cluster, long beId) {
+ // save to primaryClusterToBackends or secondaryClusterToBackends map
+ public void updateClusterToBe(String cluster, long beId, boolean isUpdatePrimary) {
// write lock
List<Long> bes = new ArrayList<Long>();
bes.add(beId);
- clusterToBackends.put(cluster, bes);
+ if (isUpdatePrimary) {
+ primaryClusterToBackends.put(cluster, bes);
+ } else {
+ secondaryClusterToBackends.put(cluster, bes);
+ }
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletRebalancer.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletRebalancer.java
index 6f4534c4b08..ae34c6b722f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletRebalancer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletRebalancer.java
@@ -499,23 +499,24 @@ public class CloudTabletRebalancer extends MasterDaemon {
List<Long> tabletIds = new ArrayList<Long>();
for (Tablet tablet : index.getTablets()) {
for (Replica replica : tablet.getReplicas()) {
- Map<String, List<Long>> clusterToBackends =
- ((CloudReplica) replica).getClusterToBackends();
- if (!clusterToBackends.containsKey(cluster)) {
+ Map<String, List<Long>> primaryClusterToBackends =
+ ((CloudReplica) replica).getprimaryClusterToBackends();
+ if (!primaryClusterToBackends.containsKey(cluster)) {
long beId = ((CloudReplica) replica).hashReplicaToBe(cluster, true);
+ ((CloudReplica) replica).updateClusterToBe(cluster, beId, true);
if (beId <= 0) {
assignedErrNum++;
continue;
}
List<Long> bes = new ArrayList<Long>();
bes.add(beId);
- clusterToBackends.put(cluster, bes);
+ primaryClusterToBackends.put(cluster, bes);
assigned = true;
beIds.add(beId);
tabletIds.add(tablet.getId());
} else {
- beIds.add(clusterToBackends.get(cluster).get(0));
+ beIds.add(primaryClusterToBackends.get(cluster).get(0));
tabletIds.add(tablet.getId());
}
}
@@ -569,9 +570,9 @@ public class CloudTabletRebalancer extends MasterDaemon {
loopCloudReplica((Database db, Table table, Partition partition, MaterializedIndex index, String cluster) -> {
for (Tablet tablet : index.getTablets()) {
for (Replica replica : tablet.getReplicas()) {
- Map<String, List<Long>> clusterToBackends =
- ((CloudReplica) replica).getClusterToBackends();
- for (Map.Entry<String, List<Long>> entry : clusterToBackends.entrySet()) {
+ Map<String, List<Long>> primaryClusterToBackends =
+ ((CloudReplica) replica).getprimaryClusterToBackends();
+ for (Map.Entry<String, List<Long>> entry : primaryClusterToBackends.entrySet()) {
if (!cluster.equals(entry.getKey())) {
continue;
}
@@ -729,7 +730,7 @@ public class CloudTabletRebalancer extends MasterDaemon {
private void updateClusterToBeMap(Tablet pickedTablet, long destBe, String clusterId,
List<UpdateCloudReplicaInfo> infos) {
CloudReplica cloudReplica = (CloudReplica) pickedTablet.getReplicas().get(0);
- cloudReplica.updateClusterToBe(clusterId, destBe);
+ cloudReplica.updateClusterToBe(clusterId, destBe, true);
Database db = Env.getCurrentInternalCatalog().getDbNullable(cloudReplica.getDbId());
if (db == null) {
return;
@@ -973,7 +974,7 @@ public class CloudTabletRebalancer extends MasterDaemon {
String clusterId = be.getCloudClusterId();
String clusterName = be.getCloudClusterName();
// update replica location info
- cloudReplica.updateClusterToBe(clusterId, dstBe);
+ cloudReplica.updateClusterToBe(clusterId, dstBe, true);
LOG.info("cloud be migrate tablet {} from srcBe={} to dstBe={}, clusterId={}, clusterName={}",
tablet.getId(), srcBe, dstBe, clusterId, clusterName);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
index ef461c74b21..b7a932486ad 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
@@ -975,7 +975,7 @@ public class CloudInternalCatalog extends InternalCatalog {
clusterId = realClusterId;
}
- ((CloudReplica) replica).updateClusterToBe(clusterId, info.getBeId());
+ ((CloudReplica) replica).updateClusterToBe(clusterId, info.getBeId(), true);
LOG.debug("update single cloud replica cluster {} replica {} be {}", info.getClusterId(),
replica.getId(), info.getBeId());
@@ -1001,7 +1001,7 @@ public class CloudInternalCatalog extends InternalCatalog {
LOG.debug("update cloud replica cluster {} replica {} be {}", info.getClusterId(),
replica.getId(), info.getBeIds().get(i));
- ((CloudReplica) replica).updateClusterToBe(clusterId, info.getBeIds().get(i));
+ ((CloudReplica) replica).updateClusterToBe(clusterId, info.getBeIds().get(i), true);
}
}
} catch (Exception e) {
diff --git a/regression-test/suites/cloud_p0/multi_cluster/test_rebalance.groovy b/regression-test/suites/cloud_p0/multi_cluster/test_rebalance.groovy
index e1735a4acd4..0aa2e83ccc2 100644
--- a/regression-test/suites/cloud_p0/multi_cluster/test_rebalance.groovy
+++ b/regression-test/suites/cloud_p0/multi_cluster/test_rebalance.groovy
@@ -63,7 +63,7 @@ suite('test_rebalance_in_cloud', 'multi_cluster') {
PARTITION p1998 VALUES [("19980101"), ("19990101")))
DISTRIBUTED BY HASH(k1) BUCKETS 3
"""
- GetDebugPoint().enableDebugPointForAllFEs("CloudReplica.getBackendIdImpl.clusterToBackends");
+ GetDebugPoint().enableDebugPointForAllFEs("CloudReplica.getBackendIdImpl.primaryClusterToBackends");
sql """set global forward_to_master=false"""
// add a be
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 11/11: [enhance](Vault) Maintain VaultName to VaultId map in FE master (#37899)
Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gavinchou pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 4f4eefa962702116bb5534c73dc3596b584c5a95
Author: AlexYue <yj...@gmail.com>
AuthorDate: Thu Jul 18 20:24:23 2024 +0800
[enhance](Vault) Maintain VaultName to VaultId map in FE master (#37899)
All the vault related sql stmt is processed in FE master, so FE master
could serve as one cache for vault related map. For the create table
stmt with vault name, it can try to get the vault id from the map first.
---
.../main/java/org/apache/doris/catalog/StorageVaultMgr.java | 3 ---
.../doris/cloud/catalog/CloudInstanceStatusChecker.java | 11 +++++++++++
.../apache/doris/cloud/datasource/CloudInternalCatalog.java | 2 +-
.../java/org/apache/doris/datasource/InternalCatalog.java | 3 ++-
4 files changed, 14 insertions(+), 5 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVaultMgr.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVaultMgr.java
index c0942d916d1..2d1df457c05 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVaultMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVaultMgr.java
@@ -58,9 +58,6 @@ public class StorageVaultMgr {
this.systemInfoService = systemInfoService;
}
- // TODO(ByteYue): The CreateStorageVault should only be handled by master
- // which indicates we can maintains one <VaultName, VaultId> map in FE master
-
public void createStorageVaultResource(CreateStorageVaultStmt stmt) throws Exception {
switch (stmt.getStorageVaultType()) {
case HDFS:
diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudInstanceStatusChecker.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudInstanceStatusChecker.java
index 9342c705e9d..ec63e5b4d49 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudInstanceStatusChecker.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudInstanceStatusChecker.java
@@ -27,6 +27,9 @@ import org.apache.doris.common.util.MasterDaemon;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
+import java.util.HashMap;
+import java.util.Map;
+
public class CloudInstanceStatusChecker extends MasterDaemon {
private static final Logger LOG = LogManager.getLogger(CloudInstanceStatusChecker.class);
private CloudSystemInfoService cloudSystemInfoService;
@@ -49,6 +52,14 @@ public class CloudInstanceStatusChecker extends MasterDaemon {
+ "cloud_unique_id={}, response={}", Config.cloud_unique_id, response);
} else {
cloudSystemInfoService.setInstanceStatus(response.getInstance().getStatus());
+ Map<String, String> vaultMap = new HashMap<>();
+ int cnt = response.getInstance().getResourceIdsCount();
+ for (int i = 0; i < cnt; i++) {
+ String name = response.getInstance().getStorageVaultNames(i);
+ String id = response.getInstance().getResourceIds(i);
+ vaultMap.put(name, id);
+ }
+ Env.getCurrentEnv().getStorageVaultMgr().refreshVaultMap(vaultMap);
Env.getCurrentEnv().getStorageVaultMgr().setDefaultStorageVault(
Pair.of(response.getInstance().getDefaultStorageVaultName(),
response.getInstance().getDefaultStorageVaultId()));
diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
index b7a932486ad..9eb1400ab52 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
@@ -130,7 +130,7 @@ public class CloudInternalCatalog extends InternalCatalog {
long version = partition.getVisibleVersion();
final String storageVaultName = tbl.getStorageVaultName();
- boolean storageVaultIdSet = false;
+ boolean storageVaultIdSet = tbl.getStorageVaultId().isEmpty();
// short totalReplicaNum = replicaAlloc.getTotalReplicaNum();
for (Map.Entry<Long, MaterializedIndex> entry : indexMap.entrySet()) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
index d4b9d5eef14..2b7f6090bb9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
@@ -2703,7 +2703,8 @@ public class InternalCatalog implements CatalogIf<Database> {
}
olapTable.setStorageVaultName(storageVaultName);
- if (storageVaultId != null) {
+ storageVaultId = env.getStorageVaultMgr().getVaultIdByName(storageVaultName);
+ if (storageVaultId != null && !storageVaultId.isEmpty()) {
olapTable.setStorageVaultId(storageVaultId);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 02/11: [feature](cloud) Support multi follower fe in cloud (#38388)
Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gavinchou pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 0e113802529e9eecac96d894186b2f124dbfd745
Author: deardeng <56...@qq.com>
AuthorDate: Fri Sep 6 00:29:37 2024 +0800
[feature](cloud) Support multi follower fe in cloud (#38388)
---
cloud/src/resource-manager/resource_manager.cpp | 65 +++++++-
docker/runtime/doris-compose/cluster.py | 29 +++-
docker/runtime/doris-compose/command.py | 21 ++-
docker/runtime/doris-compose/resource/init_fe.sh | 17 +-
.../doris/cloud/catalog/CloudClusterChecker.java | 34 +++-
.../org/apache/doris/cloud/catalog/CloudEnv.java | 26 ++-
.../doris/cloud/catalog/CloudTabletRebalancer.java | 2 +-
.../doris/cloud/system/CloudSystemInfoService.java | 23 ++-
gensrc/proto/cloud.proto | 2 +
.../org/apache/doris/regression/suite/Suite.groovy | 80 ++++++++++
.../doris/regression/suite/SuiteCluster.groovy | 23 ++-
.../multi_cluster/test_multi_follower.groovy | 176 +++++++++++++++++++++
12 files changed, 446 insertions(+), 52 deletions(-)
diff --git a/cloud/src/resource-manager/resource_manager.cpp b/cloud/src/resource-manager/resource_manager.cpp
index 25848294aa7..58122ac3f7c 100644
--- a/cloud/src/resource-manager/resource_manager.cpp
+++ b/cloud/src/resource-manager/resource_manager.cpp
@@ -157,15 +157,18 @@ bool ResourceManager::check_cluster_params_valid(const ClusterPB& cluster, std::
std::stringstream ss;
bool no_err = true;
int master_num = 0;
+ int follower_num = 0;
for (auto& n : cluster.nodes()) {
if (ClusterPB::SQL == cluster.type() && n.has_edit_log_port() && n.edit_log_port() &&
n.has_node_type() &&
(n.node_type() == NodeInfoPB_NodeType_FE_MASTER ||
- n.node_type() == NodeInfoPB_NodeType_FE_OBSERVER)) {
+ n.node_type() == NodeInfoPB_NodeType_FE_OBSERVER ||
+ n.node_type() == NodeInfoPB_NodeType_FE_FOLLOWER)) {
master_num += n.node_type() == NodeInfoPB_NodeType_FE_MASTER ? 1 : 0;
+ follower_num += n.node_type() == NodeInfoPB_NodeType_FE_FOLLOWER ? 1 : 0;
continue;
- }
- if (ClusterPB::COMPUTE == cluster.type() && n.has_heartbeat_port() && n.heartbeat_port()) {
+ } else if (ClusterPB::COMPUTE == cluster.type() && n.has_heartbeat_port() &&
+ n.heartbeat_port()) {
continue;
}
ss << "check cluster params failed, node : " << proto_to_json(n);
@@ -173,12 +176,22 @@ bool ResourceManager::check_cluster_params_valid(const ClusterPB& cluster, std::
no_err = false;
break;
}
- // ATTN: add_cluster check must have only a master node
- // add_node doesn't check it
- if (check_master_num && ClusterPB::SQL == cluster.type() && master_num != 1) {
+
+ if (check_master_num && ClusterPB::SQL == cluster.type()) {
no_err = false;
- ss << "cluster is SQL type, must have only one master node, now master count: "
- << master_num;
+ if (master_num > 0 && follower_num > 0) {
+ ss << "cluster is SQL type, and use multi follower mode, cant set master node, master "
+ "count: "
+ << master_num << " follower count: " << follower_num;
+ } else if (!follower_num && master_num != 1) {
+ ss << "cluster is SQL type, must have only one master node, now master count: "
+ << master_num;
+ } else {
+ // followers mode
+ // 1. followers 2. observers + followers
+ no_err = true;
+ ss << "";
+ }
*err = ss.str();
}
return no_err;
@@ -618,6 +631,37 @@ std::pair<TxnErrorCode, std::string> ResourceManager::get_instance(std::shared_p
return ec;
}
+// check instance pb is valid
+bool is_instance_valid(const InstanceInfoPB& instance) {
+ // check has fe node
+ for (auto& c : instance.clusters()) {
+ if (c.has_type() && c.type() == ClusterPB::SQL) {
+ int master = 0;
+ int follower = 0;
+ std::string mode = "multi-followers";
+ for (auto& n : c.nodes()) {
+ if (n.node_type() == NodeInfoPB::FE_MASTER) {
+ mode = "master-observers";
+ master++;
+ } else if (n.node_type() == NodeInfoPB::FE_FOLLOWER) {
+ follower++;
+ }
+ }
+ // if master/observers mode , not have master or have multi master, return false
+ if (mode == "master-observers" && master != 1) {
+ return false;
+ }
+ // if multi followers mode, not have follower, return false
+ if (mode == "multi-followers" && !follower) {
+ return false;
+ }
+ return true;
+ }
+ }
+ // check others ...
+ return true;
+}
+
std::string ResourceManager::modify_nodes(const std::string& instance_id,
const std::vector<NodeInfo>& to_add,
const std::vector<NodeInfo>& to_del) {
@@ -910,6 +954,11 @@ std::string ResourceManager::modify_nodes(const std::string& instance_id,
}
LOG(INFO) << "instance " << instance_id << " info: " << instance.DebugString();
+ if (!to_del.empty() && !is_instance_valid(instance)) {
+ msg = "instance invalid, cant modify, plz check";
+ LOG(WARNING) << msg;
+ return msg;
+ }
InstanceKeyInfo key_info {instance_id};
std::string key;
diff --git a/docker/runtime/doris-compose/cluster.py b/docker/runtime/doris-compose/cluster.py
index 466ccf79c1d..cbccf7550aa 100644
--- a/docker/runtime/doris-compose/cluster.py
+++ b/docker/runtime/doris-compose/cluster.py
@@ -372,6 +372,10 @@ class Node(object):
class FE(Node):
+ def init(self):
+ super().init()
+ self.init_is_follower()
+
def get_add_init_config(self):
cfg = []
if self.cluster.fe_config:
@@ -389,10 +393,20 @@ class FE(Node):
return cfg
+ def init_is_follower(self):
+ if self.cluster.is_cloud and self.cluster.fe_follower:
+ with open(self._is_follower_path(), "w") as f:
+ f.write("true")
+
+ def _is_follower_path(self):
+ return "{}/conf/is_follower".format(self.get_path())
+
def docker_env(self):
envs = super().docker_env()
if self.cluster.is_cloud:
envs["CLOUD_UNIQUE_ID"] = self.cloud_unique_id()
+ if os.path.exists(self._is_follower_path()):
+ envs["IS_FE_FOLLOWER"] = 1
return envs
def cloud_unique_id(self):
@@ -605,8 +619,8 @@ class FDB(Node):
class Cluster(object):
def __init__(self, name, subnet, image, is_cloud, fe_config, be_config,
- ms_config, recycle_config, be_disks, be_cluster, reg_be,
- coverage_dir, cloud_store_config):
+ ms_config, recycle_config, fe_follower, be_disks, be_cluster,
+ reg_be, coverage_dir, cloud_store_config):
self.name = name
self.subnet = subnet
self.image = image
@@ -615,6 +629,7 @@ class Cluster(object):
self.be_config = be_config
self.ms_config = ms_config
self.recycle_config = recycle_config
+ self.fe_follower = fe_follower
self.be_disks = be_disks
self.be_cluster = be_cluster
self.reg_be = reg_be
@@ -627,8 +642,8 @@ class Cluster(object):
@staticmethod
def new(name, image, is_cloud, fe_config, be_config, ms_config,
- recycle_config, be_disks, be_cluster, reg_be, coverage_dir,
- cloud_store_config):
+ recycle_config, fe_follower, be_disks, be_cluster, reg_be,
+ coverage_dir, cloud_store_config):
if not os.path.exists(LOCAL_DORIS_PATH):
os.makedirs(LOCAL_DORIS_PATH, exist_ok=True)
os.chmod(LOCAL_DORIS_PATH, 0o777)
@@ -637,9 +652,9 @@ class Cluster(object):
os.chmod(lock_file, 0o666)
subnet = gen_subnet_prefix16()
cluster = Cluster(name, subnet, image, is_cloud, fe_config,
- be_config, ms_config, recycle_config, be_disks,
- be_cluster, reg_be, coverage_dir,
- cloud_store_config)
+ be_config, ms_config, recycle_config,
+ fe_follower, be_disks, be_cluster, reg_be,
+ coverage_dir, cloud_store_config)
os.makedirs(cluster.get_path(), exist_ok=True)
os.makedirs(get_status_path(name), exist_ok=True)
cluster._save_meta()
diff --git a/docker/runtime/doris-compose/command.py b/docker/runtime/doris-compose/command.py
index b0863eae153..bba125d043c 100644
--- a/docker/runtime/doris-compose/command.py
+++ b/docker/runtime/doris-compose/command.py
@@ -245,6 +245,13 @@ class UpCommand(Command):
type=str,
help="Specify recycle configs for doris_cloud.conf. "\
"Example: --recycle-config \"log_level = warn\".")
+ group1.add_argument(
+ "--fe-follower",
+ default=False,
+ action=self._get_parser_bool_action(True),
+ help=
+ "The new added fe is follower but not observer. Only support in cloud mode."
+ )
group1.add_argument("--be-disks",
nargs="*",
default=["HDD=1"],
@@ -383,18 +390,20 @@ class UpCommand(Command):
args.add_ms_num = 0
args.add_recycle_num = 0
- cluster = CLUSTER.Cluster.new(args.NAME, args.IMAGE, args.cloud,
- args.fe_config, args.be_config,
- args.ms_config, args.recycle_config,
- args.be_disks, args.be_cluster,
- args.reg_be, args.coverage_dir,
- cloud_store_config)
+ cluster = CLUSTER.Cluster.new(
+ args.NAME, args.IMAGE, args.cloud, args.fe_config,
+ args.be_config, args.ms_config, args.recycle_config,
+ args.fe_follower, args.be_disks, args.be_cluster, args.reg_be,
+ args.coverage_dir, cloud_store_config)
LOG.info("Create new cluster {} succ, cluster path is {}".format(
args.NAME, cluster.get_path()))
if args.be_cluster and cluster.is_cloud:
cluster.be_cluster = args.be_cluster
+ if cluster.is_cloud:
+ cluster.fe_follower = args.fe_follower
+
_, related_nodes, _ = get_ids_related_nodes(cluster, args.fe_id,
args.be_id, args.ms_id,
args.recycle_id,
diff --git a/docker/runtime/doris-compose/resource/init_fe.sh b/docker/runtime/doris-compose/resource/init_fe.sh
index e532d0d56e1..e46a6cac9b2 100755
--- a/docker/runtime/doris-compose/resource/init_fe.sh
+++ b/docker/runtime/doris-compose/resource/init_fe.sh
@@ -81,8 +81,10 @@ fe_daemon() {
done
}
-add_cloud_fe() {
+start_cloud_fe() {
if [ -f "$REGISTER_FILE" ]; then
+ fe_daemon &
+ bash $DORIS_HOME/bin/start_fe.sh --daemon
return
fi
@@ -96,6 +98,10 @@ add_cloud_fe() {
node_type=FE_OBSERVER
fi
+ if [ "a$IS_FE_FOLLOWER" == "a1" ]; then
+ node_type=FE_FOLLOWER
+ fi
+
nodes='{
"cloud_unique_id": "'"${CLOUD_UNIQUE_ID}"'",
"ip": "'"${MY_IP}"'",
@@ -139,6 +145,10 @@ add_cloud_fe() {
fi
touch $REGISTER_FILE
+
+ fe_daemon &
+ bash $DORIS_HOME/bin/start_fe.sh --daemon
+
if [ "$MY_ID" == "1" ]; then
echo $MY_IP >$MASTER_FE_IP_FILE
fi
@@ -182,11 +192,6 @@ start_local_fe() {
fi
}
-start_cloud_fe() {
- add_cloud_fe
- bash $DORIS_HOME/bin/start_fe.sh --daemon
-}
-
main() {
trap stop_frontend SIGTERM
diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudClusterChecker.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudClusterChecker.java
index 72fd2ba353a..567dc4b3124 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudClusterChecker.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudClusterChecker.java
@@ -303,9 +303,9 @@ public class CloudClusterChecker extends MasterDaemon {
@Override
protected void runAfterCatalogReady() {
- getCloudBackends();
+ checkCloudBackends();
updateCloudMetrics();
- getCloudObserverFes();
+ checkCloudFes();
}
private void checkFeNodesMapValid() {
@@ -348,7 +348,7 @@ public class CloudClusterChecker extends MasterDaemon {
}
}
- private void getCloudObserverFes() {
+ private void checkCloudFes() {
Cloud.GetClusterResponse response = cloudSystemInfoService.getCloudCluster(
Config.cloud_sql_server_cluster_name, Config.cloud_sql_server_cluster_id, "");
if (!response.hasStatus() || !response.getStatus().hasCode()
@@ -371,11 +371,19 @@ public class CloudClusterChecker extends MasterDaemon {
LOG.debug("get cloud cluster, clusterId={} nodes={}",
Config.cloud_sql_server_cluster_id, cpb.getNodesList());
}
- List<Frontend> currentFes = Env.getCurrentEnv().getFrontends(FrontendNodeType.OBSERVER);
+ List<Frontend> currentFollowers = Env.getCurrentEnv().getFrontends(FrontendNodeType.FOLLOWER);
+ List<Frontend> currentObservers = Env.getCurrentEnv().getFrontends(FrontendNodeType.OBSERVER);
+ currentFollowers.addAll(currentObservers);
+ List<Frontend> currentFes = new ArrayList<>(currentFollowers.stream().collect(Collectors.toMap(
+ fe -> fe.getHost() + ":" + fe.getEditLogPort(),
+ fe -> fe,
+ (existing, replacement) -> existing
+ )).values());
List<Frontend> toAdd = new ArrayList<>();
List<Frontend> toDel = new ArrayList<>();
List<Cloud.NodeInfoPB> expectedFes = cpb.getNodesList();
diffNodes(toAdd, toDel, () -> {
+ // memory
Map<String, Frontend> currentMap = new HashMap<>();
String selfNode = Env.getCurrentEnv().getSelfNode().getIdent();
for (Frontend fe : currentFes) {
@@ -383,10 +391,13 @@ public class CloudClusterChecker extends MasterDaemon {
if (selfNode.equals(endpoint)) {
continue;
}
+ // add type to map key, for diff
+ endpoint = endpoint + "_" + fe.getRole();
currentMap.put(endpoint, fe);
}
return currentMap;
}, () -> {
+ // meta service
Map<String, Frontend> nodeMap = new HashMap<>();
String selfNode = Env.getCurrentEnv().getSelfNode().getIdent();
for (Cloud.NodeInfoPB node : expectedFes) {
@@ -399,9 +410,18 @@ public class CloudClusterChecker extends MasterDaemon {
if (selfNode.equals(endpoint)) {
continue;
}
- Frontend fe = new Frontend(FrontendNodeType.OBSERVER,
+ Cloud.NodeInfoPB.NodeType type = node.getNodeType();
+ // ATTN: just allow to add follower or observer
+ if (Cloud.NodeInfoPB.NodeType.FE_MASTER.equals(type)) {
+ LOG.warn("impossible !!!, get fe node {} type equel master from ms", node);
+ }
+ FrontendNodeType role = type == Cloud.NodeInfoPB.NodeType.FE_FOLLOWER
+ ? FrontendNodeType.FOLLOWER : FrontendNodeType.OBSERVER;
+ Frontend fe = new Frontend(role,
CloudEnv.genFeNodeNameFromMeta(host, node.getEditLogPort(),
node.getCtime() * 1000), host, node.getEditLogPort());
+ // add type to map key, for diff
+ endpoint = endpoint + "_" + fe.getRole();
nodeMap.put(endpoint, fe);
}
return nodeMap;
@@ -421,7 +441,7 @@ public class CloudClusterChecker extends MasterDaemon {
}
}
- private void getCloudBackends() {
+ private void checkCloudBackends() {
Map<String, List<Backend>> clusterIdToBackend = cloudSystemInfoService.getCloudClusterIdToBackend();
//rpc to ms, to get mysql user can use cluster_id
// NOTE: rpc args all empty, use cluster_unique_id to get a instance's all cluster info.
@@ -474,7 +494,7 @@ public class CloudClusterChecker extends MasterDaemon {
for (Map.Entry<String, String> entry : clusterNameToId.entrySet()) {
int aliveNum = 0;
List<Backend> bes = clusterIdToBackend.get(entry.getValue());
- if (bes == null || bes.size() == 0) {
+ if (bes == null || bes.isEmpty()) {
LOG.info("cant get be nodes by cluster {}, bes {}", entry, bes);
continue;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudEnv.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudEnv.java
index e52ea292932..415b3b41743 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudEnv.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudEnv.java
@@ -57,6 +57,7 @@ import java.io.DataInputStream;
import java.io.File;
import java.io.IOException;
import java.net.HttpURLConnection;
+import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.Optional;
@@ -149,12 +150,24 @@ public class CloudEnv extends Env {
.stream().filter(NodeInfoPB::hasNodeType).collect(Collectors.toList());
helperNodes.clear();
- helperNodes.addAll(allNodes.stream()
- .filter(nodeInfoPB -> nodeInfoPB.getNodeType() == NodeInfoPB.NodeType.FE_MASTER)
- .map(nodeInfoPB -> new HostInfo(
- Config.enable_fqdn_mode ? nodeInfoPB.getHost() : nodeInfoPB.getIp(), nodeInfoPB.getEditLogPort()))
- .collect(Collectors.toList()));
- // check only have one master node.
+ if (allNodes.stream().anyMatch(n -> n.getNodeType() == NodeInfoPB.NodeType.FE_FOLLOWER)) {
+ // multi followers mode, select first
+ Optional<HostInfo> helperNode = allNodes.stream()
+ .filter(nodeInfoPB -> nodeInfoPB.getNodeType() == NodeInfoPB.NodeType.FE_FOLLOWER)
+ .map(nodeInfoPB -> new HostInfo(
+ Config.enable_fqdn_mode ? nodeInfoPB.getHost() : nodeInfoPB.getIp(), nodeInfoPB.getEditLogPort()))
+ .min(Comparator.comparing(HostInfo::getHost));
+ helperNode.ifPresent(hostInfo -> helperNodes.add(hostInfo));
+ } else {
+ // master observers mode
+ // helper node select follower's first, just one
+ helperNodes.addAll(allNodes.stream()
+ .filter(nodeInfoPB -> nodeInfoPB.getNodeType() == NodeInfoPB.NodeType.FE_MASTER)
+ .map(nodeInfoPB -> new HostInfo(
+ Config.enable_fqdn_mode ? nodeInfoPB.getHost() : nodeInfoPB.getIp(), nodeInfoPB.getEditLogPort()))
+ .collect(Collectors.toList()));
+ // check only have one master node.
+ }
Preconditions.checkState(helperNodes.size() == 1);
Optional<NodeInfoPB> local = allNodes.stream().filter(n -> ((Config.enable_fqdn_mode ? n.getHost() : n.getIp())
@@ -191,6 +204,7 @@ public class CloudEnv extends Env {
}
LOG.info("current fe's role is {}", type == NodeInfoPB.NodeType.FE_MASTER ? "MASTER" :
+ type == NodeInfoPB.NodeType.FE_FOLLOWER ? "FOLLOWER" :
type == NodeInfoPB.NodeType.FE_OBSERVER ? "OBSERVER" : "UNKNOWN");
if (type == NodeInfoPB.NodeType.UNKNOWN) {
LOG.warn("type current not support, please check it");
diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletRebalancer.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletRebalancer.java
index fc580c4fc7e..6f4534c4b08 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletRebalancer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletRebalancer.java
@@ -1068,7 +1068,7 @@ public class CloudTabletRebalancer extends MasterDaemon {
newInfo.setPartitionId(partitionId);
newInfo.setIndexId(indexId);
newInfo.setClusterId(clusterId);
- // APPR: in unprotectUpdateCloudReplica, use batch must set tabletId = -1
+ // ATTN: in unprotectUpdateCloudReplica, use batch must set tabletId = -1
newInfo.setTabletId(-1);
rets.add(newInfo);
});
diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/system/CloudSystemInfoService.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/system/CloudSystemInfoService.java
index 48728efb003..202d576e3bf 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/cloud/system/CloudSystemInfoService.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/system/CloudSystemInfoService.java
@@ -282,20 +282,29 @@ public class CloudSystemInfoService extends SystemInfoService {
LOG.debug("updateCloudFrontends toAdd={} toDel={}", toAdd, toDel);
}
String masterIp = Env.getCurrentEnv().getMasterHost();
- for (Frontend fe : toAdd) {
+ for (Frontend fe : toDel) {
if (masterIp.equals(fe.getHost())) {
continue;
}
- Env.getCurrentEnv().addFrontend(FrontendNodeType.OBSERVER,
- fe.getHost(), fe.getEditLogPort(), fe.getNodeName());
- LOG.info("added cloud frontend={} ", fe);
+ try {
+ Env.getCurrentEnv().dropFrontend(fe.getRole(), fe.getHost(), fe.getEditLogPort());
+ LOG.info("dropped cloud frontend={} ", fe);
+ } catch (DdlException e) {
+ LOG.warn("failed to drop cloud frontend={} ", fe);
+ }
}
- for (Frontend fe : toDel) {
+
+ for (Frontend fe : toAdd) {
if (masterIp.equals(fe.getHost())) {
continue;
}
- Env.getCurrentEnv().dropFrontend(FrontendNodeType.OBSERVER, fe.getHost(), fe.getEditLogPort());
- LOG.info("dropped cloud frontend={} ", fe);
+ try {
+ Env.getCurrentEnv().addFrontend(fe.getRole(),
+ fe.getHost(), fe.getEditLogPort(), fe.getNodeName());
+ LOG.info("added cloud frontend={} ", fe);
+ } catch (DdlException e) {
+ LOG.warn("failed to add cloud frontend={} ", fe);
+ }
}
}
diff --git a/gensrc/proto/cloud.proto b/gensrc/proto/cloud.proto
index 87948fdeea9..cad16c24910 100644
--- a/gensrc/proto/cloud.proto
+++ b/gensrc/proto/cloud.proto
@@ -140,8 +140,10 @@ message ClusterPB {
message NodeInfoPB {
enum NodeType {
UNKNOWN = 0;
+ // lagacy logic for one-master-multi-observer mode
FE_MASTER = 1;
FE_OBSERVER = 2;
+ FE_FOLLOWER = 3;
}
optional string cloud_unique_id = 1;
optional string name = 2;
diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
index 247c699fcf9..9272c818199 100644
--- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
+++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
@@ -1765,6 +1765,86 @@ class Suite implements GroovyInterceptable {
}
}
+ def get_instance = { MetaService ms=null ->
+ def jsonOutput = new JsonOutput()
+
+ def get_instance_api = { check_func ->
+ httpTest {
+ op "get"
+ if (ms) {
+ endpoint ms.host+':'+ms.httpPort
+ } else {
+ endpoint context.config.metaServiceHttpAddress
+ }
+ uri "/MetaService/http/get_instance?token=${token}&instance_id=${instance_id}"
+ check check_func
+ }
+ }
+
+ def json
+ get_instance_api.call() {
+ respCode, body ->
+ log.info("get instance resp: ${body} ${respCode}".toString())
+ json = parseJson(body)
+ assertTrue(json.code.equalsIgnoreCase("OK"))
+ }
+ json.result
+ }
+
+
+ def drop_node = { unique_id, ip, bePort=0, fePort=0, nodeType,
+ cluster_name, cluster_id, MetaService ms=null ->
+ def jsonOutput = new JsonOutput()
+ def type
+ if (fePort != 0) {
+ type = "SQL"
+ } else if (bePort != 0) {
+ type = "COMPUTE"
+ }
+ def clusterInfo = [
+ type: type,
+ cluster_name : cluster_name,
+ cluster_id : cluster_id,
+ nodes: [
+ [
+ cloud_unique_id: unique_id,
+ ip: ip,
+ node_type: nodeType
+ ],
+ ]
+ ]
+ if (bePort != 0) {
+ // drop be
+ clusterInfo['nodes'][0]['heartbeat_port'] = bePort
+ } else if (fePort != 0) {
+ // drop fe
+ clusterInfo['nodes'][0]['edit_log_port'] = fePort
+ }
+ def map = [instance_id: "${instance_id}", cluster: clusterInfo]
+ def js = jsonOutput.toJson(map)
+ log.info("drop node req: ${js} ".toString())
+
+ def drop_node_api = { request_body, check_func ->
+ httpTest {
+ if (ms) {
+ endpoint ms.host+':'+ms.httpPort
+ } else {
+ endpoint context.config.metaServiceHttpAddress
+ }
+ uri "/MetaService/http/drop_node?token=${token}"
+ body request_body
+ check check_func
+ }
+ }
+
+ drop_node_api.call(js) {
+ respCode, body ->
+ log.info("drop node resp: ${body} ${respCode}".toString())
+ def json = parseJson(body)
+ assertTrue(json.code.equalsIgnoreCase("OK"))
+ }
+ }
+
def d_node = { be_unique_id, ip, port, cluster_name, cluster_id ->
def jsonOutput = new JsonOutput()
def clusterInfo = [
diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteCluster.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteCluster.groovy
index c36deac2e68..adecc71f464 100644
--- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteCluster.groovy
+++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteCluster.groovy
@@ -51,6 +51,12 @@ class ClusterOptions {
// 3. cloudMode = null, create both cloud and none-cloud cluster, depend on the running pipeline mode.
Boolean cloudMode = false
+ // in cloud mode, deployment methods are divided into
+ // 1. master - multi observers
+ // 2. mutli followers - multi observers
+ // default use 1
+ Boolean useFollowersMode = false
+
// when cloudMode = true/false, but the running pipeline is diff with cloudMode,
// skip run this docker test or not.
boolean skipRunWhenPipelineDiff = true
@@ -248,6 +254,11 @@ class SuiteCluster {
if (isCloud) {
cmd += ['--cloud']
}
+
+ if (isCloud && options.useFollowersMode) {
+ cmd += ['--fe-follower']
+ }
+
cmd += ['--wait-timeout', String.valueOf(180)]
runCmd(cmd.join(' '), -1)
@@ -377,8 +388,8 @@ class SuiteCluster {
return new Tuple4(frontends, backends, metaservices, recyclers)
}
- List<Integer> addFrontend(int num) throws Exception {
- def result = add(num, 0, null)
+ List<Integer> addFrontend(int num, boolean followerMode=false) throws Exception {
+ def result = add(num, 0, null, followerMode)
return result.first
}
@@ -387,14 +398,18 @@ class SuiteCluster {
return result.second
}
- // APPR: clusterName just used for cloud mode, 1 cluster has n bes
- Tuple2<List<Integer>, List<Integer>> add(int feNum, int beNum, String clusterName) throws Exception {
+ // ATTN: clusterName just used for cloud mode, 1 cluster has n bes
+ // ATTN: followerMode just used for cloud mode
+ Tuple2<List<Integer>, List<Integer>> add(int feNum, int beNum, String clusterName, boolean followerMode=false) throws Exception {
assert feNum > 0 || beNum > 0
def sb = new StringBuilder()
sb.append('up ' + name + ' ')
if (feNum > 0) {
sb.append('--add-fe-num ' + feNum + ' ')
+ if (followerMode) {
+ sb.append('--fe-follower' + ' ')
+ }
}
if (beNum > 0) {
sb.append('--add-be-num ' + beNum + ' ')
diff --git a/regression-test/suites/cloud_p0/multi_cluster/test_multi_follower.groovy b/regression-test/suites/cloud_p0/multi_cluster/test_multi_follower.groovy
new file mode 100644
index 00000000000..864655a5344
--- /dev/null
+++ b/regression-test/suites/cloud_p0/multi_cluster/test_multi_follower.groovy
@@ -0,0 +1,176 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.apache.doris.regression.suite.ClusterOptions
+import groovy.json.JsonSlurper
+import org.awaitility.Awaitility
+import static java.util.concurrent.TimeUnit.SECONDS
+
+suite('test_multi_followr_in_cloud', 'multi_cluster, docker') {
+ if (!isCloudMode()) {
+ return
+ }
+ def options = new ClusterOptions()
+ options.feConfigs += [
+ 'cloud_cluster_check_interval_second=1',
+ 'sys_log_verbose_modules=org',
+ 'heartbeat_interval_second=1'
+ ]
+ options.setFeNum(3)
+ options.setBeNum(1)
+ options.cloudMode = true
+ options.connectToFollower = true
+ options.useFollowersMode = true
+
+ docker(options) {
+ def f = "FOLLOWER"
+ def o = "OBSERVER"
+ def check = { int feNum, Closure checkFunc ->
+ def ret = sql_return_maparray """SHOW FRONTENDS"""
+ assertEquals(feNum, ret.size())
+ Boolean hasMaster = false
+ checkFunc.call(ret)
+ ret.each { row ->
+ if (Boolean.parseBoolean(row.IsMaster)) {
+ hasMaster = true
+ }
+ }
+ assertTrue(hasMaster)
+ }
+
+ def transferType = { String old ->
+ if (old.contains(f)) {
+ return "FE_FOLLOWER"
+ } else if (old.contains(o)) {
+ return "FE_OBSERVER"
+ }
+ }
+
+ check(3) { ret ->
+ ret.each {
+ assertTrue(it.Role.contains(f))
+ }
+ }
+ // get fe clusterName
+ def result = sql_return_maparray """ADMIN SHOW FRONTEND CONFIG LIKE '%cloud_sql_server_cluster_name%'"""
+ def feClusterName = result.Value[0]
+ result = sql_return_maparray """ADMIN SHOW FRONTEND CONFIG LIKE '%cloud_sql_server_cluster_id%'"""
+ def feClusterId = result.Value[0]
+ log.info("fe clusterName: {}, clusterId: {} ", feClusterName, feClusterId)
+ def toDropIP
+ def toDropPort
+ def toDropType
+ def toDropUniqueId
+ // add new follower
+ cluster.addFrontend(1, true)
+ dockerAwaitUntil(5) {
+ def ret = sql """SHOW FRONTENDS"""
+ log.info("show frontends: {}", ret)
+ ret.size() == 4
+ }
+ check(4) { def ret ->
+ ret.each {
+ assertTrue(it.Role.contains(f))
+ if (!Boolean.parseBoolean(it.IsMaster)) {
+ toDropIP = it.Host
+ toDropPort = it.EditLogPort
+ toDropType = transferType(it.Role)
+ }
+ }
+ }
+ log.info("ip: {}, port: {}, type: {}, uniqueId: {}", toDropIP, toDropPort, toDropType, toDropUniqueId)
+ def ms = cluster.getAllMetaservices().get(0)
+ logger.info("ms addr={}, port={}", ms.host, ms.httpPort)
+ // drop a follwer
+ def findToDropUniqueId = { clusterId, hostIP, metaServices ->
+ ret = get_instance(metaServices)
+ def toDropCluster = ret.clusters.find {
+ it.cluster_id.contains(clusterId)
+ }
+ log.info("toDropCluster: {}", toDropCluster)
+ def toDropNode = toDropCluster.nodes.find {
+ it.ip.contains(hostIP)
+ }
+ log.info("toDropNode: {}", toDropNode)
+ assertNotNull(toDropCluster)
+ assertNotNull(toDropNode)
+ toDropNode.cloud_unique_id
+ }
+
+ toDropUniqueId = findToDropUniqueId.call(feClusterId, toDropIP, ms)
+ log.info("to Drop1 ip: {}, port: {}, type: {}, uniqueId: {}", toDropIP, toDropPort, toDropType, toDropUniqueId)
+
+ drop_node(toDropUniqueId, toDropIP, 0,
+ toDropPort, toDropType, feClusterName, feClusterId, ms)
+
+ dockerAwaitUntil(5) {
+ def ret = sql """SHOW FRONTENDS"""
+ log.info("show frontends: {}", ret)
+ ret.size() == 3
+ }
+
+ check(3) { def ret ->
+ ret.each {
+ assertTrue(it.Role.contains(f))
+ }
+ }
+
+ // add a observer
+ cluster.addFrontend(1)
+ check(4) { def ret ->
+ int observerNum = 0
+ int followerNum = 0
+ ret.each {
+ if (it.Role.contains(o)) {
+ observerNum++;
+ toDropIP = it.Host
+ toDropPort = it.EditLogPort
+ toDropType = transferType(it.Role)
+ } else if (it.Role.contains(f)) {
+ followerNum++;
+ }
+ }
+ assertEquals(1, observerNum)
+ assertEquals(3, followerNum)
+ }
+
+ toDropUniqueId = findToDropUniqueId.call(feClusterId, toDropIP, ms)
+ // drop observer
+ drop_node(toDropUniqueId, toDropIP, 0,
+ toDropPort, toDropType, feClusterName, feClusterId, ms)
+
+ dockerAwaitUntil(5) {
+ def ret = sql """SHOW FRONTENDS"""
+ log.info("show frontends: {}", ret)
+ ret.size() == 3
+ }
+
+ check(3) { def ret ->
+ int observerNum = 0
+ int followerNum = 0
+ ret.each {
+ if (it.Role.contains(o)) {
+ observerNum++;
+ } else if (it.Role.contains(f)){
+ followerNum++;
+ }
+ }
+ assertEquals(0, observerNum)
+ assertEquals(3, followerNum)
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 05/11: [Enhancement](Group commit) Remove the redundant `iscloud` parameter (#40120)
Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gavinchou pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit e03b6d1faafed4e18520026ee6cdc78433047fb9
Author: abmdocrt <Yu...@gmail.com>
AuthorDate: Tue Sep 3 10:54:24 2024 +0800
[Enhancement](Group commit) Remove the redundant `iscloud` parameter (#40120)
`isCloud` parameter is redundant because we can use `Config.isCloud()`
to replace it.
---
.../doris/cloud/planner/CloudGroupCommitPlanner.java | 14 --------------
.../main/java/org/apache/doris/httpv2/rest/LoadAction.java | 11 +++++------
.../java/org/apache/doris/load/GroupCommitManager.java | 10 +++++-----
.../commands/insert/OlapGroupCommitInsertExecutor.java | 2 +-
.../java/org/apache/doris/planner/GroupCommitPlanner.java | 2 +-
.../main/java/org/apache/doris/qe/MasterOpExecutor.java | 7 +++----
.../java/org/apache/doris/service/FrontendServiceImpl.java | 2 +-
gensrc/thrift/FrontendService.thrift | 1 -
8 files changed, 16 insertions(+), 33 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/planner/CloudGroupCommitPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/planner/CloudGroupCommitPlanner.java
index 782f78e6bc4..0388ca5c5d6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/cloud/planner/CloudGroupCommitPlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/planner/CloudGroupCommitPlanner.java
@@ -18,13 +18,9 @@
package org.apache.doris.cloud.planner;
import org.apache.doris.catalog.Database;
-import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.OlapTable;
-import org.apache.doris.common.DdlException;
-import org.apache.doris.common.LoadException;
import org.apache.doris.common.UserException;
import org.apache.doris.planner.GroupCommitPlanner;
-import org.apache.doris.qe.ConnectContext;
import org.apache.doris.thrift.TUniqueId;
import org.apache.logging.log4j.LogManager;
@@ -41,14 +37,4 @@ public class CloudGroupCommitPlanner extends GroupCommitPlanner {
throws UserException, TException {
super(db, table, targetColumnNames, queryId, groupCommit);
}
-
- @Override
- protected void selectBackends(ConnectContext ctx) throws DdlException {
- try {
- backend = Env.getCurrentEnv().getGroupCommitManager()
- .selectBackendForGroupCommit(this.table.getId(), ctx, true);
- } catch (LoadException e) {
- throw new DdlException("No suitable backend");
- }
- }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/LoadAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/LoadAction.java
index c9681eb7849..1f1c5d965c2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/LoadAction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/LoadAction.java
@@ -434,7 +434,7 @@ public class LoadAction extends RestBaseController {
throw new LoadException(SystemInfoService.NO_BACKEND_LOAD_AVAILABLE_MSG + ", policy: " + policy);
}
if (groupCommit) {
- backend = selectBackendForGroupCommit("", request, tableId, false);
+ backend = selectBackendForGroupCommit("", request, tableId);
} else {
backend = Env.getCurrentSystemInfo().getBackend(backendIds.get(0));
}
@@ -449,7 +449,7 @@ public class LoadAction extends RestBaseController {
throws LoadException {
Backend backend = null;
if (groupCommit) {
- backend = selectBackendForGroupCommit(clusterName, req, tableId, true);
+ backend = selectBackendForGroupCommit(clusterName, req, tableId);
} else {
backend = StreamLoadHandler.selectBackend(clusterName);
}
@@ -665,8 +665,7 @@ public class LoadAction extends RestBaseController {
return headers.toString();
}
- private Backend selectBackendForGroupCommit(String clusterName, HttpServletRequest req, long tableId,
- boolean isCloud)
+ private Backend selectBackendForGroupCommit(String clusterName, HttpServletRequest req, long tableId)
throws LoadException {
ConnectContext ctx = new ConnectContext();
ctx.setEnv(Env.getCurrentEnv());
@@ -676,14 +675,14 @@ public class LoadAction extends RestBaseController {
// TMasterOpRequest(FrontendService.thrift)
ctx.setQualifiedUser(Auth.ADMIN_USER);
ctx.setThreadLocalInfo();
- if (isCloud) {
+ if (Config.isCloudMode()) {
ctx.setCloudCluster(clusterName);
}
Backend backend = null;
try {
backend = Env.getCurrentEnv().getGroupCommitManager()
- .selectBackendForGroupCommit(tableId, ctx, isCloud);
+ .selectBackendForGroupCommit(tableId, ctx);
} catch (DdlException e) {
throw new LoadException(e.getMessage(), e);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/GroupCommitManager.java b/fe/fe-core/src/main/java/org/apache/doris/load/GroupCommitManager.java
index e4dd77e3e20..717cf0c6103 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/GroupCommitManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/GroupCommitManager.java
@@ -186,14 +186,14 @@ public class GroupCommitManager {
return size;
}
- public Backend selectBackendForGroupCommit(long tableId, ConnectContext context, boolean isCloud)
+ public Backend selectBackendForGroupCommit(long tableId, ConnectContext context)
throws LoadException, DdlException {
// If a group commit request is sent to the follower FE, we will send this request to the master FE. master FE
// can select a BE and return this BE id to follower FE.
if (!Env.getCurrentEnv().isMaster()) {
try {
long backendId = new MasterOpExecutor(context)
- .getGroupCommitLoadBeId(tableId, context.getCloudCluster(), isCloud);
+ .getGroupCommitLoadBeId(tableId, context.getCloudCluster());
return Env.getCurrentSystemInfo().getBackend(backendId);
} catch (Exception e) {
throw new LoadException(e.getMessage());
@@ -201,11 +201,11 @@ public class GroupCommitManager {
} else {
// Master FE will select BE by itself.
return Env.getCurrentSystemInfo()
- .getBackend(selectBackendForGroupCommitInternal(tableId, context.getCloudCluster(), isCloud));
+ .getBackend(selectBackendForGroupCommitInternal(tableId, context.getCloudCluster()));
}
}
- public long selectBackendForGroupCommitInternal(long tableId, String cluster, boolean isCloud)
+ public long selectBackendForGroupCommitInternal(long tableId, String cluster)
throws LoadException, DdlException {
// Understanding Group Commit and Backend Selection Logic
//
@@ -237,7 +237,7 @@ public class GroupCommitManager {
// a BE is chosen at random. This BE is then recorded along with the mapping of table A and its load level.
// This approach ensures that group commits can effectively batch data together
// while managing the load on each BE efficiently.
- return isCloud ? selectBackendForCloudGroupCommitInternal(tableId, cluster)
+ return Config.isCloudMode() ? selectBackendForCloudGroupCommitInternal(tableId, cluster)
: selectBackendForLocalGroupCommitInternal(tableId);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/OlapGroupCommitInsertExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/OlapGroupCommitInsertExecutor.java
index a44528e93b3..239328ce93d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/OlapGroupCommitInsertExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/OlapGroupCommitInsertExecutor.java
@@ -120,7 +120,7 @@ public class OlapGroupCommitInsertExecutor extends OlapInsertExecutor {
}
try {
this.coordinator.setGroupCommitBe(Env.getCurrentEnv().getGroupCommitManager()
- .selectBackendForGroupCommit(table.getId(), ctx, false));
+ .selectBackendForGroupCommit(table.getId(), ctx));
} catch (LoadException | DdlException e) {
throw new RuntimeException(e);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/GroupCommitPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/GroupCommitPlanner.java
index bc3759d4e62..234f8e99a88 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/GroupCommitPlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/GroupCommitPlanner.java
@@ -148,7 +148,7 @@ public class GroupCommitPlanner {
protected void selectBackends(ConnectContext ctx) throws DdlException {
try {
backend = Env.getCurrentEnv().getGroupCommitManager()
- .selectBackendForGroupCommit(this.table.getId(), ctx, false);
+ .selectBackendForGroupCommit(this.table.getId(), ctx);
} catch (LoadException e) {
throw new DdlException("No suitable backend");
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/MasterOpExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/MasterOpExecutor.java
index eaab01df556..edc174d9d2f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/MasterOpExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/MasterOpExecutor.java
@@ -102,8 +102,8 @@ public class MasterOpExecutor {
waitOnReplaying();
}
- public long getGroupCommitLoadBeId(long tableId, String cluster, boolean isCloud) throws Exception {
- result = forward(buildGetGroupCommitLoadBeIdParmas(tableId, cluster, isCloud));
+ public long getGroupCommitLoadBeId(long tableId, String cluster) throws Exception {
+ result = forward(buildGetGroupCommitLoadBeIdParmas(tableId, cluster));
waitOnReplaying();
return result.groupCommitLoadBeId;
}
@@ -244,12 +244,11 @@ public class MasterOpExecutor {
return params;
}
- private TMasterOpRequest buildGetGroupCommitLoadBeIdParmas(long tableId, String cluster, boolean isCloud) {
+ private TMasterOpRequest buildGetGroupCommitLoadBeIdParmas(long tableId, String cluster) {
final TGroupCommitInfo groupCommitParams = new TGroupCommitInfo();
groupCommitParams.setGetGroupCommitLoadBeId(true);
groupCommitParams.setGroupCommitLoadTableId(tableId);
groupCommitParams.setCluster(cluster);
- groupCommitParams.setIsCloud(isCloud);
final TMasterOpRequest params = new TMasterOpRequest();
// node ident
diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
index f64f55a47fe..2d65001cbb5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
@@ -1041,7 +1041,7 @@ public class FrontendServiceImpl implements FrontendService.Iface {
final TMasterOpResult result = new TMasterOpResult();
try {
result.setGroupCommitLoadBeId(Env.getCurrentEnv().getGroupCommitManager()
- .selectBackendForGroupCommitInternal(info.groupCommitLoadTableId, info.cluster, info.isCloud));
+ .selectBackendForGroupCommitInternal(info.groupCommitLoadTableId, info.cluster));
} catch (LoadException | DdlException e) {
throw new TException(e.getMessage());
}
diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift
index e6152bd2aa4..89333d60665 100644
--- a/gensrc/thrift/FrontendService.thrift
+++ b/gensrc/thrift/FrontendService.thrift
@@ -553,7 +553,6 @@ struct TGroupCommitInfo{
1: optional bool getGroupCommitLoadBeId
2: optional i64 groupCommitLoadTableId
3: optional string cluster
- 4: optional bool isCloud
5: optional bool updateLoadData
6: optional i64 tableId
7: optional i64 receiveData
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org