You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by "liaoxin01 (via GitHub)" <gi...@apache.org> on 2024/02/23 06:53:09 UTC
[PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
liaoxin01 opened a new pull request, #31317:
URL: https://github.com/apache/doris/pull/31317
## Proposed changes
Issue Number: close #xxx
<!--Describe your changes.-->
## Further comments
If this is a relatively large or complex change, kick off the discussion at [dev@doris.apache.org](mailto:dev@doris.apache.org) by explaining why you chose the solution you did and what alternatives you considered, etc...
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "github-actions[bot] (via GitHub)" <gi...@apache.org>.
github-actions[bot] commented on code in PR #31317:
URL: https://github.com/apache/doris/pull/31317#discussion_r1505663785
##########
be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp:
##########
@@ -0,0 +1,165 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "cloud/cloud_engine_calc_delete_bitmap_task.h"
+
+#include <memory>
+
+#include "cloud/cloud_meta_mgr.h"
+#include "cloud/cloud_tablet.h"
+#include "common/status.h"
+#include "olap/base_tablet.h"
+#include "olap/olap_common.h"
+#include "olap/rowset/rowset.h"
+#include "olap/tablet_fwd.h"
+#include "olap/tablet_meta.h"
+#include "olap/txn_manager.h"
+#include "olap/utils.h"
+
+namespace doris {
+
+CloudEngineCalcDeleteBitmapTask::CloudEngineCalcDeleteBitmapTask(
+ CloudStorageEngine& engine, const TCalcDeleteBitmapRequest& cal_delete_bitmap_req,
+ std::vector<TTabletId>* error_tablet_ids, std::vector<TTabletId>* succ_tablet_ids)
+ : _engine(engine),
+ _cal_delete_bitmap_req(cal_delete_bitmap_req),
+ _error_tablet_ids(error_tablet_ids),
+ _succ_tablet_ids(succ_tablet_ids) {}
+
+void CloudEngineCalcDeleteBitmapTask::add_error_tablet_id(int64_t tablet_id, const Status& err) {
+ std::lock_guard<std::mutex> lck(_mutex);
+ _error_tablet_ids->push_back(tablet_id);
+ if (_res.ok() || _res.is<ErrorCode::DELETE_BITMAP_LOCK_ERROR>()) {
+ _res = err;
+ }
+}
+
+void CloudEngineCalcDeleteBitmapTask::add_succ_tablet_id(int64_t tablet_id) {
+ std::lock_guard<std::mutex> lck(_mutex);
+ _succ_tablet_ids->push_back(tablet_id);
+}
+
+Status CloudEngineCalcDeleteBitmapTask::execute() {
+ int64_t transaction_id = _cal_delete_bitmap_req.transaction_id;
+ OlapStopWatch watch;
+ VLOG_NOTICE << "begin to calculate delete bitmap. transaction_id=" << transaction_id;
+ std::unique_ptr<ThreadPoolToken> token =
+ _engine.calc_tablet_delete_bitmap_task_thread_pool()->new_token(
+ ThreadPool::ExecutionMode::CONCURRENT);
+
+ for (const auto& partition : _cal_delete_bitmap_req.partitions) {
+ int64_t version = partition.version;
+ for (auto tablet_id : partition.tablet_ids) {
+ auto base_tablet = DORIS_TRY(_engine.get_tablet(tablet_id));
+ std::shared_ptr<CloudTablet> tablet =
+ std::dynamic_pointer_cast<CloudTablet>(base_tablet);
+ if (tablet == nullptr) {
+ LOG(WARNING) << "can't get tablet when calculate delete bitmap. tablet_id="
+ << tablet_id;
+ _error_tablet_ids->push_back(tablet_id);
+ _res = Status::Error<ErrorCode::PUSH_TABLE_NOT_EXIST>(
+ "can't get tablet when calculate delete bitmap. tablet_id={}", tablet_id);
+ break;
+ }
+
+ Status st = tablet->sync_rowsets();
+ if (!st.ok() && !st.is<ErrorCode::INVALID_TABLET_STATE>()) {
+ return st;
+ }
+ if (st.is<ErrorCode::INVALID_TABLET_STATE>()) [[unlikely]] {
+ add_succ_tablet_id(tablet->tablet_id());
+ LOG(INFO)
+ << "tablet is under alter process, delete bitmap will be calculated later, "
+ "tablet_id: "
+ << tablet->tablet_id() << " txn_id: " << transaction_id
+ << ", request_version=" << version;
+ continue;
+ }
+ int64_t max_version = tablet->max_version_unlocked();
+ if (version != max_version + 1) {
+ _error_tablet_ids->push_back(tablet_id);
+ _res = Status::Error<ErrorCode::DELETE_BITMAP_LOCK_ERROR, false>(
+ "version not continuous");
+ LOG(WARNING) << "version not continuous, current max version=" << max_version
+ << ", request_version=" << version
+ << " tablet_id=" << tablet->tablet_id();
+ break;
+ }
+
+ auto tablet_calc_delete_bitmap_ptr = std::make_shared<CloudTabletCalcDeleteBitmapTask>(
+ _engine, this, tablet, transaction_id, version);
+ auto submit_st = token->submit_func([=]() { tablet_calc_delete_bitmap_ptr->handle(); });
+ CHECK(submit_st.ok());
+ }
+ }
+ // wait for all finished
+ token->wait();
+
+ LOG(INFO) << "finish to calculate delete bitmap on transaction."
+ << "transaction_id=" << transaction_id << ", cost(us): " << watch.get_elapse_time_us()
+ << ", error_tablet_size=" << _error_tablet_ids->size()
+ << ", res=" << _res.to_string();
+ return _res;
+}
+
+CloudTabletCalcDeleteBitmapTask::CloudTabletCalcDeleteBitmapTask(
+ CloudStorageEngine& engine, CloudEngineCalcDeleteBitmapTask* engine_task,
+ std::shared_ptr<CloudTablet> tablet, int64_t transaction_id, int64_t version)
+ : _engine(engine),
+ _engine_calc_delete_bitmap_task(engine_task),
+ _tablet(tablet),
+ _transaction_id(transaction_id),
+ _version(version) {}
+
+void CloudTabletCalcDeleteBitmapTask::handle() {
+ RowsetSharedPtr rowset;
Review Comment:
warning: method 'handle' can be made const [readability-make-member-function-const]
```suggestion
void CloudTabletCalcDeleteBitmapTask::handle() const {
```
be/src/cloud/cloud_engine_calc_delete_bitmap_task.h:38:
```diff
- void handle();
+ void handle() const;
```
##########
be/src/olap/base_tablet.cpp:
##########
@@ -1059,4 +1063,181 @@ Status BaseTablet::_capture_consistent_rowsets_unlocked(
return Status::OK();
}
+Status BaseTablet::check_delete_bitmap_correctness(DeleteBitmapPtr delete_bitmap,
+ int64_t max_version, int64_t txn_id,
+ const RowsetIdUnorderedSet& rowset_ids,
+ std::vector<RowsetSharedPtr>* rowsets) {
+ RowsetIdUnorderedSet missing_ids;
+ for (const auto& rowsetid : rowset_ids) {
+ if (!delete_bitmap->delete_bitmap.contains({rowsetid, DeleteBitmap::INVALID_SEGMENT_ID,
+ DeleteBitmap::TEMP_VERSION_COMMON})) {
+ missing_ids.insert(rowsetid);
+ }
+ }
+
+ if (!missing_ids.empty()) {
+ LOG(WARNING) << "[txn_id:" << txn_id << "][tablet_id:" << tablet_id()
+ << "][max_version: " << max_version
+ << "] check delete bitmap correctness failed!";
+ rapidjson::Document root;
+ root.SetObject();
+ rapidjson::Document required_rowsets_arr;
+ required_rowsets_arr.SetArray();
+ rapidjson::Document missing_rowsets_arr;
+ missing_rowsets_arr.SetArray();
+
+ if (rowsets != nullptr) {
+ for (const auto& rowset : *rowsets) {
+ rapidjson::Value value;
+ std::string version_str = rowset->get_rowset_info_str();
+ value.SetString(version_str.c_str(), version_str.length(),
+ required_rowsets_arr.GetAllocator());
+ required_rowsets_arr.PushBack(value, required_rowsets_arr.GetAllocator());
+ }
+ } else {
+ std::vector<RowsetSharedPtr> rowsets;
+ {
+ std::shared_lock meta_rlock(_meta_lock);
+ rowsets = get_rowset_by_ids(&rowset_ids);
+ }
+ for (const auto& rowset : rowsets) {
+ rapidjson::Value value;
+ std::string version_str = rowset->get_rowset_info_str();
+ value.SetString(version_str.c_str(), version_str.length(),
+ required_rowsets_arr.GetAllocator());
+ required_rowsets_arr.PushBack(value, required_rowsets_arr.GetAllocator());
+ }
+ }
+ for (const auto& missing_rowset_id : missing_ids) {
+ rapidjson::Value miss_value;
+ std::string rowset_id_str = missing_rowset_id.to_string();
+ miss_value.SetString(rowset_id_str.c_str(), rowset_id_str.length(),
+ missing_rowsets_arr.GetAllocator());
+ missing_rowsets_arr.PushBack(miss_value, missing_rowsets_arr.GetAllocator());
+ }
+
+ root.AddMember("required_rowsets", required_rowsets_arr, root.GetAllocator());
+ root.AddMember("missing_rowsets", missing_rowsets_arr, root.GetAllocator());
+ rapidjson::StringBuffer strbuf;
+ rapidjson::PrettyWriter<rapidjson::StringBuffer> writer(strbuf);
+ root.Accept(writer);
+ std::string rowset_status_string = std::string(strbuf.GetString());
+ LOG_EVERY_SECOND(WARNING) << rowset_status_string;
+ // let it crash if correctness check failed in Debug mode
+ DCHECK(false) << "delete bitmap correctness check failed in publish phase!";
+ return Status::InternalError("check delete bitmap failed!");
+ }
+ return Status::OK();
+}
+
+void BaseTablet::_remove_sentinel_mark_from_delete_bitmap(DeleteBitmapPtr delete_bitmap) {
+ for (auto it = delete_bitmap->delete_bitmap.begin(), end = delete_bitmap->delete_bitmap.end();
+ it != end;) {
+ if (std::get<1>(it->first) == DeleteBitmap::INVALID_SEGMENT_ID) {
+ it = delete_bitmap->delete_bitmap.erase(it);
+ } else {
+ ++it;
+ }
+ }
+}
+
+Status BaseTablet::update_delete_bitmap(const BaseTabletSPtr& self, const TabletTxnInfo* txn_info,
Review Comment:
warning: function 'update_delete_bitmap' exceeds recommended size/complexity thresholds [readability-function-size]
```cpp
Status BaseTablet::update_delete_bitmap(const BaseTabletSPtr& self, const TabletTxnInfo* txn_info,
^
```
<details>
<summary>Additional context</summary>
**be/src/olap/base_tablet.cpp:1143:** 95 lines including whitespace and comments (threshold 80)
```cpp
Status BaseTablet::update_delete_bitmap(const BaseTabletSPtr& self, const TabletTxnInfo* txn_info,
^
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "liaoxin01 (via GitHub)" <gi...@apache.org>.
liaoxin01 commented on PR #31317:
URL: https://github.com/apache/doris/pull/31317#issuecomment-1969252801
run cloud_p0
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "dataroaring (via GitHub)" <gi...@apache.org>.
dataroaring merged PR #31317:
URL: https://github.com/apache/doris/pull/31317
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "github-actions[bot] (via GitHub)" <gi...@apache.org>.
github-actions[bot] commented on code in PR #31317:
URL: https://github.com/apache/doris/pull/31317#discussion_r1505667971
##########
be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp:
##########
@@ -0,0 +1,165 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "cloud/cloud_engine_calc_delete_bitmap_task.h"
+
+#include <memory>
+
+#include "cloud/cloud_meta_mgr.h"
+#include "cloud/cloud_tablet.h"
+#include "common/status.h"
+#include "olap/base_tablet.h"
+#include "olap/olap_common.h"
+#include "olap/rowset/rowset.h"
+#include "olap/tablet_fwd.h"
+#include "olap/tablet_meta.h"
+#include "olap/txn_manager.h"
+#include "olap/utils.h"
+
+namespace doris {
+
+CloudEngineCalcDeleteBitmapTask::CloudEngineCalcDeleteBitmapTask(
+ CloudStorageEngine& engine, const TCalcDeleteBitmapRequest& cal_delete_bitmap_req,
+ std::vector<TTabletId>* error_tablet_ids, std::vector<TTabletId>* succ_tablet_ids)
+ : _engine(engine),
+ _cal_delete_bitmap_req(cal_delete_bitmap_req),
+ _error_tablet_ids(error_tablet_ids),
+ _succ_tablet_ids(succ_tablet_ids) {}
+
+void CloudEngineCalcDeleteBitmapTask::add_error_tablet_id(int64_t tablet_id, const Status& err) {
Review Comment:
warning: method 'add_error_tablet_id' can be made static [readability-convert-member-functions-to-static]
be/src/cloud/cloud_engine_calc_delete_bitmap_task.h:58:
```diff
- void add_error_tablet_id(int64_t tablet_id, const Status& err);
+ static void add_error_tablet_id(int64_t tablet_id, const Status& err);
```
##########
be/src/olap/base_tablet.cpp:
##########
@@ -1059,4 +1063,181 @@ Status BaseTablet::_capture_consistent_rowsets_unlocked(
return Status::OK();
}
+Status BaseTablet::check_delete_bitmap_correctness(DeleteBitmapPtr delete_bitmap,
+ int64_t max_version, int64_t txn_id,
+ const RowsetIdUnorderedSet& rowset_ids,
+ std::vector<RowsetSharedPtr>* rowsets) {
+ RowsetIdUnorderedSet missing_ids;
+ for (const auto& rowsetid : rowset_ids) {
+ if (!delete_bitmap->delete_bitmap.contains({rowsetid, DeleteBitmap::INVALID_SEGMENT_ID,
+ DeleteBitmap::TEMP_VERSION_COMMON})) {
+ missing_ids.insert(rowsetid);
+ }
+ }
+
+ if (!missing_ids.empty()) {
+ LOG(WARNING) << "[txn_id:" << txn_id << "][tablet_id:" << tablet_id()
+ << "][max_version: " << max_version
+ << "] check delete bitmap correctness failed!";
+ rapidjson::Document root;
+ root.SetObject();
+ rapidjson::Document required_rowsets_arr;
+ required_rowsets_arr.SetArray();
+ rapidjson::Document missing_rowsets_arr;
+ missing_rowsets_arr.SetArray();
+
+ if (rowsets != nullptr) {
+ for (const auto& rowset : *rowsets) {
+ rapidjson::Value value;
+ std::string version_str = rowset->get_rowset_info_str();
+ value.SetString(version_str.c_str(), version_str.length(),
+ required_rowsets_arr.GetAllocator());
+ required_rowsets_arr.PushBack(value, required_rowsets_arr.GetAllocator());
+ }
+ } else {
+ std::vector<RowsetSharedPtr> rowsets;
+ {
+ std::shared_lock meta_rlock(_meta_lock);
+ rowsets = get_rowset_by_ids(&rowset_ids);
+ }
+ for (const auto& rowset : rowsets) {
+ rapidjson::Value value;
+ std::string version_str = rowset->get_rowset_info_str();
+ value.SetString(version_str.c_str(), version_str.length(),
+ required_rowsets_arr.GetAllocator());
+ required_rowsets_arr.PushBack(value, required_rowsets_arr.GetAllocator());
+ }
+ }
+ for (const auto& missing_rowset_id : missing_ids) {
+ rapidjson::Value miss_value;
+ std::string rowset_id_str = missing_rowset_id.to_string();
+ miss_value.SetString(rowset_id_str.c_str(), rowset_id_str.length(),
+ missing_rowsets_arr.GetAllocator());
+ missing_rowsets_arr.PushBack(miss_value, missing_rowsets_arr.GetAllocator());
+ }
+
+ root.AddMember("required_rowsets", required_rowsets_arr, root.GetAllocator());
+ root.AddMember("missing_rowsets", missing_rowsets_arr, root.GetAllocator());
+ rapidjson::StringBuffer strbuf;
+ rapidjson::PrettyWriter<rapidjson::StringBuffer> writer(strbuf);
+ root.Accept(writer);
+ std::string rowset_status_string = std::string(strbuf.GetString());
+ LOG_EVERY_SECOND(WARNING) << rowset_status_string;
+ // let it crash if correctness check failed in Debug mode
+ DCHECK(false) << "delete bitmap correctness check failed in publish phase!";
+ return Status::InternalError("check delete bitmap failed!");
+ }
+ return Status::OK();
+}
+
+void BaseTablet::_remove_sentinel_mark_from_delete_bitmap(DeleteBitmapPtr delete_bitmap) {
+ for (auto it = delete_bitmap->delete_bitmap.begin(), end = delete_bitmap->delete_bitmap.end();
+ it != end;) {
+ if (std::get<1>(it->first) == DeleteBitmap::INVALID_SEGMENT_ID) {
+ it = delete_bitmap->delete_bitmap.erase(it);
+ } else {
+ ++it;
+ }
+ }
+}
+
+Status BaseTablet::update_delete_bitmap(const BaseTabletSPtr& self, const TabletTxnInfo* txn_info,
Review Comment:
warning: function 'update_delete_bitmap' exceeds recommended size/complexity thresholds [readability-function-size]
```cpp
Status BaseTablet::update_delete_bitmap(const BaseTabletSPtr& self, const TabletTxnInfo* txn_info,
^
```
<details>
<summary>Additional context</summary>
**be/src/olap/base_tablet.cpp:1143:** 96 lines including whitespace and comments (threshold 80)
```cpp
Status BaseTablet::update_delete_bitmap(const BaseTabletSPtr& self, const TabletTxnInfo* txn_info,
^
```
</details>
##########
be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp:
##########
@@ -0,0 +1,165 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "cloud/cloud_engine_calc_delete_bitmap_task.h"
+
+#include <memory>
+
+#include "cloud/cloud_meta_mgr.h"
+#include "cloud/cloud_tablet.h"
+#include "common/status.h"
+#include "olap/base_tablet.h"
+#include "olap/olap_common.h"
+#include "olap/rowset/rowset.h"
+#include "olap/tablet_fwd.h"
+#include "olap/tablet_meta.h"
+#include "olap/txn_manager.h"
+#include "olap/utils.h"
+
+namespace doris {
+
+CloudEngineCalcDeleteBitmapTask::CloudEngineCalcDeleteBitmapTask(
+ CloudStorageEngine& engine, const TCalcDeleteBitmapRequest& cal_delete_bitmap_req,
+ std::vector<TTabletId>* error_tablet_ids, std::vector<TTabletId>* succ_tablet_ids)
+ : _engine(engine),
+ _cal_delete_bitmap_req(cal_delete_bitmap_req),
+ _error_tablet_ids(error_tablet_ids),
+ _succ_tablet_ids(succ_tablet_ids) {}
+
+void CloudEngineCalcDeleteBitmapTask::add_error_tablet_id(int64_t tablet_id, const Status& err) {
+ std::lock_guard<std::mutex> lck(_mutex);
+ _error_tablet_ids->push_back(tablet_id);
+ if (_res.ok() || _res.is<ErrorCode::DELETE_BITMAP_LOCK_ERROR>()) {
+ _res = err;
+ }
+}
+
+void CloudEngineCalcDeleteBitmapTask::add_succ_tablet_id(int64_t tablet_id) {
+ std::lock_guard<std::mutex> lck(_mutex);
+ _succ_tablet_ids->push_back(tablet_id);
+}
+
+Status CloudEngineCalcDeleteBitmapTask::execute() {
+ int64_t transaction_id = _cal_delete_bitmap_req.transaction_id;
+ OlapStopWatch watch;
+ VLOG_NOTICE << "begin to calculate delete bitmap. transaction_id=" << transaction_id;
+ std::unique_ptr<ThreadPoolToken> token =
+ _engine.calc_tablet_delete_bitmap_task_thread_pool()->new_token(
+ ThreadPool::ExecutionMode::CONCURRENT);
+
+ for (const auto& partition : _cal_delete_bitmap_req.partitions) {
+ int64_t version = partition.version;
+ for (auto tablet_id : partition.tablet_ids) {
+ auto base_tablet = DORIS_TRY(_engine.get_tablet(tablet_id));
+ std::shared_ptr<CloudTablet> tablet =
+ std::dynamic_pointer_cast<CloudTablet>(base_tablet);
+ if (tablet == nullptr) {
+ LOG(WARNING) << "can't get tablet when calculate delete bitmap. tablet_id="
+ << tablet_id;
+ _error_tablet_ids->push_back(tablet_id);
+ _res = Status::Error<ErrorCode::PUSH_TABLE_NOT_EXIST>(
+ "can't get tablet when calculate delete bitmap. tablet_id={}", tablet_id);
+ break;
+ }
+
+ Status st = tablet->sync_rowsets();
+ if (!st.ok() && !st.is<ErrorCode::INVALID_TABLET_STATE>()) {
+ return st;
+ }
+ if (st.is<ErrorCode::INVALID_TABLET_STATE>()) [[unlikely]] {
+ add_succ_tablet_id(tablet->tablet_id());
+ LOG(INFO)
+ << "tablet is under alter process, delete bitmap will be calculated later, "
+ "tablet_id: "
+ << tablet->tablet_id() << " txn_id: " << transaction_id
+ << ", request_version=" << version;
+ continue;
+ }
+ int64_t max_version = tablet->max_version_unlocked();
+ if (version != max_version + 1) {
+ _error_tablet_ids->push_back(tablet_id);
+ _res = Status::Error<ErrorCode::DELETE_BITMAP_LOCK_ERROR, false>(
+ "version not continuous");
+ LOG(WARNING) << "version not continuous, current max version=" << max_version
+ << ", request_version=" << version
+ << " tablet_id=" << tablet->tablet_id();
+ break;
+ }
+
+ auto tablet_calc_delete_bitmap_ptr = std::make_shared<CloudTabletCalcDeleteBitmapTask>(
+ _engine, this, tablet, transaction_id, version);
+ auto submit_st = token->submit_func([=]() { tablet_calc_delete_bitmap_ptr->handle(); });
+ CHECK(submit_st.ok());
+ }
+ }
+ // wait for all finished
+ token->wait();
+
+ LOG(INFO) << "finish to calculate delete bitmap on transaction."
+ << "transaction_id=" << transaction_id << ", cost(us): " << watch.get_elapse_time_us()
+ << ", error_tablet_size=" << _error_tablet_ids->size()
+ << ", res=" << _res.to_string();
+ return _res;
+}
+
+CloudTabletCalcDeleteBitmapTask::CloudTabletCalcDeleteBitmapTask(
+ CloudStorageEngine& engine, CloudEngineCalcDeleteBitmapTask* engine_task,
+ std::shared_ptr<CloudTablet> tablet, int64_t transaction_id, int64_t version)
+ : _engine(engine),
+ _engine_calc_delete_bitmap_task(engine_task),
+ _tablet(tablet),
+ _transaction_id(transaction_id),
+ _version(version) {}
+
+void CloudTabletCalcDeleteBitmapTask::handle() {
Review Comment:
warning: method 'handle' can be made const [readability-make-member-function-const]
```suggestion
void CloudTabletCalcDeleteBitmapTask::handle() const {
```
be/src/cloud/cloud_engine_calc_delete_bitmap_task.h:39:
```diff
- void handle();
+ void handle() const;
```
##########
be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp:
##########
@@ -0,0 +1,165 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "cloud/cloud_engine_calc_delete_bitmap_task.h"
+
+#include <memory>
+
+#include "cloud/cloud_meta_mgr.h"
+#include "cloud/cloud_tablet.h"
+#include "common/status.h"
+#include "olap/base_tablet.h"
+#include "olap/olap_common.h"
+#include "olap/rowset/rowset.h"
+#include "olap/tablet_fwd.h"
+#include "olap/tablet_meta.h"
+#include "olap/txn_manager.h"
+#include "olap/utils.h"
+
+namespace doris {
+
+CloudEngineCalcDeleteBitmapTask::CloudEngineCalcDeleteBitmapTask(
+ CloudStorageEngine& engine, const TCalcDeleteBitmapRequest& cal_delete_bitmap_req,
+ std::vector<TTabletId>* error_tablet_ids, std::vector<TTabletId>* succ_tablet_ids)
+ : _engine(engine),
+ _cal_delete_bitmap_req(cal_delete_bitmap_req),
+ _error_tablet_ids(error_tablet_ids),
+ _succ_tablet_ids(succ_tablet_ids) {}
+
+void CloudEngineCalcDeleteBitmapTask::add_error_tablet_id(int64_t tablet_id, const Status& err) {
+ std::lock_guard<std::mutex> lck(_mutex);
+ _error_tablet_ids->push_back(tablet_id);
+ if (_res.ok() || _res.is<ErrorCode::DELETE_BITMAP_LOCK_ERROR>()) {
+ _res = err;
+ }
+}
+
+void CloudEngineCalcDeleteBitmapTask::add_succ_tablet_id(int64_t tablet_id) {
Review Comment:
warning: method 'add_succ_tablet_id' can be made static [readability-convert-member-functions-to-static]
be/src/cloud/cloud_engine_calc_delete_bitmap_task.h:59:
```diff
- void add_succ_tablet_id(int64_t tablet_id);
+ static void add_succ_tablet_id(int64_t tablet_id);
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "liaoxin01 (via GitHub)" <gi...@apache.org>.
liaoxin01 commented on PR #31317:
URL: https://github.com/apache/doris/pull/31317#issuecomment-1968952467
run buildall
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "doris-robot (via GitHub)" <gi...@apache.org>.
doris-robot commented on PR #31317:
URL: https://github.com/apache/doris/pull/31317#issuecomment-1969123668
TeamCity be ut coverage result:
Function Coverage: 35.63% (8547/23989)
Line Coverage: 27.44% (69376/252827)
Region Coverage: 26.61% (35975/135169)
Branch Coverage: 23.43% (18386/78476)
Coverage Report: http://coverage.selectdb-in.cc/coverage/0a23e751291ddf15970f07c0384db0ab6c5606aa_0a23e751291ddf15970f07c0384db0ab6c5606aa/report/index.html
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "doris-robot (via GitHub)" <gi...@apache.org>.
doris-robot commented on PR #31317:
URL: https://github.com/apache/doris/pull/31317#issuecomment-1970692637
TeamCity be ut coverage result:
Function Coverage: 35.63% (8550/23999)
Line Coverage: 27.44% (69394/252859)
Region Coverage: 26.61% (35983/135232)
Branch Coverage: 23.42% (18389/78512)
Coverage Report: http://coverage.selectdb-in.cc/coverage/855559f1046c84941399efe6b5240367ba209876_855559f1046c84941399efe6b5240367ba209876/report/index.html
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "liaoxin01 (via GitHub)" <gi...@apache.org>.
liaoxin01 commented on PR #31317:
URL: https://github.com/apache/doris/pull/31317#issuecomment-1972968942
run buildall
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "github-actions[bot] (via GitHub)" <gi...@apache.org>.
github-actions[bot] commented on PR #31317:
URL: https://github.com/apache/doris/pull/31317#issuecomment-1973196216
PR approved by at least one committer and no changes requested.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "doris-robot (via GitHub)" <gi...@apache.org>.
doris-robot commented on PR #31317:
URL: https://github.com/apache/doris/pull/31317#issuecomment-1970550227
<details>
<summary>TPC-H: <b>Total hot run time: 37545 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpch-tools
Tpch sf100 test result on commit 2c80b3d53760bb63ae90fcf0ed7a6ee5c67be4a0, data reload: false
------ Round 1 ----------------------------------
q1 17768 4023 3999 3999
q2 2038 142 131 131
q3 10575 933 938 933
q4 4657 923 933 923
q5 7638 2819 2958 2819
q6 177 122 121 121
q7 1230 753 735 735
q8 9254 1982 2029 1982
q9 7121 6246 6212 6212
q10 8161 2485 2510 2485
q11 420 201 209 201
q12 749 308 308 308
q13 17971 3192 3197 3192
q14 268 254 253 253
q15 534 500 494 494
q16 476 425 404 404
q17 939 886 834 834
q18 6797 5929 5981 5929
q19 1541 1502 1500 1500
q20 542 276 272 272
q21 6386 3539 3539 3539
q22 799 302 279 279
Total cold run time: 106041 ms
Total hot run time: 37545 ms
----- Round 2, with runtime_filter_mode=off -----
q1 3993 4021 3975 3975
q2 302 214 219 214
q3 2987 2935 2940 2935
q4 1858 1782 1804 1782
q5 5143 5133 5099 5099
q6 199 115 117 115
q7 2153 1739 1756 1739
q8 3125 3190 3196 3190
q9 8225 8213 8197 8197
q10 6067 3662 3650 3650
q11 510 448 442 442
q12 699 543 539 539
q13 9575 3067 3023 3023
q14 284 246 257 246
q15 545 487 488 487
q16 517 490 436 436
q17 1694 1661 1661 1661
q18 7872 7447 7270 7270
q19 1643 1623 1623 1623
q20 2072 1897 1914 1897
q21 4808 4620 4623 4620
q22 508 441 462 441
Total cold run time: 64779 ms
Total hot run time: 53581 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "doris-robot (via GitHub)" <gi...@apache.org>.
doris-robot commented on PR #31317:
URL: https://github.com/apache/doris/pull/31317#issuecomment-1970562493
<details>
<summary>TPC-DS: <b>Total hot run time: 169096 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpcds-tools
TPC-DS sf100 test result on commit 2c80b3d53760bb63ae90fcf0ed7a6ee5c67be4a0, data reload: false
query1 931 340 337 337
query2 6555 1765 1698 1698
query3 6700 205 202 202
query4 23064 20668 20490 20490
query5 4278 362 358 358
query6 263 167 176 167
query7 4608 294 299 294
query8 267 196 196 196
query9 8441 2301 2277 2277
query10 415 211 209 209
query11 14452 14266 14060 14060
query12 137 86 81 81
query13 1659 423 421 421
query14 8596 6762 6294 6294
query15 203 175 182 175
query16 7098 275 252 252
query17 1020 546 495 495
query18 1898 268 276 268
query19 188 148 157 148
query20 87 82 86 82
query21 204 120 115 115
query22 4817 4666 4699 4666
query23 30448 29768 29761 29761
query24 11640 3094 3137 3094
query25 617 354 343 343
query26 1657 156 154 154
query27 3042 317 315 315
query28 7040 1829 1848 1829
query29 1121 562 559 559
query30 282 130 138 130
query31 855 689 689 689
query32 96 59 55 55
query33 710 221 223 221
query34 1044 488 477 477
query35 829 743 738 738
query36 917 880 887 880
query37 134 65 62 62
query38 3052 3000 2933 2933
query39 1323 1272 1253 1253
query40 281 104 100 100
query41 38 36 35 35
query42 106 98 96 96
query43 478 428 433 428
query44 1080 682 720 682
query45 188 177 168 168
query46 1048 789 780 780
query47 1540 1482 1442 1442
query48 421 356 348 348
query49 1118 285 293 285
query50 761 370 371 370
query51 4402 4269 4286 4269
query52 106 92 87 87
query53 338 269 267 267
query54 306 227 220 220
query55 87 88 80 80
query56 221 205 206 205
query57 999 922 882 882
query58 212 186 205 186
query59 2133 2033 2167 2033
query60 228 212 224 212
query61 86 84 92 84
query62 594 359 362 359
query63 293 260 257 257
query64 6550 3046 3198 3046
query65 3244 3219 3214 3214
query66 1441 314 310 310
query67 14081 14232 14298 14232
query68 4944 565 576 565
query69 525 388 373 373
query70 1225 1219 1165 1165
query71 359 246 252 246
query72 6234 2815 2686 2686
query73 712 318 321 318
query74 6259 6023 5992 5992
query75 3063 2392 2393 2392
query76 2791 1008 1111 1008
query77 347 226 231 226
query78 9096 8558 8438 8438
query79 1293 523 533 523
query80 932 355 345 345
query81 488 215 214 214
query82 959 87 80 80
query83 221 122 121 121
query84 277 78 78 78
query85 1321 358 344 344
query86 369 297 295 295
query87 3245 3078 3129 3078
query88 2657 2325 2250 2250
query89 394 337 344 337
query90 1844 164 172 164
query91 153 123 125 123
query92 56 50 53 50
query93 1191 515 510 510
query94 1228 186 181 181
query95 449 345 346 345
query96 572 256 269 256
query97 4252 4156 4108 4108
query98 220 214 187 187
query99 1029 651 643 643
Total cold run time: 261773 ms
Total hot run time: 169096 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "doris-robot (via GitHub)" <gi...@apache.org>.
doris-robot commented on PR #31317:
URL: https://github.com/apache/doris/pull/31317#issuecomment-1971277459
<details>
<summary>ClickBench: <b>Total hot run time: 32.74 s</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/clickbench-tools
ClickBench test result on commit 4b84f9d7a277eea6e617ded20cd5b7b82ecbe1a9, data reload: false
query1 0.04 0.03 0.03
query2 0.06 0.02 0.03
query3 0.24 0.06 0.06
query4 1.64 0.10 0.09
query5 0.54 0.52 0.52
query6 1.31 0.66 0.68
query7 0.02 0.01 0.01
query8 0.04 0.03 0.03
query9 0.58 0.51 0.51
query10 0.56 0.58 0.58
query11 0.13 0.10 0.10
query12 0.12 0.10 0.10
query13 0.63 0.61 0.62
query14 0.80 0.81 0.81
query15 0.84 0.82 0.81
query16 0.37 0.39 0.38
query17 0.95 1.02 0.97
query18 0.27 0.25 0.24
query19 1.79 1.69 1.72
query20 0.01 0.00 0.01
query21 15.42 0.65 0.62
query22 3.28 3.19 2.95
query23 17.14 0.99 0.95
query24 2.28 0.27 0.53
query25 0.21 0.06 0.05
query26 0.16 0.13 0.14
query27 0.04 0.02 0.04
query28 12.11 0.84 0.82
query29 12.52 3.30 3.32
query30 0.62 0.54 0.52
query31 2.78 0.35 0.36
query32 3.32 0.48 0.48
query33 3.20 3.25 3.27
query34 15.34 4.78 4.75
query35 4.74 4.74 4.74
query36 1.14 1.06 1.06
query37 0.07 0.04 0.05
query38 0.04 0.03 0.03
query39 0.02 0.02 0.02
query40 0.16 0.12 0.12
query41 0.07 0.02 0.02
query42 0.02 0.01 0.02
query43 0.03 0.02 0.02
Total cold run time: 105.65 s
Total hot run time: 32.74 s
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "doris-robot (via GitHub)" <gi...@apache.org>.
doris-robot commented on PR #31317:
URL: https://github.com/apache/doris/pull/31317#issuecomment-1971283590
Load test result on machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
```
Load test result on commit 4b84f9d7a277eea6e617ded20cd5b7b82ecbe1a9 with default session variables
Stream load json: 19 seconds loaded 2358488459 Bytes, about 118 MB/s
Stream load orc: 59 seconds loaded 1101869774 Bytes, about 17 MB/s
Stream load parquet: 31 seconds loaded 861443392 Bytes, about 26 MB/s
Insert into select: 17.8 seconds inserted 10000000 Rows, about 561K ops/s
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "liaoxin01 (via GitHub)" <gi...@apache.org>.
liaoxin01 commented on code in PR #31317:
URL: https://github.com/apache/doris/pull/31317#discussion_r1508822030
##########
be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp:
##########
@@ -0,0 +1,165 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "cloud/cloud_engine_calc_delete_bitmap_task.h"
+
+#include <memory>
+
+#include "cloud/cloud_meta_mgr.h"
+#include "cloud/cloud_tablet.h"
+#include "common/status.h"
+#include "olap/base_tablet.h"
+#include "olap/olap_common.h"
+#include "olap/rowset/rowset.h"
+#include "olap/tablet_fwd.h"
+#include "olap/tablet_meta.h"
+#include "olap/txn_manager.h"
+#include "olap/utils.h"
+
+namespace doris {
+
+CloudEngineCalcDeleteBitmapTask::CloudEngineCalcDeleteBitmapTask(
+ CloudStorageEngine& engine, const TCalcDeleteBitmapRequest& cal_delete_bitmap_req,
+ std::vector<TTabletId>* error_tablet_ids, std::vector<TTabletId>* succ_tablet_ids)
+ : _engine(engine),
+ _cal_delete_bitmap_req(cal_delete_bitmap_req),
+ _error_tablet_ids(error_tablet_ids),
+ _succ_tablet_ids(succ_tablet_ids) {}
+
+void CloudEngineCalcDeleteBitmapTask::add_error_tablet_id(int64_t tablet_id, const Status& err) {
+ std::lock_guard<std::mutex> lck(_mutex);
+ _error_tablet_ids->push_back(tablet_id);
+ if (_res.ok() || _res.is<ErrorCode::DELETE_BITMAP_LOCK_ERROR>()) {
+ _res = err;
+ }
+}
+
+void CloudEngineCalcDeleteBitmapTask::add_succ_tablet_id(int64_t tablet_id) {
+ std::lock_guard<std::mutex> lck(_mutex);
+ _succ_tablet_ids->push_back(tablet_id);
+}
+
+Status CloudEngineCalcDeleteBitmapTask::execute() {
+ int64_t transaction_id = _cal_delete_bitmap_req.transaction_id;
+ OlapStopWatch watch;
+ VLOG_NOTICE << "begin to calculate delete bitmap. transaction_id=" << transaction_id;
+ std::unique_ptr<ThreadPoolToken> token =
+ _engine.calc_tablet_delete_bitmap_task_thread_pool()->new_token(
+ ThreadPool::ExecutionMode::CONCURRENT);
+
+ for (const auto& partition : _cal_delete_bitmap_req.partitions) {
+ int64_t version = partition.version;
+ for (auto tablet_id : partition.tablet_ids) {
+ auto base_tablet = DORIS_TRY(_engine.get_tablet(tablet_id));
+ std::shared_ptr<CloudTablet> tablet =
+ std::dynamic_pointer_cast<CloudTablet>(base_tablet);
+ if (tablet == nullptr) {
+ LOG(WARNING) << "can't get tablet when calculate delete bitmap. tablet_id="
+ << tablet_id;
+ _error_tablet_ids->push_back(tablet_id);
+ _res = Status::Error<ErrorCode::PUSH_TABLE_NOT_EXIST>(
+ "can't get tablet when calculate delete bitmap. tablet_id={}", tablet_id);
+ break;
+ }
+
+ Status st = tablet->sync_rowsets();
+ if (!st.ok() && !st.is<ErrorCode::INVALID_TABLET_STATE>()) {
+ return st;
+ }
+ if (st.is<ErrorCode::INVALID_TABLET_STATE>()) [[unlikely]] {
+ add_succ_tablet_id(tablet->tablet_id());
+ LOG(INFO)
+ << "tablet is under alter process, delete bitmap will be calculated later, "
Review Comment:
schema change and compaction will be submitted in next pr.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "doris-robot (via GitHub)" <gi...@apache.org>.
doris-robot commented on PR #31317:
URL: https://github.com/apache/doris/pull/31317#issuecomment-1970376891
<details>
<summary>TPC-DS: <b>Total hot run time: 168946 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpcds-tools
TPC-DS sf100 test result on commit ced38f8389a29b7ffe8a5684b92d24002d3a26e8, data reload: false
query1 956 344 337 337
query2 6527 1694 1790 1694
query3 6701 203 199 199
query4 23192 20666 20603 20603
query5 4314 360 351 351
query6 254 163 174 163
query7 4610 298 298 298
query8 271 190 194 190
query9 8485 2279 2248 2248
query10 418 219 221 219
query11 14626 14193 14099 14099
query12 137 82 80 80
query13 1632 431 437 431
query14 8629 6253 6303 6253
query15 205 173 171 171
query16 7079 269 253 253
query17 1029 525 504 504
query18 1893 267 264 264
query19 196 145 148 145
query20 85 82 79 79
query21 198 129 119 119
query22 4835 4658 4602 4602
query23 30625 29883 29937 29883
query24 12034 3131 3095 3095
query25 652 345 335 335
query26 1895 148 162 148
query27 3068 304 307 304
query28 7060 1820 1830 1820
query29 1275 547 549 547
query30 277 129 151 129
query31 861 690 704 690
query32 98 59 56 56
query33 724 238 211 211
query34 1048 480 498 480
query35 835 754 754 754
query36 969 871 889 871
query37 259 63 62 62
query38 3068 2989 2967 2967
query39 1322 1280 1249 1249
query40 278 97 97 97
query41 38 36 36 36
query42 98 101 101 101
query43 454 427 434 427
query44 1060 693 707 693
query45 194 176 177 176
query46 1039 799 754 754
query47 1555 1377 1379 1377
query48 410 345 339 339
query49 1169 305 293 293
query50 766 362 365 362
query51 4393 4282 4357 4282
query52 106 90 91 90
query53 338 256 268 256
query54 292 225 227 225
query55 79 78 80 78
query56 214 206 194 194
query57 999 917 895 895
query58 221 194 199 194
query59 2147 2117 2088 2088
query60 246 213 207 207
query61 86 89 83 83
query62 584 362 334 334
query63 288 257 255 255
query64 6527 3007 3121 3007
query65 3261 3205 3193 3193
query66 1424 324 322 322
query67 14182 14034 14154 14034
query68 5053 552 563 552
query69 538 369 373 369
query70 1155 1238 1230 1230
query71 390 243 249 243
query72 6246 2806 2648 2648
query73 721 317 314 314
query74 6321 6074 6036 6036
query75 3065 2450 2432 2432
query76 3003 1072 1177 1072
query77 350 244 231 231
query78 9063 8551 8416 8416
query79 1088 533 514 514
query80 901 350 344 344
query81 466 195 194 194
query82 1282 87 79 79
query83 220 122 125 122
query84 277 78 77 77
query85 1350 342 335 335
query86 363 306 303 303
query87 3281 3096 3072 3072
query88 2655 2215 2224 2215
query89 400 336 328 328
query90 1905 161 157 157
query91 149 123 123 123
query92 54 48 47 47
query93 1043 521 510 510
query94 1221 172 179 172
query95 427 340 336 336
query96 569 260 257 257
query97 4281 4107 4165 4107
query98 208 200 188 188
query99 1041 628 677 628
Total cold run time: 263876 ms
Total hot run time: 168946 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "doris-robot (via GitHub)" <gi...@apache.org>.
doris-robot commented on PR #31317:
URL: https://github.com/apache/doris/pull/31317#issuecomment-1970685164
Load test result on machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
```
Load test result on commit 855559f1046c84941399efe6b5240367ba209876 with default session variables
Stream load json: 18 seconds loaded 2358488459 Bytes, about 124 MB/s
Stream load orc: 59 seconds loaded 1101869774 Bytes, about 17 MB/s
Stream load parquet: 32 seconds loaded 861443392 Bytes, about 25 MB/s
Insert into select: 18.9 seconds inserted 10000000 Rows, about 529K ops/s
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "github-actions[bot] (via GitHub)" <gi...@apache.org>.
github-actions[bot] commented on code in PR #31317:
URL: https://github.com/apache/doris/pull/31317#discussion_r1508834637
##########
be/src/olap/base_tablet.cpp:
##########
@@ -1059,4 +1064,200 @@
return Status::OK();
}
+Status BaseTablet::check_delete_bitmap_correctness(DeleteBitmapPtr delete_bitmap,
+ int64_t max_version, int64_t txn_id,
+ const RowsetIdUnorderedSet& rowset_ids,
+ std::vector<RowsetSharedPtr>* rowsets) {
+ RowsetIdUnorderedSet missing_ids;
+ for (const auto& rowsetid : rowset_ids) {
+ if (!delete_bitmap->delete_bitmap.contains({rowsetid, DeleteBitmap::INVALID_SEGMENT_ID,
+ DeleteBitmap::TEMP_VERSION_COMMON})) {
+ missing_ids.insert(rowsetid);
+ }
+ }
+
+ if (!missing_ids.empty()) {
+ LOG(WARNING) << "[txn_id:" << txn_id << "][tablet_id:" << tablet_id()
+ << "][max_version: " << max_version
+ << "] check delete bitmap correctness failed!";
+ rapidjson::Document root;
+ root.SetObject();
+ rapidjson::Document required_rowsets_arr;
+ required_rowsets_arr.SetArray();
+ rapidjson::Document missing_rowsets_arr;
+ missing_rowsets_arr.SetArray();
+
+ if (rowsets != nullptr) {
+ for (const auto& rowset : *rowsets) {
+ rapidjson::Value value;
+ std::string version_str = rowset->get_rowset_info_str();
+ value.SetString(version_str.c_str(), version_str.length(),
+ required_rowsets_arr.GetAllocator());
+ required_rowsets_arr.PushBack(value, required_rowsets_arr.GetAllocator());
+ }
+ } else {
+ std::vector<RowsetSharedPtr> rowsets;
+ {
+ std::shared_lock meta_rlock(_meta_lock);
+ rowsets = get_rowset_by_ids(&rowset_ids);
+ }
+ for (const auto& rowset : rowsets) {
+ rapidjson::Value value;
+ std::string version_str = rowset->get_rowset_info_str();
+ value.SetString(version_str.c_str(), version_str.length(),
+ required_rowsets_arr.GetAllocator());
+ required_rowsets_arr.PushBack(value, required_rowsets_arr.GetAllocator());
+ }
+ }
+ for (const auto& missing_rowset_id : missing_ids) {
+ rapidjson::Value miss_value;
+ std::string rowset_id_str = missing_rowset_id.to_string();
+ miss_value.SetString(rowset_id_str.c_str(), rowset_id_str.length(),
+ missing_rowsets_arr.GetAllocator());
+ missing_rowsets_arr.PushBack(miss_value, missing_rowsets_arr.GetAllocator());
+ }
+
+ root.AddMember("required_rowsets", required_rowsets_arr, root.GetAllocator());
+ root.AddMember("missing_rowsets", missing_rowsets_arr, root.GetAllocator());
+ rapidjson::StringBuffer strbuf;
+ rapidjson::PrettyWriter<rapidjson::StringBuffer> writer(strbuf);
+ root.Accept(writer);
+ std::string rowset_status_string = std::string(strbuf.GetString());
+ LOG_EVERY_SECOND(WARNING) << rowset_status_string;
+ // let it crash if correctness check failed in Debug mode
+ DCHECK(false) << "delete bitmap correctness check failed in publish phase!";
+ return Status::InternalError("check delete bitmap failed!");
+ }
+ return Status::OK();
+}
+
+void BaseTablet::_remove_sentinel_mark_from_delete_bitmap(DeleteBitmapPtr delete_bitmap) {
+ for (auto it = delete_bitmap->delete_bitmap.begin(), end = delete_bitmap->delete_bitmap.end();
+ it != end;) {
+ if (std::get<1>(it->first) == DeleteBitmap::INVALID_SEGMENT_ID) {
+ it = delete_bitmap->delete_bitmap.erase(it);
+ } else {
+ ++it;
+ }
+ }
+}
+
+Status BaseTablet::update_delete_bitmap(const BaseTabletSPtr& self, const TabletTxnInfo* txn_info,
Review Comment:
warning: function 'update_delete_bitmap' exceeds recommended size/complexity thresholds [readability-function-size]
```cpp
Status BaseTablet::update_delete_bitmap(const BaseTabletSPtr& self, const TabletTxnInfo* txn_info,
^
```
<details>
<summary>Additional context</summary>
**be/src/olap/base_tablet.cpp:1144:** 115 lines including whitespace and comments (threshold 80)
```cpp
Status BaseTablet::update_delete_bitmap(const BaseTabletSPtr& self, const TabletTxnInfo* txn_info,
^
```
</details>
##########
be/src/olap/base_tablet.cpp:
##########
@@ -1059,4 +1064,200 @@ Status BaseTablet::_capture_consistent_rowsets_unlocked(
return Status::OK();
}
+Status BaseTablet::check_delete_bitmap_correctness(DeleteBitmapPtr delete_bitmap,
+ int64_t max_version, int64_t txn_id,
+ const RowsetIdUnorderedSet& rowset_ids,
+ std::vector<RowsetSharedPtr>* rowsets) {
+ RowsetIdUnorderedSet missing_ids;
+ for (const auto& rowsetid : rowset_ids) {
+ if (!delete_bitmap->delete_bitmap.contains({rowsetid, DeleteBitmap::INVALID_SEGMENT_ID,
+ DeleteBitmap::TEMP_VERSION_COMMON})) {
+ missing_ids.insert(rowsetid);
+ }
+ }
+
+ if (!missing_ids.empty()) {
+ LOG(WARNING) << "[txn_id:" << txn_id << "][tablet_id:" << tablet_id()
+ << "][max_version: " << max_version
+ << "] check delete bitmap correctness failed!";
+ rapidjson::Document root;
+ root.SetObject();
+ rapidjson::Document required_rowsets_arr;
+ required_rowsets_arr.SetArray();
+ rapidjson::Document missing_rowsets_arr;
+ missing_rowsets_arr.SetArray();
+
+ if (rowsets != nullptr) {
+ for (const auto& rowset : *rowsets) {
+ rapidjson::Value value;
+ std::string version_str = rowset->get_rowset_info_str();
+ value.SetString(version_str.c_str(), version_str.length(),
+ required_rowsets_arr.GetAllocator());
+ required_rowsets_arr.PushBack(value, required_rowsets_arr.GetAllocator());
+ }
+ } else {
+ std::vector<RowsetSharedPtr> rowsets;
+ {
+ std::shared_lock meta_rlock(_meta_lock);
+ rowsets = get_rowset_by_ids(&rowset_ids);
+ }
+ for (const auto& rowset : rowsets) {
+ rapidjson::Value value;
+ std::string version_str = rowset->get_rowset_info_str();
+ value.SetString(version_str.c_str(), version_str.length(),
+ required_rowsets_arr.GetAllocator());
+ required_rowsets_arr.PushBack(value, required_rowsets_arr.GetAllocator());
+ }
+ }
+ for (const auto& missing_rowset_id : missing_ids) {
+ rapidjson::Value miss_value;
+ std::string rowset_id_str = missing_rowset_id.to_string();
+ miss_value.SetString(rowset_id_str.c_str(), rowset_id_str.length(),
+ missing_rowsets_arr.GetAllocator());
+ missing_rowsets_arr.PushBack(miss_value, missing_rowsets_arr.GetAllocator());
+ }
+
+ root.AddMember("required_rowsets", required_rowsets_arr, root.GetAllocator());
+ root.AddMember("missing_rowsets", missing_rowsets_arr, root.GetAllocator());
+ rapidjson::StringBuffer strbuf;
+ rapidjson::PrettyWriter<rapidjson::StringBuffer> writer(strbuf);
+ root.Accept(writer);
+ std::string rowset_status_string = std::string(strbuf.GetString());
+ LOG_EVERY_SECOND(WARNING) << rowset_status_string;
+ // let it crash if correctness check failed in Debug mode
+ DCHECK(false) << "delete bitmap correctness check failed in publish phase!";
+ return Status::InternalError("check delete bitmap failed!");
+ }
+ return Status::OK();
+}
+
+void BaseTablet::_remove_sentinel_mark_from_delete_bitmap(DeleteBitmapPtr delete_bitmap) {
+ for (auto it = delete_bitmap->delete_bitmap.begin(), end = delete_bitmap->delete_bitmap.end();
+ it != end;) {
+ if (std::get<1>(it->first) == DeleteBitmap::INVALID_SEGMENT_ID) {
+ it = delete_bitmap->delete_bitmap.erase(it);
+ } else {
+ ++it;
+ }
+ }
+}
+
+Status BaseTablet::update_delete_bitmap(const BaseTabletSPtr& self, const TabletTxnInfo* txn_info,
Review Comment:
warning: function 'update_delete_bitmap' has cognitive complexity of 56 (threshold 50) [readability-function-cognitive-complexity]
```cpp
Status BaseTablet::update_delete_bitmap(const BaseTabletSPtr& self, const TabletTxnInfo* txn_info,
^
```
<details>
<summary>Additional context</summary>
**be/src/olap/base_tablet.cpp:1162:** +1, including nesting penalty of 0, nesting level increased to 1
```cpp
if (txn_info->partial_update_info && txn_info->partial_update_info->is_partial_update) {
^
```
**be/src/olap/base_tablet.cpp:1162:** +1
```cpp
if (txn_info->partial_update_info && txn_info->partial_update_info->is_partial_update) {
^
```
**be/src/olap/base_tablet.cpp:1168:** +1, including nesting penalty of 0, nesting level increased to 1
```cpp
RETURN_IF_ERROR(std::dynamic_pointer_cast<BetaRowset>(rowset)->load_segments(&segments));
^
```
**be/src/common/status.h:541:** expanded from macro 'RETURN_IF_ERROR'
```cpp
do { \
^
```
**be/src/olap/base_tablet.cpp:1168:** +2, including nesting penalty of 1, nesting level increased to 2
```cpp
RETURN_IF_ERROR(std::dynamic_pointer_cast<BetaRowset>(rowset)->load_segments(&segments));
^
```
**be/src/common/status.h:543:** expanded from macro 'RETURN_IF_ERROR'
```cpp
if (UNLIKELY(!_status_.ok())) { \
^
```
**be/src/olap/base_tablet.cpp:1174:** +1, including nesting penalty of 0, nesting level increased to 1
```cpp
if (self->tablet_state() == TABLET_NOTREADY) {
^
```
**be/src/olap/base_tablet.cpp:1179:** +1, including nesting penalty of 0, nesting level increased to 1
```cpp
RETURN_IF_ERROR(self->get_all_rs_id_unlocked(cur_version - 1, &cur_rowset_ids));
^
```
**be/src/common/status.h:541:** expanded from macro 'RETURN_IF_ERROR'
```cpp
do { \
^
```
**be/src/olap/base_tablet.cpp:1179:** +2, including nesting penalty of 1, nesting level increased to 2
```cpp
RETURN_IF_ERROR(self->get_all_rs_id_unlocked(cur_version - 1, &cur_rowset_ids));
^
```
**be/src/common/status.h:543:** expanded from macro 'RETURN_IF_ERROR'
```cpp
if (UNLIKELY(!_status_.ok())) { \
^
```
**be/src/olap/base_tablet.cpp:1198:** +1, including nesting penalty of 0, nesting level increased to 1
```cpp
if (segments.size() <= 1) {
^
```
**be/src/olap/base_tablet.cpp:1199:** +2, including nesting penalty of 1, nesting level increased to 2
```cpp
RETURN_IF_ERROR(calc_delete_bitmap(self, rowset, segments, specified_rowsets, delete_bitmap,
^
```
**be/src/common/status.h:541:** expanded from macro 'RETURN_IF_ERROR'
```cpp
do { \
^
```
**be/src/olap/base_tablet.cpp:1199:** +3, including nesting penalty of 2, nesting level increased to 3
```cpp
RETURN_IF_ERROR(calc_delete_bitmap(self, rowset, segments, specified_rowsets, delete_bitmap,
^
```
**be/src/common/status.h:543:** expanded from macro 'RETURN_IF_ERROR'
```cpp
if (UNLIKELY(!_status_.ok())) { \
^
```
**be/src/olap/base_tablet.cpp:1202:** +1, nesting level increased to 1
```cpp
} else {
^
```
**be/src/olap/base_tablet.cpp:1204:** +2, including nesting penalty of 1, nesting level increased to 2
```cpp
RETURN_IF_ERROR(calc_delete_bitmap(self, rowset, segments, specified_rowsets, delete_bitmap,
^
```
**be/src/common/status.h:541:** expanded from macro 'RETURN_IF_ERROR'
```cpp
do { \
^
```
**be/src/olap/base_tablet.cpp:1204:** +3, including nesting penalty of 2, nesting level increased to 3
```cpp
RETURN_IF_ERROR(calc_delete_bitmap(self, rowset, segments, specified_rowsets, delete_bitmap,
^
```
**be/src/common/status.h:543:** expanded from macro 'RETURN_IF_ERROR'
```cpp
if (UNLIKELY(!_status_.ok())) { \
^
```
**be/src/olap/base_tablet.cpp:1206:** +2, including nesting penalty of 1, nesting level increased to 2
```cpp
RETURN_IF_ERROR(token->wait());
^
```
**be/src/common/status.h:541:** expanded from macro 'RETURN_IF_ERROR'
```cpp
do { \
^
```
**be/src/olap/base_tablet.cpp:1206:** +3, including nesting penalty of 2, nesting level increased to 3
```cpp
RETURN_IF_ERROR(token->wait());
^
```
**be/src/common/status.h:543:** expanded from macro 'RETURN_IF_ERROR'
```cpp
if (UNLIKELY(!_status_.ok())) { \
^
```
**be/src/olap/base_tablet.cpp:1210:** +1, including nesting penalty of 0, nesting level increased to 1
```cpp
if (watch.get_elapse_time_us() < 1 * 1000 * 1000) {
^
```
**be/src/olap/base_tablet.cpp:1212:** +1, nesting level increased to 1
```cpp
} else {
^
```
**be/src/olap/base_tablet.cpp:1220:** nesting level increased to 1
```cpp
[](size_t sum, const segment_v2::SegmentSharedPtr& s) { return sum += s->num_rows(); });
^
```
**be/src/olap/base_tablet.cpp:1227:** +1, including nesting penalty of 0, nesting level increased to 1
```cpp
if (config::enable_merge_on_write_correctness_check && rowset->num_rows() != 0) {
^
```
**be/src/olap/base_tablet.cpp:1227:** +1
```cpp
if (config::enable_merge_on_write_correctness_check && rowset->num_rows() != 0) {
^
```
**be/src/olap/base_tablet.cpp:1232:** +2, including nesting penalty of 1, nesting level increased to 2
```cpp
if (!st.ok()) {
^
```
**be/src/olap/base_tablet.cpp:1238:** +1, including nesting penalty of 0, nesting level increased to 1
```cpp
if (txn_info->partial_update_info && txn_info->partial_update_info->is_partial_update) {
^
```
**be/src/olap/base_tablet.cpp:1238:** +1
```cpp
if (txn_info->partial_update_info && txn_info->partial_update_info->is_partial_update) {
^
```
**be/src/olap/base_tablet.cpp:1239:** +2, including nesting penalty of 1, nesting level increased to 2
```cpp
DBUG_EXECUTE_IF("Tablet.update_delete_bitmap.partial_update_write_rowset_fail", {
^
```
**be/src/util/debug_points.h:34:** expanded from macro 'DBUG_EXECUTE_IF'
```cpp
if (UNLIKELY(config::enable_debug_points)) { \
^
```
**be/src/olap/base_tablet.cpp:1239:** +3, including nesting penalty of 2, nesting level increased to 3
```cpp
DBUG_EXECUTE_IF("Tablet.update_delete_bitmap.partial_update_write_rowset_fail", {
^
```
**be/src/util/debug_points.h:36:** expanded from macro 'DBUG_EXECUTE_IF'
```cpp
if (dp) { \
^
```
**be/src/olap/base_tablet.cpp:1240:** +4, including nesting penalty of 3, nesting level increased to 4
```cpp
if (rand() % 100 < (100 * dp->param("percent", 0.5))) {
^
```
**be/src/olap/base_tablet.cpp:1248:** +2, including nesting penalty of 1, nesting level increased to 2
```cpp
RETURN_IF_ERROR(rowset_writer->flush());
^
```
**be/src/common/status.h:541:** expanded from macro 'RETURN_IF_ERROR'
```cpp
do { \
^
```
**be/src/olap/base_tablet.cpp:1248:** +3, including nesting penalty of 2, nesting level increased to 3
```cpp
RETURN_IF_ERROR(rowset_writer->flush());
^
```
**be/src/common/status.h:543:** expanded from macro 'RETURN_IF_ERROR'
```cpp
if (UNLIKELY(!_status_.ok())) { \
^
```
**be/src/olap/base_tablet.cpp:1250:** +2, including nesting penalty of 1, nesting level increased to 2
```cpp
RETURN_IF_ERROR(rowset_writer->build(transient_rowset));
^
```
**be/src/common/status.h:541:** expanded from macro 'RETURN_IF_ERROR'
```cpp
do { \
^
```
**be/src/olap/base_tablet.cpp:1250:** +3, including nesting penalty of 2, nesting level increased to 3
```cpp
RETURN_IF_ERROR(rowset_writer->build(transient_rowset));
^
```
**be/src/common/status.h:543:** expanded from macro 'RETURN_IF_ERROR'
```cpp
if (UNLIKELY(!_status_.ok())) { \
^
```
**be/src/olap/base_tablet.cpp:1257:** +1, including nesting penalty of 0, nesting level increased to 1
```cpp
RETURN_IF_ERROR(self->save_delete_bitmap(txn_info, txn_id, delete_bitmap, rowset_writer.get(),
^
```
**be/src/common/status.h:541:** expanded from macro 'RETURN_IF_ERROR'
```cpp
do { \
^
```
**be/src/olap/base_tablet.cpp:1257:** +2, including nesting penalty of 1, nesting level increased to 2
```cpp
RETURN_IF_ERROR(self->save_delete_bitmap(txn_info, txn_id, delete_bitmap, rowset_writer.get(),
^
```
**be/src/common/status.h:543:** expanded from macro 'RETURN_IF_ERROR'
```cpp
if (UNLIKELY(!_status_.ok())) { \
^
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "liaoxin01 (via GitHub)" <gi...@apache.org>.
liaoxin01 commented on PR #31317:
URL: https://github.com/apache/doris/pull/31317#issuecomment-1973024534
run buildall
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "doris-robot (via GitHub)" <gi...@apache.org>.
doris-robot commented on PR #31317:
URL: https://github.com/apache/doris/pull/31317#issuecomment-1971391901
TeamCity be ut coverage result:
Function Coverage: 35.61% (8549/24007)
Line Coverage: 27.41% (69373/253057)
Region Coverage: 26.58% (35985/135378)
Branch Coverage: 23.41% (18392/78572)
Coverage Report: http://coverage.selectdb-in.cc/coverage/d9c2af4561e15f527a03a81ce52c99aef371d66e_d9c2af4561e15f527a03a81ce52c99aef371d66e/report/index.html
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "liaoxin01 (via GitHub)" <gi...@apache.org>.
liaoxin01 commented on PR #31317:
URL: https://github.com/apache/doris/pull/31317#issuecomment-1964235068
run buildall
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "liaoxin01 (via GitHub)" <gi...@apache.org>.
liaoxin01 commented on PR #31317:
URL: https://github.com/apache/doris/pull/31317#issuecomment-1961161410
run buildall
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "doris-robot (via GitHub)" <gi...@apache.org>.
doris-robot commented on PR #31317:
URL: https://github.com/apache/doris/pull/31317#issuecomment-1968741585
<details>
<summary>TPC-DS: <b>Total hot run time: 169427 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpcds-tools
TPC-DS sf100 test result on commit 79b90d0e6d1ee594376d460be1d722f84c24b1e8, data reload: false
query1 932 338 346 338
query2 6524 1739 1829 1739
query3 6705 210 209 209
query4 23339 20584 20475 20475
query5 4244 357 357 357
query6 264 173 169 169
query7 4599 302 303 302
query8 268 191 184 184
query9 8393 2251 2252 2251
query10 422 225 215 215
query11 14672 14391 14121 14121
query12 130 83 84 83
query13 1658 433 432 432
query14 8835 6473 6437 6437
query15 206 175 177 175
query16 7086 264 253 253
query17 1032 523 518 518
query18 1912 265 262 262
query19 192 152 150 150
query20 85 83 82 82
query21 199 127 115 115
query22 4762 4607 4564 4564
query23 30659 29929 29840 29840
query24 11962 3138 3060 3060
query25 655 360 346 346
query26 1841 155 155 155
query27 3020 309 314 309
query28 7002 1822 1798 1798
query29 1163 573 572 572
query30 290 130 139 130
query31 871 702 694 694
query32 99 59 63 59
query33 728 231 215 215
query34 1046 489 489 489
query35 866 754 755 754
query36 898 904 838 838
query37 126 62 62 62
query38 3071 2968 2959 2959
query39 1319 1261 1247 1247
query40 276 102 97 97
query41 38 38 35 35
query42 104 97 102 97
query43 465 433 431 431
query44 1065 701 714 701
query45 192 178 175 175
query46 1043 803 810 803
query47 1565 1469 1465 1465
query48 422 352 348 348
query49 1174 288 297 288
query50 771 378 382 378
query51 4380 4287 4293 4287
query52 104 91 92 91
query53 343 270 270 270
query54 302 233 224 224
query55 89 79 85 79
query56 213 201 191 191
query57 949 883 885 883
query58 207 199 189 189
query59 2228 2149 2105 2105
query60 231 214 207 207
query61 83 82 86 82
query62 595 364 357 357
query63 277 263 271 263
query64 6578 3027 3112 3027
query65 3242 3236 3210 3210
query66 1441 326 316 316
query67 14338 14257 13985 13985
query68 4459 577 568 568
query69 535 383 379 379
query70 1224 1203 1254 1203
query71 329 251 247 247
query72 6034 2808 2646 2646
query73 708 325 323 323
query74 6343 6046 6012 6012
query75 3096 2448 2400 2400
query76 2660 1085 1175 1085
query77 342 240 241 240
query78 8884 8627 8402 8402
query79 1296 547 546 546
query80 1003 369 342 342
query81 469 197 198 197
query82 951 81 86 81
query83 227 123 120 120
query84 286 81 78 78
query85 1327 357 345 345
query86 398 300 308 300
query87 3257 3104 3095 3095
query88 2830 2391 2371 2371
query89 409 336 342 336
query90 1839 168 169 168
query91 156 126 123 123
query92 54 49 47 47
query93 1201 509 504 504
query94 1235 178 179 178
query95 440 339 342 339
query96 575 267 274 267
query97 4294 4119 4170 4119
query98 218 197 196 196
query99 1129 677 628 628
Total cold run time: 262998 ms
Total hot run time: 169427 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "doris-robot (via GitHub)" <gi...@apache.org>.
doris-robot commented on PR #31317:
URL: https://github.com/apache/doris/pull/31317#issuecomment-1966814607
<details>
<summary>ClickBench: <b>Total hot run time: 32.3 s</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/clickbench-tools
ClickBench test result on commit ee84a0fb4b4a31f62ca947631bdc7ceaf39e9833, data reload: false
query1 0.04 0.03 0.04
query2 0.06 0.03 0.02
query3 0.24 0.06 0.06
query4 1.66 0.10 0.10
query5 0.52 0.52 0.52
query6 1.29 0.66 0.71
query7 0.02 0.01 0.01
query8 0.04 0.03 0.03
query9 0.56 0.52 0.51
query10 0.58 0.59 0.57
query11 0.12 0.10 0.09
query12 0.13 0.10 0.10
query13 0.62 0.62 0.61
query14 0.79 0.81 0.80
query15 0.84 0.82 0.82
query16 0.40 0.37 0.40
query17 1.00 0.97 1.01
query18 0.27 0.24 0.24
query19 1.80 1.74 1.68
query20 0.02 0.01 0.01
query21 15.40 0.64 0.65
query22 3.26 3.42 2.56
query23 17.86 1.04 0.94
query24 2.01 0.56 0.37
query25 0.46 0.16 0.05
query26 0.17 0.15 0.16
query27 0.06 0.06 0.04
query28 12.04 0.83 0.84
query29 12.50 3.28 3.23
query30 0.60 0.55 0.52
query31 2.79 0.34 0.35
query32 3.36 0.48 0.48
query33 3.20 3.18 3.20
query34 15.36 4.73 4.73
query35 4.74 4.76 4.73
query36 1.14 1.07 1.07
query37 0.06 0.04 0.05
query38 0.05 0.03 0.02
query39 0.02 0.01 0.02
query40 0.17 0.13 0.13
query41 0.08 0.02 0.01
query42 0.02 0.01 0.02
query43 0.03 0.02 0.02
Total cold run time: 106.38 s
Total hot run time: 32.3 s
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "liaoxin01 (via GitHub)" <gi...@apache.org>.
liaoxin01 commented on PR #31317:
URL: https://github.com/apache/doris/pull/31317#issuecomment-1968582791
run buildall
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "doris-robot (via GitHub)" <gi...@apache.org>.
doris-robot commented on PR #31317:
URL: https://github.com/apache/doris/pull/31317#issuecomment-1969079585
Load test result on machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
```
Load test result on commit 0a23e751291ddf15970f07c0384db0ab6c5606aa with default session variables
Stream load json: 19 seconds loaded 2358488459 Bytes, about 118 MB/s
Stream load orc: 59 seconds loaded 1101869774 Bytes, about 17 MB/s
Stream load parquet: 32 seconds loaded 861443392 Bytes, about 25 MB/s
Insert into select: 18.0 seconds inserted 10000000 Rows, about 555K ops/s
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "doris-robot (via GitHub)" <gi...@apache.org>.
doris-robot commented on PR #31317:
URL: https://github.com/apache/doris/pull/31317#issuecomment-1970380823
<details>
<summary>ClickBench: <b>Total hot run time: 32.61 s</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/clickbench-tools
ClickBench test result on commit ced38f8389a29b7ffe8a5684b92d24002d3a26e8, data reload: false
query1 0.04 0.03 0.03
query2 0.06 0.02 0.02
query3 0.24 0.06 0.07
query4 1.67 0.10 0.10
query5 0.52 0.52 0.51
query6 1.29 0.68 0.65
query7 0.02 0.02 0.01
query8 0.04 0.02 0.02
query9 0.55 0.50 0.52
query10 0.57 0.57 0.57
query11 0.12 0.09 0.10
query12 0.13 0.10 0.10
query13 0.63 0.61 0.62
query14 0.79 0.82 0.80
query15 0.84 0.83 0.82
query16 0.36 0.37 0.39
query17 1.02 0.98 0.95
query18 0.27 0.25 0.25
query19 1.83 1.78 1.70
query20 0.02 0.01 0.01
query21 15.41 0.67 0.66
query22 2.55 3.99 2.70
query23 17.41 1.10 0.94
query24 2.14 0.37 0.58
query25 0.29 0.06 0.03
query26 0.19 0.15 0.16
query27 0.04 0.03 0.03
query28 12.12 0.84 0.83
query29 12.56 3.33 3.53
query30 0.58 0.56 0.54
query31 2.80 0.35 0.36
query32 3.33 0.47 0.48
query33 3.20 3.24 3.24
query34 15.37 4.73 4.75
query35 4.78 4.74 4.74
query36 1.13 1.07 1.07
query37 0.06 0.04 0.04
query38 0.04 0.02 0.02
query39 0.02 0.02 0.02
query40 0.16 0.14 0.13
query41 0.07 0.02 0.02
query42 0.02 0.02 0.01
query43 0.03 0.02 0.02
Total cold run time: 105.31 s
Total hot run time: 32.61 s
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "liaoxin01 (via GitHub)" <gi...@apache.org>.
liaoxin01 commented on PR #31317:
URL: https://github.com/apache/doris/pull/31317#issuecomment-1970509270
run buildall
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "doris-robot (via GitHub)" <gi...@apache.org>.
doris-robot commented on PR #31317:
URL: https://github.com/apache/doris/pull/31317#issuecomment-1970383253
Load test result on machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
```
Load test result on commit ced38f8389a29b7ffe8a5684b92d24002d3a26e8 with default session variables
Stream load json: 18 seconds loaded 2358488459 Bytes, about 124 MB/s
Stream load orc: 58 seconds loaded 1101869774 Bytes, about 18 MB/s
Stream load parquet: 31 seconds loaded 861443392 Bytes, about 26 MB/s
Insert into select: 18.2 seconds inserted 10000000 Rows, about 549K ops/s
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "liaoxin01 (via GitHub)" <gi...@apache.org>.
liaoxin01 commented on PR #31317:
URL: https://github.com/apache/doris/pull/31317#issuecomment-1971109000
run buildall
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "github-actions[bot] (via GitHub)" <gi...@apache.org>.
github-actions[bot] commented on code in PR #31317:
URL: https://github.com/apache/doris/pull/31317#discussion_r1504301059
##########
be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp:
##########
@@ -0,0 +1,162 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "cloud/cloud_engine_calc_delete_bitmap_task.h"
+
+#include <memory>
+
+#include "cloud/cloud_meta_mgr.h"
+#include "cloud/cloud_tablet.h"
+#include "common/status.h"
+#include "olap/base_tablet.h"
+#include "olap/olap_common.h"
+#include "olap/rowset/rowset.h"
+#include "olap/tablet_fwd.h"
+#include "olap/tablet_meta.h"
+#include "olap/utils.h"
+
+namespace doris {
+
+CloudEngineCalcDeleteBitmapTask::CloudEngineCalcDeleteBitmapTask(
+ CloudStorageEngine& engine, const TCalcDeleteBitmapRequest& cal_delete_bitmap_req,
+ std::vector<TTabletId>* error_tablet_ids, std::vector<TTabletId>* succ_tablet_ids)
+ : _engine(engine),
+ _cal_delete_bitmap_req(cal_delete_bitmap_req),
+ _error_tablet_ids(error_tablet_ids),
+ _succ_tablet_ids(succ_tablet_ids) {}
+
+void CloudEngineCalcDeleteBitmapTask::add_error_tablet_id(int64_t tablet_id, const Status& err) {
Review Comment:
warning: method 'add_error_tablet_id' can be made static [readability-convert-member-functions-to-static]
be/src/cloud/cloud_engine_calc_delete_bitmap_task.h:57:
```diff
- void add_error_tablet_id(int64_t tablet_id, const Status& err);
+ static void add_error_tablet_id(int64_t tablet_id, const Status& err);
```
##########
be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp:
##########
@@ -0,0 +1,162 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "cloud/cloud_engine_calc_delete_bitmap_task.h"
+
+#include <memory>
+
+#include "cloud/cloud_meta_mgr.h"
+#include "cloud/cloud_tablet.h"
+#include "common/status.h"
+#include "olap/base_tablet.h"
+#include "olap/olap_common.h"
+#include "olap/rowset/rowset.h"
+#include "olap/tablet_fwd.h"
+#include "olap/tablet_meta.h"
+#include "olap/utils.h"
+
+namespace doris {
+
+CloudEngineCalcDeleteBitmapTask::CloudEngineCalcDeleteBitmapTask(
+ CloudStorageEngine& engine, const TCalcDeleteBitmapRequest& cal_delete_bitmap_req,
+ std::vector<TTabletId>* error_tablet_ids, std::vector<TTabletId>* succ_tablet_ids)
+ : _engine(engine),
+ _cal_delete_bitmap_req(cal_delete_bitmap_req),
+ _error_tablet_ids(error_tablet_ids),
+ _succ_tablet_ids(succ_tablet_ids) {}
+
+void CloudEngineCalcDeleteBitmapTask::add_error_tablet_id(int64_t tablet_id, const Status& err) {
+ std::lock_guard<std::mutex> lck(_mutex);
+ _error_tablet_ids->push_back(tablet_id);
+ if (_res.ok() || _res.is<ErrorCode::DELETE_BITMAP_LOCK_ERROR>()) {
+ _res = err;
+ }
+}
+
+void CloudEngineCalcDeleteBitmapTask::add_succ_tablet_id(int64_t tablet_id) {
Review Comment:
warning: method 'add_succ_tablet_id' can be made static [readability-convert-member-functions-to-static]
be/src/cloud/cloud_engine_calc_delete_bitmap_task.h:58:
```diff
- void add_succ_tablet_id(int64_t tablet_id);
+ static void add_succ_tablet_id(int64_t tablet_id);
```
##########
be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp:
##########
@@ -0,0 +1,162 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "cloud/cloud_engine_calc_delete_bitmap_task.h"
+
+#include <memory>
+
+#include "cloud/cloud_meta_mgr.h"
+#include "cloud/cloud_tablet.h"
+#include "common/status.h"
+#include "olap/base_tablet.h"
+#include "olap/olap_common.h"
+#include "olap/rowset/rowset.h"
+#include "olap/tablet_fwd.h"
+#include "olap/tablet_meta.h"
+#include "olap/utils.h"
+
+namespace doris {
+
+CloudEngineCalcDeleteBitmapTask::CloudEngineCalcDeleteBitmapTask(
+ CloudStorageEngine& engine, const TCalcDeleteBitmapRequest& cal_delete_bitmap_req,
+ std::vector<TTabletId>* error_tablet_ids, std::vector<TTabletId>* succ_tablet_ids)
+ : _engine(engine),
+ _cal_delete_bitmap_req(cal_delete_bitmap_req),
+ _error_tablet_ids(error_tablet_ids),
+ _succ_tablet_ids(succ_tablet_ids) {}
+
+void CloudEngineCalcDeleteBitmapTask::add_error_tablet_id(int64_t tablet_id, const Status& err) {
+ std::lock_guard<std::mutex> lck(_mutex);
+ _error_tablet_ids->push_back(tablet_id);
+ if (_res.ok() || _res.is<ErrorCode::DELETE_BITMAP_LOCK_ERROR>()) {
+ _res = err;
+ }
+}
+
+void CloudEngineCalcDeleteBitmapTask::add_succ_tablet_id(int64_t tablet_id) {
+ std::lock_guard<std::mutex> lck(_mutex);
+ _succ_tablet_ids->push_back(tablet_id);
+}
+
+Status CloudEngineCalcDeleteBitmapTask::execute() {
+ int64_t transaction_id = _cal_delete_bitmap_req.transaction_id;
+ OlapStopWatch watch;
+ VLOG_NOTICE << "begin to calculate delete bitmap. transaction_id=" << transaction_id;
+ std::unique_ptr<ThreadPoolToken> token =
+ _engine.calc_tablet_delete_bitmap_task_thread_pool()->new_token(
+ ThreadPool::ExecutionMode::CONCURRENT);
+
+ for (const auto& partition : _cal_delete_bitmap_req.partitions) {
+ int64_t version = partition.version;
+ for (auto tablet_id : partition.tablet_ids) {
+ auto base_tablet = DORIS_TRY(_engine.get_tablet(tablet_id));
+ std::shared_ptr<CloudTablet> tablet =
+ std::dynamic_pointer_cast<CloudTablet>(base_tablet);
+ if (tablet == nullptr) {
+ LOG(WARNING) << "can't get tablet when calculate delete bitmap. tablet_id="
+ << tablet_id;
+ _error_tablet_ids->push_back(tablet_id);
+ _res = Status::Error<ErrorCode::PUSH_TABLE_NOT_EXIST>(
+ "can't get tablet when calculate delete bitmap. tablet_id={}", tablet_id);
+ break;
+ }
+
+ Status st = tablet->sync_rowsets();
+ if (!st.ok() && !st.is<ErrorCode::INVALID_TABLET_STATE>()) {
+ return st;
+ }
+ if (st.is<ErrorCode::INVALID_TABLET_STATE>()) [[unlikely]] {
+ add_succ_tablet_id(tablet->tablet_id());
+ LOG(INFO)
+ << "tablet is under alter process, delete bitmap will be calculated later, "
+ "tablet_id: "
+ << tablet->tablet_id() << " txn_id: " << transaction_id
+ << ", request_version=" << version;
+ continue;
+ }
+ int64_t max_version = tablet->max_version_unlocked();
+ if (version != max_version + 1) {
+ _error_tablet_ids->push_back(tablet_id);
+ _res = Status::Error<ErrorCode::DELETE_BITMAP_LOCK_ERROR, false>(
+ "version not continuous");
+ LOG(WARNING) << "version not continuous, current max version=" << max_version
+ << ", request_version=" << version
+ << " tablet_id=" << tablet->tablet_id();
+ break;
+ }
+
+ auto tablet_calc_delete_bitmap_ptr = std::make_shared<CloudTabletCalcDeleteBitmapTask>(
+ _engine, this, tablet, transaction_id, version);
+ auto submit_st = token->submit_func([=]() { tablet_calc_delete_bitmap_ptr->handle(); });
+ CHECK(submit_st.ok());
+ }
+ }
+ // wait for all finished
+ token->wait();
+
+ LOG(INFO) << "finish to calculate delete bitmap on transaction."
+ << "transaction_id=" << transaction_id << ", cost(us): " << watch.get_elapse_time_us()
+ << ", error_tablet_size=" << _error_tablet_ids->size()
+ << ", res=" << _res.to_string();
+ return _res;
+}
+
+CloudTabletCalcDeleteBitmapTask::CloudTabletCalcDeleteBitmapTask(CloudStorageEngine& engine,
+ CloudEngineCalcDeleteBitmapTask* engine_task,
+ std::shared_ptr<CloudTablet> tablet,
+ int64_t transaction_id, int64_t version)
+ : _engine(engine),
+ _engine_calc_delete_bitmap_task(engine_task),
+ _tablet(tablet),
+ _transaction_id(transaction_id),
+ _version(version) {}
+
+void CloudTabletCalcDeleteBitmapTask::handle() {
Review Comment:
warning: method 'handle' can be made const [readability-make-member-function-const]
```suggestion
void CloudTabletCalcDeleteBitmapTask::handle() const {
```
be/src/cloud/cloud_engine_calc_delete_bitmap_task.h:38:
```diff
- void handle();
+ void handle() const;
```
##########
be/src/cloud/cloud_tablet.cpp:
##########
@@ -443,4 +480,129 @@
_cumulative_point = new_point;
}
+Status CloudTablet::update_delete_bitmap(const std::shared_ptr<CloudTablet>& self,
Review Comment:
warning: function 'update_delete_bitmap' exceeds recommended size/complexity thresholds [readability-function-size]
```cpp
Status CloudTablet::update_delete_bitmap(const std::shared_ptr<CloudTablet>& self,
^
```
<details>
<summary>Additional context</summary>
**be/src/cloud/cloud_tablet.cpp:482:** 118 lines including whitespace and comments (threshold 80)
```cpp
Status CloudTablet::update_delete_bitmap(const std::shared_ptr<CloudTablet>& self,
^
```
</details>
##########
be/src/cloud/cloud_tablet.cpp:
##########
@@ -443,4 +480,129 @@ void CloudTablet::set_cumulative_layer_point(int64_t new_point) {
_cumulative_point = new_point;
}
+Status CloudTablet::update_delete_bitmap(const std::shared_ptr<CloudTablet>& self,
Review Comment:
warning: function 'update_delete_bitmap' has cognitive complexity of 59 (threshold 50) [readability-function-cognitive-complexity]
```cpp
Status CloudTablet::update_delete_bitmap(const std::shared_ptr<CloudTablet>& self,
^
```
<details>
<summary>Additional context</summary>
**be/src/cloud/cloud_tablet.cpp:499:** +1, including nesting penalty of 0, nesting level increased to 1
```cpp
RETURN_IF_ERROR(beta_rowset->load_segments(&segments));
^
```
**be/src/common/status.h:541:** expanded from macro 'RETURN_IF_ERROR'
```cpp
do { \
^
```
**be/src/cloud/cloud_tablet.cpp:499:** +2, including nesting penalty of 1, nesting level increased to 2
```cpp
RETURN_IF_ERROR(beta_rowset->load_segments(&segments));
^
```
**be/src/common/status.h:543:** expanded from macro 'RETURN_IF_ERROR'
```cpp
if (UNLIKELY(!_status_.ok())) { \
^
```
**be/src/cloud/cloud_tablet.cpp:505:** +1, including nesting penalty of 0, nesting level increased to 1
```cpp
if (self->tablet_state() == TABLET_NOTREADY) {
^
```
**be/src/cloud/cloud_tablet.cpp:510:** +1, including nesting penalty of 0, nesting level increased to 1
```cpp
RETURN_IF_ERROR(self->get_all_rs_id_unlocked(cur_version - 1, &cur_rowset_ids));
^
```
**be/src/common/status.h:541:** expanded from macro 'RETURN_IF_ERROR'
```cpp
do { \
^
```
**be/src/cloud/cloud_tablet.cpp:510:** +2, including nesting penalty of 1, nesting level increased to 2
```cpp
RETURN_IF_ERROR(self->get_all_rs_id_unlocked(cur_version - 1, &cur_rowset_ids));
^
```
**be/src/common/status.h:543:** expanded from macro 'RETURN_IF_ERROR'
```cpp
if (UNLIKELY(!_status_.ok())) { \
^
```
**be/src/cloud/cloud_tablet.cpp:529:** +1, including nesting penalty of 0, nesting level increased to 1
```cpp
if (segments.size() <= 1) {
^
```
**be/src/cloud/cloud_tablet.cpp:530:** +2, including nesting penalty of 1, nesting level increased to 2
```cpp
RETURN_IF_ERROR(calc_delete_bitmap(self, rowset, segments, specified_rowsets, delete_bitmap,
^
```
**be/src/common/status.h:541:** expanded from macro 'RETURN_IF_ERROR'
```cpp
do { \
^
```
**be/src/cloud/cloud_tablet.cpp:530:** +3, including nesting penalty of 2, nesting level increased to 3
```cpp
RETURN_IF_ERROR(calc_delete_bitmap(self, rowset, segments, specified_rowsets, delete_bitmap,
^
```
**be/src/common/status.h:543:** expanded from macro 'RETURN_IF_ERROR'
```cpp
if (UNLIKELY(!_status_.ok())) { \
^
```
**be/src/cloud/cloud_tablet.cpp:533:** +1, nesting level increased to 1
```cpp
} else {
^
```
**be/src/cloud/cloud_tablet.cpp:535:** +2, including nesting penalty of 1, nesting level increased to 2
```cpp
RETURN_IF_ERROR(calc_delete_bitmap(self, rowset, segments, specified_rowsets, delete_bitmap,
^
```
**be/src/common/status.h:541:** expanded from macro 'RETURN_IF_ERROR'
```cpp
do { \
^
```
**be/src/cloud/cloud_tablet.cpp:535:** +3, including nesting penalty of 2, nesting level increased to 3
```cpp
RETURN_IF_ERROR(calc_delete_bitmap(self, rowset, segments, specified_rowsets, delete_bitmap,
^
```
**be/src/common/status.h:543:** expanded from macro 'RETURN_IF_ERROR'
```cpp
if (UNLIKELY(!_status_.ok())) { \
^
```
**be/src/cloud/cloud_tablet.cpp:537:** +2, including nesting penalty of 1, nesting level increased to 2
```cpp
RETURN_IF_ERROR(token->wait());
^
```
**be/src/common/status.h:541:** expanded from macro 'RETURN_IF_ERROR'
```cpp
do { \
^
```
**be/src/cloud/cloud_tablet.cpp:537:** +3, including nesting penalty of 2, nesting level increased to 3
```cpp
RETURN_IF_ERROR(token->wait());
^
```
**be/src/common/status.h:543:** expanded from macro 'RETURN_IF_ERROR'
```cpp
if (UNLIKELY(!_status_.ok())) { \
^
```
**be/src/cloud/cloud_tablet.cpp:540:** +1, including nesting penalty of 0, nesting level increased to 1
```cpp
if (watch.get_elapse_time_us() < 1 * 1000 * 1000) {
^
```
**be/src/cloud/cloud_tablet.cpp:542:** +1, nesting level increased to 1
```cpp
} else {
^
```
**be/src/cloud/cloud_tablet.cpp:550:** nesting level increased to 1
```cpp
[](size_t sum, const segment_v2::SegmentSharedPtr& s) { return sum += s->num_rows(); });
^
```
**be/src/cloud/cloud_tablet.cpp:561:** +1, including nesting penalty of 0, nesting level increased to 1
```cpp
if (config::enable_merge_on_write_correctness_check && rowset->num_rows() != 0) {
^
```
**be/src/cloud/cloud_tablet.cpp:566:** +2, including nesting penalty of 1, nesting level increased to 2
```cpp
if (!st.ok()) {
^
```
**be/src/cloud/cloud_tablet.cpp:572:** +1, including nesting penalty of 0, nesting level increased to 1
```cpp
if (partial_update_info && partial_update_info->is_partial_update &&
^
```
**be/src/cloud/cloud_tablet.cpp:572:** +1
```cpp
if (partial_update_info && partial_update_info->is_partial_update &&
^
```
**be/src/cloud/cloud_tablet.cpp:574:** +2, including nesting penalty of 1, nesting level increased to 2
```cpp
DBUG_EXECUTE_IF("CloudTablet.update_delete_bitmap.partial_update_write_rowset_fail", {
^
```
**be/src/util/debug_points.h:34:** expanded from macro 'DBUG_EXECUTE_IF'
```cpp
if (UNLIKELY(config::enable_debug_points)) { \
^
```
**be/src/cloud/cloud_tablet.cpp:574:** +3, including nesting penalty of 2, nesting level increased to 3
```cpp
DBUG_EXECUTE_IF("CloudTablet.update_delete_bitmap.partial_update_write_rowset_fail", {
^
```
**be/src/util/debug_points.h:36:** expanded from macro 'DBUG_EXECUTE_IF'
```cpp
if (dp) { \
^
```
**be/src/cloud/cloud_tablet.cpp:575:** +4, including nesting penalty of 3, nesting level increased to 4
```cpp
if (rand() % 100 < (100 * dp->param("percent", 0.5))) {
^
```
**be/src/cloud/cloud_tablet.cpp:583:** +2, including nesting penalty of 1, nesting level increased to 2
```cpp
RETURN_IF_ERROR(rowset_writer->flush());
^
```
**be/src/common/status.h:541:** expanded from macro 'RETURN_IF_ERROR'
```cpp
do { \
^
```
**be/src/cloud/cloud_tablet.cpp:583:** +3, including nesting penalty of 2, nesting level increased to 3
```cpp
RETURN_IF_ERROR(rowset_writer->flush());
^
```
**be/src/common/status.h:543:** expanded from macro 'RETURN_IF_ERROR'
```cpp
if (UNLIKELY(!_status_.ok())) { \
^
```
**be/src/cloud/cloud_tablet.cpp:585:** +2, including nesting penalty of 1, nesting level increased to 2
```cpp
RETURN_IF_ERROR(rowset_writer->build(transient_rowset));
^
```
**be/src/common/status.h:541:** expanded from macro 'RETURN_IF_ERROR'
```cpp
do { \
^
```
**be/src/cloud/cloud_tablet.cpp:585:** +3, including nesting penalty of 2, nesting level increased to 3
```cpp
RETURN_IF_ERROR(rowset_writer->build(transient_rowset));
^
```
**be/src/common/status.h:543:** expanded from macro 'RETURN_IF_ERROR'
```cpp
if (UNLIKELY(!_status_.ok())) { \
^
```
**be/src/cloud/cloud_tablet.cpp:588:** +2, including nesting penalty of 1, nesting level increased to 2
```cpp
RETURN_IF_ERROR(self->_engine.meta_mgr().update_tmp_rowset(*rowset_meta));
^
```
**be/src/common/status.h:541:** expanded from macro 'RETURN_IF_ERROR'
```cpp
do { \
^
```
**be/src/cloud/cloud_tablet.cpp:588:** +3, including nesting penalty of 2, nesting level increased to 3
```cpp
RETURN_IF_ERROR(self->_engine.meta_mgr().update_tmp_rowset(*rowset_meta));
^
```
**be/src/common/status.h:543:** expanded from macro 'RETURN_IF_ERROR'
```cpp
if (UNLIKELY(!_status_.ok())) { \
^
```
**be/src/cloud/cloud_tablet.cpp:595:** +1, including nesting penalty of 0, nesting level increased to 1
```cpp
for (auto iter = delete_bitmap->delete_bitmap.begin();
^
```
**be/src/cloud/cloud_tablet.cpp:601:** +1, including nesting penalty of 0, nesting level increased to 1
```cpp
RETURN_IF_ERROR(self->_engine.meta_mgr().update_delete_bitmap(
^
```
**be/src/common/status.h:541:** expanded from macro 'RETURN_IF_ERROR'
```cpp
do { \
^
```
**be/src/cloud/cloud_tablet.cpp:601:** +2, including nesting penalty of 1, nesting level increased to 2
```cpp
RETURN_IF_ERROR(self->_engine.meta_mgr().update_delete_bitmap(
^
```
**be/src/common/status.h:543:** expanded from macro 'RETURN_IF_ERROR'
```cpp
if (UNLIKELY(!_status_.ok())) { \
^
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "doris-robot (via GitHub)" <gi...@apache.org>.
doris-robot commented on PR #31317:
URL: https://github.com/apache/doris/pull/31317#issuecomment-1970655389
<details>
<summary>TPC-H: <b>Total hot run time: 37829 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpch-tools
Tpch sf100 test result on commit 855559f1046c84941399efe6b5240367ba209876, data reload: false
------ Round 1 ----------------------------------
q1 17664 4093 4075 4075
q2 2048 138 131 131
q3 10593 944 939 939
q4 4664 938 928 928
q5 7593 2861 3005 2861
q6 181 124 122 122
q7 1235 757 746 746
q8 9250 2015 2039 2015
q9 7245 6327 6291 6291
q10 8203 2506 2482 2482
q11 416 216 205 205
q12 745 306 314 306
q13 17932 3217 3185 3185
q14 278 244 249 244
q15 555 502 493 493
q16 468 393 398 393
q17 935 914 881 881
q18 6727 5962 5991 5962
q19 1585 1513 1522 1513
q20 548 277 272 272
q21 7070 3598 3516 3516
q22 813 279 269 269
Total cold run time: 106748 ms
Total hot run time: 37829 ms
----- Round 2, with runtime_filter_mode=off -----
q1 4005 4008 3990 3990
q2 307 211 216 211
q3 2958 2947 2931 2931
q4 1840 1806 1818 1806
q5 5143 5126 5184 5126
q6 202 114 114 114
q7 2136 1738 1700 1700
q8 3125 3200 3196 3196
q9 8246 8212 8242 8212
q10 6105 3662 3658 3658
q11 530 421 431 421
q12 700 533 535 533
q13 11812 3011 3035 3011
q14 271 262 270 262
q15 549 486 496 486
q16 492 474 444 444
q17 1707 1663 1673 1663
q18 7825 7615 7382 7382
q19 1668 1641 1623 1623
q20 2081 1893 1889 1889
q21 4753 4639 4597 4597
q22 513 452 463 452
Total cold run time: 66968 ms
Total hot run time: 53707 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "liaoxin01 (via GitHub)" <gi...@apache.org>.
liaoxin01 commented on PR #31317:
URL: https://github.com/apache/doris/pull/31317#issuecomment-1971383041
run feut
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "doris-robot (via GitHub)" <gi...@apache.org>.
doris-robot commented on PR #31317:
URL: https://github.com/apache/doris/pull/31317#issuecomment-1968754807
TeamCity be ut coverage result:
Function Coverage: 35.64% (8548/23987)
Line Coverage: 27.44% (69374/252805)
Region Coverage: 26.62% (35976/135155)
Branch Coverage: 23.43% (18386/78470)
Coverage Report: http://coverage.selectdb-in.cc/coverage/79b90d0e6d1ee594376d460be1d722f84c24b1e8_79b90d0e6d1ee594376d460be1d722f84c24b1e8/report/index.html
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "doris-robot (via GitHub)" <gi...@apache.org>.
doris-robot commented on PR #31317:
URL: https://github.com/apache/doris/pull/31317#issuecomment-1968755995
Load test result on machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
```
Load test result on commit 79b90d0e6d1ee594376d460be1d722f84c24b1e8 with default session variables
Stream load json: 19 seconds loaded 2358488459 Bytes, about 118 MB/s
Stream load orc: 59 seconds loaded 1101869774 Bytes, about 17 MB/s
Stream load parquet: 32 seconds loaded 861443392 Bytes, about 25 MB/s
Insert into select: 17.8 seconds inserted 10000000 Rows, about 561K ops/s
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "doris-robot (via GitHub)" <gi...@apache.org>.
doris-robot commented on PR #31317:
URL: https://github.com/apache/doris/pull/31317#issuecomment-1970569402
<details>
<summary>ClickBench: <b>Total hot run time: 32.1 s</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/clickbench-tools
ClickBench test result on commit 2c80b3d53760bb63ae90fcf0ed7a6ee5c67be4a0, data reload: false
query1 0.04 0.03 0.03
query2 0.06 0.02 0.02
query3 0.23 0.06 0.05
query4 1.67 0.10 0.10
query5 0.53 0.52 0.52
query6 1.20 0.70 0.68
query7 0.02 0.01 0.01
query8 0.03 0.02 0.02
query9 0.55 0.51 0.52
query10 0.58 0.55 0.56
query11 0.14 0.10 0.10
query12 0.13 0.11 0.10
query13 0.63 0.61 0.60
query14 0.79 0.81 0.80
query15 0.83 0.82 0.79
query16 0.37 0.40 0.40
query17 0.97 1.05 0.98
query18 0.26 0.25 0.23
query19 1.83 1.78 1.75
query20 0.01 0.02 0.02
query21 15.41 0.61 0.63
query22 3.16 3.43 1.91
query23 17.43 1.03 1.01
query24 2.12 0.59 0.59
query25 0.28 0.06 0.04
query26 0.18 0.14 0.14
query27 0.04 0.03 0.03
query28 11.90 0.85 0.83
query29 12.67 3.37 3.30
query30 0.59 0.53 0.52
query31 2.80 0.35 0.35
query32 3.35 0.49 0.48
query33 3.18 3.22 3.19
query34 15.35 4.75 4.77
query35 4.77 4.75 4.76
query36 1.13 1.07 1.07
query37 0.07 0.05 0.05
query38 0.04 0.03 0.03
query39 0.02 0.01 0.02
query40 0.17 0.14 0.13
query41 0.07 0.01 0.02
query42 0.02 0.02 0.02
query43 0.03 0.02 0.02
Total cold run time: 105.65 s
Total hot run time: 32.1 s
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "doris-robot (via GitHub)" <gi...@apache.org>.
doris-robot commented on PR #31317:
URL: https://github.com/apache/doris/pull/31317#issuecomment-1966821734
Load test result on machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
```
Load test result on commit ee84a0fb4b4a31f62ca947631bdc7ceaf39e9833 with default session variables
Stream load json: 19 seconds loaded 2358488459 Bytes, about 118 MB/s
Stream load orc: 59 seconds loaded 1101869774 Bytes, about 17 MB/s
Stream load parquet: 32 seconds loaded 861443392 Bytes, about 25 MB/s
Insert into select: 17.9 seconds inserted 10000000 Rows, about 558K ops/s
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "github-actions[bot] (via GitHub)" <gi...@apache.org>.
github-actions[bot] commented on code in PR #31317:
URL: https://github.com/apache/doris/pull/31317#discussion_r1502693573
##########
be/src/cloud/cloud_txn_delete_bitmap_cache.cpp:
##########
@@ -0,0 +1,193 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "cloud/cloud_txn_delete_bitmap_cache.h"
+
+#include <fmt/core.h>
+
+#include <chrono>
+#include <memory>
+#include <shared_mutex>
+
+#include "common/status.h"
+#include "common/sync_point.h"
+#include "olap/olap_common.h"
+#include "olap/tablet_meta.h"
+
+namespace doris {
+
+// TODO sweep time
+CloudTxnDeleteBitmapCache::CloudTxnDeleteBitmapCache(size_t size_in_bytes)
+ : LRUCachePolicy(CachePolicy::CacheType::CLOUD_TXN_DELETE_BITMAP_CACHE, size_in_bytes,
+ LRUCacheType::SIZE, config::pk_index_page_cache_stale_sweep_time_sec, 4),
+ _stop_latch(1) {}
+
+CloudTxnDeleteBitmapCache::~CloudTxnDeleteBitmapCache() {
Review Comment:
warning: use '= default' to define a trivial destructor [modernize-use-equals-default]
```cpp
CloudTxnDeleteBitmapCache::~CloudTxnDeleteBitmapCache() {
^
```
##########
be/src/cloud/cloud_txn_delete_bitmap_cache.cpp:
##########
@@ -0,0 +1,193 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "cloud/cloud_txn_delete_bitmap_cache.h"
+
+#include <fmt/core.h>
+
+#include <chrono>
+#include <memory>
+#include <shared_mutex>
+
+#include "common/status.h"
+#include "common/sync_point.h"
+#include "olap/olap_common.h"
+#include "olap/tablet_meta.h"
+
+namespace doris {
+
+// TODO sweep time
+CloudTxnDeleteBitmapCache::CloudTxnDeleteBitmapCache(size_t size_in_bytes)
+ : LRUCachePolicy(CachePolicy::CacheType::CLOUD_TXN_DELETE_BITMAP_CACHE, size_in_bytes,
+ LRUCacheType::SIZE, config::pk_index_page_cache_stale_sweep_time_sec, 4),
+ _stop_latch(1) {}
+
+CloudTxnDeleteBitmapCache::~CloudTxnDeleteBitmapCache() {
+ _stop_latch.count_down();
+ _clean_thread->join();
+}
+
+Status CloudTxnDeleteBitmapCache::init() {
Review Comment:
warning: method 'init' can be made static [readability-convert-member-functions-to-static]
be/src/cloud/cloud_txn_delete_bitmap_cache.h:37:
```diff
- Status init();
+ static Status init();
```
##########
be/src/cloud/cloud_txn_delete_bitmap_cache.h:
##########
@@ -0,0 +1,95 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <mutex>
+
+#include "olap/lru_cache.h"
+#include "olap/olap_common.h"
+#include "olap/partial_update_info.h"
+#include "olap/rowset/rowset.h"
+#include "olap/tablet_meta.h"
+#include "util/countdown_latch.h"
+
+namespace doris {
+
+// Record transaction related delete bitmaps using a lru cache.
+class CloudTxnDeleteBitmapCache : public LRUCachePolicy {
+public:
+ CloudTxnDeleteBitmapCache(size_t size_in_bytes);
+
+ ~CloudTxnDeleteBitmapCache();
Review Comment:
warning: annotate this function with 'override' or (rarely) 'final' [modernize-use-override]
```suggestion
~CloudTxnDeleteBitmapCache() override;
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "doris-robot (via GitHub)" <gi...@apache.org>.
doris-robot commented on PR #31317:
URL: https://github.com/apache/doris/pull/31317#issuecomment-1966774551
<details>
<summary>TPC-H: <b>Total hot run time: 38013 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpch-tools
Tpch sf100 test result on commit ee84a0fb4b4a31f62ca947631bdc7ceaf39e9833, data reload: false
------ Round 1 ----------------------------------
q1 17816 4026 4055 4026
q2 2049 141 137 137
q3 10665 956 954 954
q4 4678 954 962 954
q5 7634 2847 3054 2847
q6 180 125 120 120
q7 1251 764 765 764
q8 9321 2037 2031 2031
q9 7705 6295 6329 6295
q10 8189 2503 2507 2503
q11 413 202 209 202
q12 770 304 312 304
q13 17981 3240 3238 3238
q14 278 250 270 250
q15 535 498 495 495
q16 465 413 418 413
q17 944 902 889 889
q18 6717 5905 6020 5905
q19 1572 1516 1524 1516
q20 571 301 271 271
q21 6903 3620 3603 3603
q22 813 296 298 296
Total cold run time: 107450 ms
Total hot run time: 38013 ms
----- Round 2, with runtime_filter_mode=off -----
q1 4029 3994 3989 3989
q2 304 230 214 214
q3 2960 2963 2923 2923
q4 1883 1820 1811 1811
q5 5151 5111 5135 5111
q6 198 116 114 114
q7 2120 1792 1725 1725
q8 3145 3212 3206 3206
q9 8257 8314 8262 8262
q10 6084 3668 3644 3644
q11 513 434 440 434
q12 677 512 517 512
q13 8772 3033 3039 3033
q14 271 250 256 250
q15 536 499 492 492
q16 519 470 466 466
q17 1715 1691 1698 1691
q18 7835 7512 7405 7405
q19 1685 1644 1625 1625
q20 2102 1915 1901 1901
q21 4773 4611 4630 4611
q22 521 455 454 454
Total cold run time: 64050 ms
Total hot run time: 53873 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "doris-robot (via GitHub)" <gi...@apache.org>.
doris-robot commented on PR #31317:
URL: https://github.com/apache/doris/pull/31317#issuecomment-1969042602
<details>
<summary>TPC-H: <b>Total hot run time: 37916 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpch-tools
Tpch sf100 test result on commit 0a23e751291ddf15970f07c0384db0ab6c5606aa, data reload: false
------ Round 1 ----------------------------------
q1 17691 4063 4064 4063
q2 2049 143 135 135
q3 10649 928 932 928
q4 4674 945 951 945
q5 7606 2909 3024 2909
q6 178 128 123 123
q7 1234 765 770 765
q8 9324 2010 2032 2010
q9 7213 6301 6322 6301
q10 8190 2511 2483 2483
q11 423 203 196 196
q12 786 316 315 315
q13 17965 3194 3210 3194
q14 278 248 280 248
q15 535 499 492 492
q16 468 415 414 414
q17 950 880 874 874
q18 6783 5912 6033 5912
q19 1559 1516 1517 1516
q20 543 274 276 274
q21 6312 3549 3563 3549
q22 800 278 270 270
Total cold run time: 106210 ms
Total hot run time: 37916 ms
----- Round 2, with runtime_filter_mode=off -----
q1 3994 3996 3984 3984
q2 307 214 219 214
q3 3000 2963 2890 2890
q4 1839 1834 1787 1787
q5 5203 5099 5088 5088
q6 197 114 116 114
q7 2147 1754 1783 1754
q8 3127 3195 3205 3195
q9 8263 8251 8312 8251
q10 6070 3634 3667 3634
q11 530 433 424 424
q12 675 530 536 530
q13 13325 3048 3076 3048
q14 271 264 277 264
q15 535 495 496 495
q16 514 481 479 479
q17 1699 1681 1677 1677
q18 7875 7393 7307 7307
q19 1867 1647 1636 1636
q20 2106 1915 1893 1893
q21 4743 4561 4659 4561
q22 535 440 449 440
Total cold run time: 68822 ms
Total hot run time: 53665 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "doris-robot (via GitHub)" <gi...@apache.org>.
doris-robot commented on PR #31317:
URL: https://github.com/apache/doris/pull/31317#issuecomment-1970382125
TeamCity be ut coverage result:
Function Coverage: 35.63% (8549/23991)
Line Coverage: 27.45% (69393/252816)
Region Coverage: 26.61% (35984/135216)
Branch Coverage: 23.43% (18397/78504)
Coverage Report: http://coverage.selectdb-in.cc/coverage/ced38f8389a29b7ffe8a5684b92d24002d3a26e8_ced38f8389a29b7ffe8a5684b92d24002d3a26e8/report/index.html
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "github-actions[bot] (via GitHub)" <gi...@apache.org>.
github-actions[bot] commented on PR #31317:
URL: https://github.com/apache/doris/pull/31317#issuecomment-1971224330
PR approved by at least one committer and no changes requested.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "liaoxin01 (via GitHub)" <gi...@apache.org>.
liaoxin01 commented on PR #31317:
URL: https://github.com/apache/doris/pull/31317#issuecomment-1971340305
run buildall
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "zhannngchen (via GitHub)" <gi...@apache.org>.
zhannngchen commented on code in PR #31317:
URL: https://github.com/apache/doris/pull/31317#discussion_r1507668133
##########
be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp:
##########
@@ -0,0 +1,165 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "cloud/cloud_engine_calc_delete_bitmap_task.h"
+
+#include <memory>
+
+#include "cloud/cloud_meta_mgr.h"
+#include "cloud/cloud_tablet.h"
+#include "common/status.h"
+#include "olap/base_tablet.h"
+#include "olap/olap_common.h"
+#include "olap/rowset/rowset.h"
+#include "olap/tablet_fwd.h"
+#include "olap/tablet_meta.h"
+#include "olap/txn_manager.h"
+#include "olap/utils.h"
+
+namespace doris {
+
+CloudEngineCalcDeleteBitmapTask::CloudEngineCalcDeleteBitmapTask(
+ CloudStorageEngine& engine, const TCalcDeleteBitmapRequest& cal_delete_bitmap_req,
+ std::vector<TTabletId>* error_tablet_ids, std::vector<TTabletId>* succ_tablet_ids)
+ : _engine(engine),
+ _cal_delete_bitmap_req(cal_delete_bitmap_req),
+ _error_tablet_ids(error_tablet_ids),
+ _succ_tablet_ids(succ_tablet_ids) {}
+
+void CloudEngineCalcDeleteBitmapTask::add_error_tablet_id(int64_t tablet_id, const Status& err) {
+ std::lock_guard<std::mutex> lck(_mutex);
+ _error_tablet_ids->push_back(tablet_id);
+ if (_res.ok() || _res.is<ErrorCode::DELETE_BITMAP_LOCK_ERROR>()) {
+ _res = err;
+ }
+}
+
+void CloudEngineCalcDeleteBitmapTask::add_succ_tablet_id(int64_t tablet_id) {
+ std::lock_guard<std::mutex> lck(_mutex);
+ _succ_tablet_ids->push_back(tablet_id);
+}
+
+Status CloudEngineCalcDeleteBitmapTask::execute() {
+ int64_t transaction_id = _cal_delete_bitmap_req.transaction_id;
+ OlapStopWatch watch;
+ VLOG_NOTICE << "begin to calculate delete bitmap. transaction_id=" << transaction_id;
+ std::unique_ptr<ThreadPoolToken> token =
+ _engine.calc_tablet_delete_bitmap_task_thread_pool()->new_token(
+ ThreadPool::ExecutionMode::CONCURRENT);
+
+ for (const auto& partition : _cal_delete_bitmap_req.partitions) {
+ int64_t version = partition.version;
+ for (auto tablet_id : partition.tablet_ids) {
+ auto base_tablet = DORIS_TRY(_engine.get_tablet(tablet_id));
+ std::shared_ptr<CloudTablet> tablet =
+ std::dynamic_pointer_cast<CloudTablet>(base_tablet);
+ if (tablet == nullptr) {
+ LOG(WARNING) << "can't get tablet when calculate delete bitmap. tablet_id="
+ << tablet_id;
+ _error_tablet_ids->push_back(tablet_id);
+ _res = Status::Error<ErrorCode::PUSH_TABLE_NOT_EXIST>(
+ "can't get tablet when calculate delete bitmap. tablet_id={}", tablet_id);
+ break;
+ }
+
+ Status st = tablet->sync_rowsets();
+ if (!st.ok() && !st.is<ErrorCode::INVALID_TABLET_STATE>()) {
+ return st;
+ }
+ if (st.is<ErrorCode::INVALID_TABLET_STATE>()) [[unlikely]] {
+ add_succ_tablet_id(tablet->tablet_id());
+ LOG(INFO)
+ << "tablet is under alter process, delete bitmap will be calculated later, "
+ "tablet_id: "
+ << tablet->tablet_id() << " txn_id: " << transaction_id
+ << ", request_version=" << version;
+ continue;
+ }
+ int64_t max_version = tablet->max_version_unlocked();
+ if (version != max_version + 1) {
+ _error_tablet_ids->push_back(tablet_id);
+ _res = Status::Error<ErrorCode::DELETE_BITMAP_LOCK_ERROR, false>(
+ "version not continuous");
+ LOG(WARNING) << "version not continuous, current max version=" << max_version
+ << ", request_version=" << version
+ << " tablet_id=" << tablet->tablet_id();
+ break;
+ }
+
+ auto tablet_calc_delete_bitmap_ptr = std::make_shared<CloudTabletCalcDeleteBitmapTask>(
+ _engine, this, tablet, transaction_id, version);
+ auto submit_st = token->submit_func([=]() { tablet_calc_delete_bitmap_ptr->handle(); });
+ CHECK(submit_st.ok());
Review Comment:
if we can handle the error, we should not use CHECK
##########
be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp:
##########
@@ -0,0 +1,165 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "cloud/cloud_engine_calc_delete_bitmap_task.h"
+
+#include <memory>
+
+#include "cloud/cloud_meta_mgr.h"
+#include "cloud/cloud_tablet.h"
+#include "common/status.h"
+#include "olap/base_tablet.h"
+#include "olap/olap_common.h"
+#include "olap/rowset/rowset.h"
+#include "olap/tablet_fwd.h"
+#include "olap/tablet_meta.h"
+#include "olap/txn_manager.h"
+#include "olap/utils.h"
+
+namespace doris {
+
+CloudEngineCalcDeleteBitmapTask::CloudEngineCalcDeleteBitmapTask(
+ CloudStorageEngine& engine, const TCalcDeleteBitmapRequest& cal_delete_bitmap_req,
+ std::vector<TTabletId>* error_tablet_ids, std::vector<TTabletId>* succ_tablet_ids)
+ : _engine(engine),
+ _cal_delete_bitmap_req(cal_delete_bitmap_req),
+ _error_tablet_ids(error_tablet_ids),
+ _succ_tablet_ids(succ_tablet_ids) {}
+
+void CloudEngineCalcDeleteBitmapTask::add_error_tablet_id(int64_t tablet_id, const Status& err) {
+ std::lock_guard<std::mutex> lck(_mutex);
+ _error_tablet_ids->push_back(tablet_id);
+ if (_res.ok() || _res.is<ErrorCode::DELETE_BITMAP_LOCK_ERROR>()) {
+ _res = err;
+ }
+}
+
+void CloudEngineCalcDeleteBitmapTask::add_succ_tablet_id(int64_t tablet_id) {
+ std::lock_guard<std::mutex> lck(_mutex);
+ _succ_tablet_ids->push_back(tablet_id);
+}
+
+Status CloudEngineCalcDeleteBitmapTask::execute() {
+ int64_t transaction_id = _cal_delete_bitmap_req.transaction_id;
+ OlapStopWatch watch;
+ VLOG_NOTICE << "begin to calculate delete bitmap. transaction_id=" << transaction_id;
+ std::unique_ptr<ThreadPoolToken> token =
+ _engine.calc_tablet_delete_bitmap_task_thread_pool()->new_token(
+ ThreadPool::ExecutionMode::CONCURRENT);
+
+ for (const auto& partition : _cal_delete_bitmap_req.partitions) {
+ int64_t version = partition.version;
+ for (auto tablet_id : partition.tablet_ids) {
+ auto base_tablet = DORIS_TRY(_engine.get_tablet(tablet_id));
+ std::shared_ptr<CloudTablet> tablet =
+ std::dynamic_pointer_cast<CloudTablet>(base_tablet);
+ if (tablet == nullptr) {
+ LOG(WARNING) << "can't get tablet when calculate delete bitmap. tablet_id="
+ << tablet_id;
+ _error_tablet_ids->push_back(tablet_id);
+ _res = Status::Error<ErrorCode::PUSH_TABLE_NOT_EXIST>(
+ "can't get tablet when calculate delete bitmap. tablet_id={}", tablet_id);
+ break;
+ }
+
+ Status st = tablet->sync_rowsets();
+ if (!st.ok() && !st.is<ErrorCode::INVALID_TABLET_STATE>()) {
+ return st;
+ }
+ if (st.is<ErrorCode::INVALID_TABLET_STATE>()) [[unlikely]] {
+ add_succ_tablet_id(tablet->tablet_id());
+ LOG(INFO)
+ << "tablet is under alter process, delete bitmap will be calculated later, "
Review Comment:
No extra processing needed for schema change on cloud? I don't see any relevant changes
##########
be/src/cloud/cloud_txn_delete_bitmap_cache.cpp:
##########
@@ -0,0 +1,192 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "cloud/cloud_txn_delete_bitmap_cache.h"
+
+#include <fmt/core.h>
+
+#include <chrono>
+#include <memory>
+#include <shared_mutex>
+
+#include "common/status.h"
+#include "common/sync_point.h"
+#include "olap/olap_common.h"
+#include "olap/tablet_meta.h"
+
+namespace doris {
+
+CloudTxnDeleteBitmapCache::CloudTxnDeleteBitmapCache(size_t size_in_bytes)
+ : LRUCachePolicy(CachePolicy::CacheType::CLOUD_TXN_DELETE_BITMAP_CACHE, size_in_bytes,
+ LRUCacheType::SIZE, 86400, 4),
+ _stop_latch(1) {}
+
+CloudTxnDeleteBitmapCache::~CloudTxnDeleteBitmapCache() {
+ _stop_latch.count_down();
+ _clean_thread->join();
+}
+
+Status CloudTxnDeleteBitmapCache::init() {
+ auto st = Thread::create(
+ "CloudTxnDeleteBitmapCache", "clean_txn_dbm_thread",
+ [this]() { this->_clean_thread_callback(); }, &_clean_thread);
+ if (!st.ok()) {
+ LOG(WARNING) << "failed to create thread for CloudTxnDeleteBitmapCache, error: " << st;
+ }
+ return st;
+}
+
+Status CloudTxnDeleteBitmapCache::get_tablet_txn_info(
+ TTransactionId transaction_id, int64_t tablet_id, RowsetSharedPtr* rowset,
+ DeleteBitmapPtr* delete_bitmap, RowsetIdUnorderedSet* rowset_ids, int64_t* txn_expiration,
+ std::shared_ptr<PartialUpdateInfo>* partial_update_info) {
+ {
+ std::shared_lock<std::shared_mutex> rlock(_rwlock);
+ TxnKey key(transaction_id, tablet_id);
+ auto iter = _txn_map.find(key);
+ if (iter == _txn_map.end()) {
+ return Status::Error<ErrorCode::NOT_FOUND, false>(
+ "not found txn info, tablet_id={}, transaction_id={}", tablet_id,
+ transaction_id);
+ }
+ *rowset = iter->second.rowset;
+ *txn_expiration = iter->second.txn_expiration;
+ *partial_update_info = iter->second.partial_update_info;
+ }
+ std::string key_str = fmt::format("{}/{}", transaction_id, tablet_id);
+ CacheKey key(key_str);
+ Cache::Handle* handle = cache()->lookup(key);
+
+ DeleteBitmapCacheValue* val =
+ handle == nullptr ? nullptr
+ : reinterpret_cast<DeleteBitmapCacheValue*>(cache()->value(handle));
+ if (val) {
+ *delete_bitmap = val->delete_bitmap;
+ *rowset_ids = val->rowset_ids;
+ // must call release handle to reduce the reference count,
+ // otherwise there will be memory leak
+ cache()->release(handle);
+ } else {
+ LOG_INFO("cache missed when get delete bitmap")
+ .tag("txn_id", transaction_id)
+ .tag("tablt_id", tablet_id);
Review Comment:
typo:tablet_id
##########
be/src/cloud/cloud_txn_delete_bitmap_cache.cpp:
##########
@@ -0,0 +1,192 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "cloud/cloud_txn_delete_bitmap_cache.h"
+
+#include <fmt/core.h>
+
+#include <chrono>
+#include <memory>
+#include <shared_mutex>
+
+#include "common/status.h"
+#include "common/sync_point.h"
+#include "olap/olap_common.h"
+#include "olap/tablet_meta.h"
+
+namespace doris {
+
+CloudTxnDeleteBitmapCache::CloudTxnDeleteBitmapCache(size_t size_in_bytes)
+ : LRUCachePolicy(CachePolicy::CacheType::CLOUD_TXN_DELETE_BITMAP_CACHE, size_in_bytes,
+ LRUCacheType::SIZE, 86400, 4),
+ _stop_latch(1) {}
+
+CloudTxnDeleteBitmapCache::~CloudTxnDeleteBitmapCache() {
+ _stop_latch.count_down();
+ _clean_thread->join();
+}
+
+Status CloudTxnDeleteBitmapCache::init() {
+ auto st = Thread::create(
+ "CloudTxnDeleteBitmapCache", "clean_txn_dbm_thread",
+ [this]() { this->_clean_thread_callback(); }, &_clean_thread);
+ if (!st.ok()) {
+ LOG(WARNING) << "failed to create thread for CloudTxnDeleteBitmapCache, error: " << st;
+ }
+ return st;
+}
+
+Status CloudTxnDeleteBitmapCache::get_tablet_txn_info(
+ TTransactionId transaction_id, int64_t tablet_id, RowsetSharedPtr* rowset,
+ DeleteBitmapPtr* delete_bitmap, RowsetIdUnorderedSet* rowset_ids, int64_t* txn_expiration,
+ std::shared_ptr<PartialUpdateInfo>* partial_update_info) {
+ {
+ std::shared_lock<std::shared_mutex> rlock(_rwlock);
+ TxnKey key(transaction_id, tablet_id);
+ auto iter = _txn_map.find(key);
+ if (iter == _txn_map.end()) {
+ return Status::Error<ErrorCode::NOT_FOUND, false>(
+ "not found txn info, tablet_id={}, transaction_id={}", tablet_id,
+ transaction_id);
+ }
+ *rowset = iter->second.rowset;
+ *txn_expiration = iter->second.txn_expiration;
+ *partial_update_info = iter->second.partial_update_info;
+ }
+ std::string key_str = fmt::format("{}/{}", transaction_id, tablet_id);
+ CacheKey key(key_str);
+ Cache::Handle* handle = cache()->lookup(key);
+
+ DeleteBitmapCacheValue* val =
+ handle == nullptr ? nullptr
+ : reinterpret_cast<DeleteBitmapCacheValue*>(cache()->value(handle));
+ if (val) {
+ *delete_bitmap = val->delete_bitmap;
+ *rowset_ids = val->rowset_ids;
+ // must call release handle to reduce the reference count,
+ // otherwise there will be memory leak
+ cache()->release(handle);
+ } else {
+ LOG_INFO("cache missed when get delete bitmap")
+ .tag("txn_id", transaction_id)
+ .tag("tablt_id", tablet_id);
+ // Becasue of the rowset_ids become empty, all delete bitmap
+ // will be recalculate in CalcDeleteBitmapTask
+ *delete_bitmap = std::make_shared<DeleteBitmap>(tablet_id);
+ }
+ return Status::OK();
+}
+
+void CloudTxnDeleteBitmapCache::set_tablet_txn_info(
+ TTransactionId transaction_id, int64_t tablet_id, DeleteBitmapPtr delete_bitmap,
+ const RowsetIdUnorderedSet& rowset_ids, RowsetSharedPtr rowset, int64_t txn_expiration,
+ std::shared_ptr<PartialUpdateInfo> partial_update_info) {
+ if (txn_expiration <= 0) {
+ txn_expiration = duration_cast<std::chrono::seconds>(
+ std::chrono::system_clock::now().time_since_epoch())
+ .count() +
+ 120;
+ }
+ {
+ std::unique_lock<std::shared_mutex> wlock(_rwlock);
+ TxnKey txn_key(transaction_id, tablet_id);
+ _txn_map[txn_key] = TxnVal(rowset, txn_expiration, std::move(partial_update_info));
+ _expiration_txn.emplace(txn_expiration, txn_key);
+ }
+ std::string key_str = fmt::format("{}/{}", transaction_id, tablet_id);
+ CacheKey key(key_str);
+
+ auto val = new DeleteBitmapCacheValue(delete_bitmap, rowset_ids);
+ auto deleter = [](const CacheKey&, void* value) {
+ delete (DeleteBitmapCacheValue*)value; // Just delete to reclaim
+ };
+ size_t charge = sizeof(DeleteBitmapCacheValue);
+ for (auto& [k, v] : val->delete_bitmap->delete_bitmap) {
+ charge += v.getSizeInBytes();
+ }
+ auto handle = cache()->insert(key, val, charge, deleter, CachePriority::NORMAL);
+ // must call release handle to reduce the reference count,
+ // otherwise there will be memory leak
+ cache()->release(handle);
+ LOG_INFO("set txn related delete bitmap")
+ .tag("txn_id", transaction_id)
+ .tag("expiration", txn_expiration)
+ .tag("tablt_id", tablet_id)
+ .tag("delete_bitmap_size", charge);
+}
+
+void CloudTxnDeleteBitmapCache::update_tablet_txn_info(TTransactionId transaction_id,
+ int64_t tablet_id,
+ DeleteBitmapPtr delete_bitmap,
+ const RowsetIdUnorderedSet& rowset_ids) {
+ std::string key_str = fmt::format("{}/{}", transaction_id, tablet_id);
+ CacheKey key(key_str);
+
+ DeleteBitmapPtr new_delete_bitmap = std::make_shared<DeleteBitmap>(tablet_id);
+ *new_delete_bitmap = *delete_bitmap;
Review Comment:
why do we need to copy value here?
##########
be/src/cloud/cloud_tablets_channel.cpp:
##########
@@ -235,7 +235,16 @@ Status CloudTabletsChannel::close(LoadChannel* parent, const PTabletWriterAddBlo
}
}
- // TODO(plat1ko): 6. set txn related delete bitmap if necessary
+ // 6. set txn related delete bitmap if necessary
Review Comment:
do we need to sync rowset before L216(auto st = writer->submit_calc_delete_bitmap_task();)?
##########
be/src/cloud/cloud_tablet.cpp:
##########
@@ -443,4 +479,52 @@ void CloudTablet::set_cumulative_layer_point(int64_t new_point) {
_cumulative_point = new_point;
}
+CalcDeleteBitmapExecutor* CloudTablet::calc_delete_bitmap_executor() {
+ return _engine.calc_delete_bitmap_executor();
+}
+
+Status CloudTablet::save_delete_bitmap(const TabletTxnInfo* txn_info, int64_t txn_id,
+ DeleteBitmapPtr delete_bitmap, RowsetWriter* rowset_writer,
+ const RowsetIdUnorderedSet& cur_rowset_ids) {
+ RowsetSharedPtr rowset = txn_info->rowset;
+ int64_t cur_version = rowset->start_version();
+ // update delete bitmap info, in order to avoid recalculation when trying again
+ _engine.txn_delete_bitmap_cache().update_tablet_txn_info(txn_id, tablet_id(), delete_bitmap,
+ cur_rowset_ids);
+
+ if (txn_info->partial_update_info && txn_info->partial_update_info->is_partial_update &&
+ rowset_writer->num_rows() > 0) {
+ DBUG_EXECUTE_IF("CloudTablet.update_delete_bitmap.partial_update_write_rowset_fail", {
Review Comment:
the debug point name changed, we should update the regression-test for cloud as well?
##########
be/src/cloud/cloud_txn_delete_bitmap_cache.cpp:
##########
@@ -0,0 +1,192 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "cloud/cloud_txn_delete_bitmap_cache.h"
+
+#include <fmt/core.h>
+
+#include <chrono>
+#include <memory>
+#include <shared_mutex>
+
+#include "common/status.h"
+#include "common/sync_point.h"
+#include "olap/olap_common.h"
+#include "olap/tablet_meta.h"
+
+namespace doris {
+
+CloudTxnDeleteBitmapCache::CloudTxnDeleteBitmapCache(size_t size_in_bytes)
+ : LRUCachePolicy(CachePolicy::CacheType::CLOUD_TXN_DELETE_BITMAP_CACHE, size_in_bytes,
+ LRUCacheType::SIZE, 86400, 4),
+ _stop_latch(1) {}
+
+CloudTxnDeleteBitmapCache::~CloudTxnDeleteBitmapCache() {
+ _stop_latch.count_down();
+ _clean_thread->join();
+}
+
+Status CloudTxnDeleteBitmapCache::init() {
+ auto st = Thread::create(
+ "CloudTxnDeleteBitmapCache", "clean_txn_dbm_thread",
+ [this]() { this->_clean_thread_callback(); }, &_clean_thread);
+ if (!st.ok()) {
+ LOG(WARNING) << "failed to create thread for CloudTxnDeleteBitmapCache, error: " << st;
+ }
+ return st;
+}
+
+Status CloudTxnDeleteBitmapCache::get_tablet_txn_info(
+ TTransactionId transaction_id, int64_t tablet_id, RowsetSharedPtr* rowset,
+ DeleteBitmapPtr* delete_bitmap, RowsetIdUnorderedSet* rowset_ids, int64_t* txn_expiration,
+ std::shared_ptr<PartialUpdateInfo>* partial_update_info) {
+ {
+ std::shared_lock<std::shared_mutex> rlock(_rwlock);
+ TxnKey key(transaction_id, tablet_id);
+ auto iter = _txn_map.find(key);
+ if (iter == _txn_map.end()) {
+ return Status::Error<ErrorCode::NOT_FOUND, false>(
+ "not found txn info, tablet_id={}, transaction_id={}", tablet_id,
+ transaction_id);
+ }
+ *rowset = iter->second.rowset;
+ *txn_expiration = iter->second.txn_expiration;
+ *partial_update_info = iter->second.partial_update_info;
+ }
+ std::string key_str = fmt::format("{}/{}", transaction_id, tablet_id);
+ CacheKey key(key_str);
+ Cache::Handle* handle = cache()->lookup(key);
+
+ DeleteBitmapCacheValue* val =
+ handle == nullptr ? nullptr
+ : reinterpret_cast<DeleteBitmapCacheValue*>(cache()->value(handle));
+ if (val) {
+ *delete_bitmap = val->delete_bitmap;
+ *rowset_ids = val->rowset_ids;
+ // must call release handle to reduce the reference count,
+ // otherwise there will be memory leak
+ cache()->release(handle);
+ } else {
+ LOG_INFO("cache missed when get delete bitmap")
+ .tag("txn_id", transaction_id)
+ .tag("tablt_id", tablet_id);
+ // Becasue of the rowset_ids become empty, all delete bitmap
+ // will be recalculate in CalcDeleteBitmapTask
+ *delete_bitmap = std::make_shared<DeleteBitmap>(tablet_id);
+ }
+ return Status::OK();
+}
+
+void CloudTxnDeleteBitmapCache::set_tablet_txn_info(
+ TTransactionId transaction_id, int64_t tablet_id, DeleteBitmapPtr delete_bitmap,
+ const RowsetIdUnorderedSet& rowset_ids, RowsetSharedPtr rowset, int64_t txn_expiration,
+ std::shared_ptr<PartialUpdateInfo> partial_update_info) {
+ if (txn_expiration <= 0) {
+ txn_expiration = duration_cast<std::chrono::seconds>(
+ std::chrono::system_clock::now().time_since_epoch())
+ .count() +
+ 120;
+ }
+ {
+ std::unique_lock<std::shared_mutex> wlock(_rwlock);
+ TxnKey txn_key(transaction_id, tablet_id);
+ _txn_map[txn_key] = TxnVal(rowset, txn_expiration, std::move(partial_update_info));
+ _expiration_txn.emplace(txn_expiration, txn_key);
+ }
+ std::string key_str = fmt::format("{}/{}", transaction_id, tablet_id);
+ CacheKey key(key_str);
+
+ auto val = new DeleteBitmapCacheValue(delete_bitmap, rowset_ids);
+ auto deleter = [](const CacheKey&, void* value) {
+ delete (DeleteBitmapCacheValue*)value; // Just delete to reclaim
+ };
+ size_t charge = sizeof(DeleteBitmapCacheValue);
+ for (auto& [k, v] : val->delete_bitmap->delete_bitmap) {
+ charge += v.getSizeInBytes();
+ }
+ auto handle = cache()->insert(key, val, charge, deleter, CachePriority::NORMAL);
+ // must call release handle to reduce the reference count,
+ // otherwise there will be memory leak
+ cache()->release(handle);
+ LOG_INFO("set txn related delete bitmap")
+ .tag("txn_id", transaction_id)
+ .tag("expiration", txn_expiration)
+ .tag("tablt_id", tablet_id)
Review Comment:
ditto
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "liaoxin01 (via GitHub)" <gi...@apache.org>.
liaoxin01 commented on code in PR #31317:
URL: https://github.com/apache/doris/pull/31317#discussion_r1508825518
##########
be/src/cloud/cloud_tablets_channel.cpp:
##########
@@ -235,7 +235,16 @@ Status CloudTabletsChannel::close(LoadChannel* parent, const PTabletWriterAddBlo
}
}
- // TODO(plat1ko): 6. set txn related delete bitmap if necessary
+ // 6. set txn related delete bitmap if necessary
Review Comment:
Sync rowset is a heavy operation in cloud mode, we will consider whether to add it later.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "doris-robot (via GitHub)" <gi...@apache.org>.
doris-robot commented on PR #31317:
URL: https://github.com/apache/doris/pull/31317#issuecomment-1970546696
TeamCity be ut coverage result:
Function Coverage: 35.63% (8550/23997)
Line Coverage: 27.44% (69392/252855)
Region Coverage: 26.62% (35993/135230)
Branch Coverage: 23.43% (18394/78512)
Coverage Report: http://coverage.selectdb-in.cc/coverage/2c80b3d53760bb63ae90fcf0ed7a6ee5c67be4a0_2c80b3d53760bb63ae90fcf0ed7a6ee5c67be4a0/report/index.html
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "doris-robot (via GitHub)" <gi...@apache.org>.
doris-robot commented on PR #31317:
URL: https://github.com/apache/doris/pull/31317#issuecomment-1969073488
<details>
<summary>ClickBench: <b>Total hot run time: 31.33 s</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/clickbench-tools
ClickBench test result on commit 0a23e751291ddf15970f07c0384db0ab6c5606aa, data reload: false
query1 0.04 0.04 0.03
query2 0.06 0.02 0.02
query3 0.23 0.07 0.06
query4 1.66 0.11 0.10
query5 0.53 0.52 0.52
query6 1.30 0.69 0.68
query7 0.02 0.01 0.02
query8 0.03 0.02 0.02
query9 0.56 0.50 0.52
query10 0.56 0.56 0.55
query11 0.13 0.10 0.10
query12 0.12 0.10 0.11
query13 0.63 0.61 0.62
query14 0.77 0.83 0.80
query15 0.85 0.84 0.81
query16 0.38 0.36 0.37
query17 0.99 0.94 1.01
query18 0.27 0.25 0.25
query19 1.78 1.68 1.76
query20 0.01 0.02 0.01
query21 15.40 0.68 0.60
query22 3.62 4.35 2.00
query23 17.45 1.05 0.89
query24 2.16 0.28 0.22
query25 0.42 0.23 0.05
query26 0.15 0.12 0.12
query27 0.05 0.05 0.04
query28 12.35 0.83 0.83
query29 12.52 3.15 3.49
query30 0.59 0.54 0.46
query31 2.78 0.36 0.36
query32 3.35 0.48 0.49
query33 3.23 3.21 3.28
query34 15.36 4.78 4.72
query35 4.76 4.73 4.74
query36 1.14 1.06 1.06
query37 0.07 0.05 0.05
query38 0.05 0.03 0.02
query39 0.02 0.02 0.02
query40 0.17 0.12 0.13
query41 0.07 0.01 0.01
query42 0.03 0.02 0.02
query43 0.03 0.02 0.02
Total cold run time: 106.69 s
Total hot run time: 31.33 s
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "liaoxin01 (via GitHub)" <gi...@apache.org>.
liaoxin01 commented on PR #31317:
URL: https://github.com/apache/doris/pull/31317#issuecomment-1970572746
run buildall
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "liaoxin01 (via GitHub)" <gi...@apache.org>.
liaoxin01 commented on PR #31317:
URL: https://github.com/apache/doris/pull/31317#issuecomment-1966612167
run buildall
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "doris-robot (via GitHub)" <gi...@apache.org>.
doris-robot commented on PR #31317:
URL: https://github.com/apache/doris/pull/31317#issuecomment-1970671385
<details>
<summary>TPC-DS: <b>Total hot run time: 168844 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpcds-tools
TPC-DS sf100 test result on commit 855559f1046c84941399efe6b5240367ba209876, data reload: false
query1 934 359 341 341
query2 6527 1721 1686 1686
query3 6706 212 203 203
query4 23204 20441 20500 20441
query5 4221 362 371 362
query6 286 174 173 173
query7 4610 290 301 290
query8 257 190 205 190
query9 8455 2275 2275 2275
query10 414 216 211 211
query11 14484 14119 13974 13974
query12 136 85 83 83
query13 1645 430 411 411
query14 8523 6348 6359 6348
query15 209 172 172 172
query16 7091 267 264 264
query17 1031 522 504 504
query18 1895 268 270 268
query19 192 161 150 150
query20 86 81 82 81
query21 202 121 127 121
query22 4774 4629 4597 4597
query23 30529 29834 29826 29826
query24 12109 3166 3134 3134
query25 658 353 345 345
query26 1877 154 155 154
query27 3062 319 302 302
query28 7070 1822 1820 1820
query29 1187 566 558 558
query30 286 133 137 133
query31 878 673 701 673
query32 96 63 60 60
query33 720 222 227 222
query34 1077 469 477 469
query35 867 743 742 742
query36 923 810 856 810
query37 153 65 63 63
query38 3188 3006 2987 2987
query39 1319 1256 1246 1246
query40 278 99 102 99
query41 39 37 35 35
query42 107 98 104 98
query43 441 411 416 411
query44 1058 683 704 683
query45 191 176 175 175
query46 1038 802 771 771
query47 1518 1398 1459 1398
query48 425 337 336 336
query49 1164 299 299 299
query50 765 363 359 359
query51 4384 4327 4337 4327
query52 97 95 88 88
query53 331 269 270 269
query54 286 223 223 223
query55 91 81 81 81
query56 217 202 193 193
query57 978 844 877 844
query58 216 198 191 191
query59 2120 2043 2118 2043
query60 230 219 211 211
query61 88 88 93 88
query62 593 330 349 330
query63 281 267 264 264
query64 6567 3027 3106 3027
query65 3246 3206 3191 3191
query66 1458 328 331 328
query67 14485 14087 14104 14087
query68 5045 543 555 543
query69 534 371 370 370
query70 1175 1149 1261 1149
query71 396 250 245 245
query72 6257 2834 2712 2712
query73 705 315 313 313
query74 6292 6040 5989 5989
query75 3070 2428 2405 2405
query76 2945 1027 1179 1027
query77 351 237 253 237
query78 9040 8554 8491 8491
query79 1084 523 518 518
query80 929 355 343 343
query81 478 194 200 194
query82 1310 84 87 84
query83 220 124 118 118
query84 276 75 81 75
query85 1181 366 356 356
query86 385 305 304 304
query87 3264 3125 3098 3098
query88 2616 2251 2219 2219
query89 398 375 328 328
query90 1904 163 161 161
query91 152 125 137 125
query92 53 46 49 46
query93 1053 522 499 499
query94 1221 183 183 183
query95 442 338 340 338
query96 576 259 259 259
query97 4304 4107 4137 4107
query98 225 203 192 192
query99 1060 682 678 678
Total cold run time: 263514 ms
Total hot run time: 168844 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "doris-robot (via GitHub)" <gi...@apache.org>.
doris-robot commented on PR #31317:
URL: https://github.com/apache/doris/pull/31317#issuecomment-1970680068
<details>
<summary>ClickBench: <b>Total hot run time: 31.04 s</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/clickbench-tools
ClickBench test result on commit 855559f1046c84941399efe6b5240367ba209876, data reload: false
query1 0.04 0.04 0.04
query2 0.06 0.02 0.02
query3 0.22 0.06 0.06
query4 1.67 0.10 0.10
query5 0.52 0.51 0.51
query6 1.29 0.66 0.68
query7 0.02 0.01 0.01
query8 0.03 0.02 0.02
query9 0.56 0.49 0.50
query10 0.55 0.57 0.58
query11 0.12 0.09 0.10
query12 0.12 0.10 0.10
query13 0.63 0.61 0.61
query14 0.79 0.81 0.80
query15 0.83 0.82 0.83
query16 0.38 0.38 0.37
query17 0.98 0.99 0.97
query18 0.26 0.25 0.26
query19 1.81 1.75 1.73
query20 0.01 0.01 0.01
query21 15.40 0.66 0.63
query22 3.20 4.32 1.40
query23 17.63 1.06 1.09
query24 2.15 0.76 0.18
query25 0.27 0.10 0.04
query26 0.18 0.14 0.15
query27 0.03 0.03 0.03
query28 12.16 0.84 0.83
query29 12.63 3.48 3.20
query30 0.58 0.52 0.52
query31 2.80 0.36 0.36
query32 3.32 0.46 0.48
query33 3.17 3.19 3.22
query34 15.37 4.75 4.74
query35 4.77 4.77 4.72
query36 1.12 1.05 1.08
query37 0.07 0.05 0.05
query38 0.04 0.03 0.03
query39 0.02 0.02 0.01
query40 0.16 0.13 0.16
query41 0.07 0.02 0.01
query42 0.02 0.02 0.01
query43 0.03 0.02 0.02
Total cold run time: 106.08 s
Total hot run time: 31.04 s
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "doris-robot (via GitHub)" <gi...@apache.org>.
doris-robot commented on PR #31317:
URL: https://github.com/apache/doris/pull/31317#issuecomment-1971291174
TeamCity be ut coverage result:
Function Coverage: 35.61% (8549/24007)
Line Coverage: 27.42% (69378/253056)
Region Coverage: 26.59% (35995/135374)
Branch Coverage: 23.41% (18390/78570)
Coverage Report: http://coverage.selectdb-in.cc/coverage/4b84f9d7a277eea6e617ded20cd5b7b82ecbe1a9_4b84f9d7a277eea6e617ded20cd5b7b82ecbe1a9/report/index.html
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "liaoxin01 (via GitHub)" <gi...@apache.org>.
liaoxin01 commented on PR #31317:
URL: https://github.com/apache/doris/pull/31317#issuecomment-1971411632
run feut
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "doris-robot (via GitHub)" <gi...@apache.org>.
doris-robot commented on PR #31317:
URL: https://github.com/apache/doris/pull/31317#issuecomment-1960815555
Thank you for your contribution to Apache Doris.
Don't know what should be done next? See [How to process your PR](https://cwiki.apache.org/confluence/display/DORIS/How+to+process+your+PR)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "zhannngchen (via GitHub)" <gi...@apache.org>.
zhannngchen commented on code in PR #31317:
URL: https://github.com/apache/doris/pull/31317#discussion_r1508851464
##########
be/src/cloud/cloud_tablets_channel.cpp:
##########
@@ -235,7 +235,16 @@ Status CloudTabletsChannel::close(LoadChannel* parent, const PTabletWriterAddBlo
}
}
- // TODO(plat1ko): 6. set txn related delete bitmap if necessary
+ // 6. set txn related delete bitmap if necessary
Review Comment:
It's fine to add it later, or you can make some performance test on that.
In our performance optimization half a year ago, calculating delete bitmap for incremental rowsets before commit can significantly reduce the computational cost of the publish phase
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "doris-robot (via GitHub)" <gi...@apache.org>.
doris-robot commented on PR #31317:
URL: https://github.com/apache/doris/pull/31317#issuecomment-1970368607
<details>
<summary>TPC-H: <b>Total hot run time: 38041 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpch-tools
Tpch sf100 test result on commit ced38f8389a29b7ffe8a5684b92d24002d3a26e8, data reload: false
------ Round 1 ----------------------------------
q1 17640 4096 4056 4056
q2 2040 149 129 129
q3 10570 1004 938 938
q4 4664 958 941 941
q5 7607 2894 2996 2894
q6 181 121 121 121
q7 1237 758 755 755
q8 9341 2042 2018 2018
q9 7769 6315 6307 6307
q10 8213 2481 2489 2481
q11 412 202 206 202
q12 716 313 306 306
q13 17930 3225 3199 3199
q14 278 245 247 245
q15 538 499 492 492
q16 477 399 402 399
q17 947 885 873 873
q18 6779 6039 6050 6039
q19 1555 1518 1524 1518
q20 563 281 265 265
q21 6802 3619 3583 3583
q22 809 280 284 280
Total cold run time: 107068 ms
Total hot run time: 38041 ms
----- Round 2, with runtime_filter_mode=off -----
q1 4032 4021 4040 4021
q2 309 215 216 215
q3 2971 2928 2940 2928
q4 1845 1842 1833 1833
q5 5124 5104 5132 5104
q6 199 114 117 114
q7 2122 1722 1781 1722
q8 3133 3188 3214 3188
q9 8244 8268 8233 8233
q10 6106 3672 3658 3658
q11 527 432 430 430
q12 685 514 533 514
q13 7757 3042 3067 3042
q14 279 248 258 248
q15 545 488 499 488
q16 499 474 451 451
q17 1704 1691 1682 1682
q18 7996 7420 7429 7420
q19 1670 1634 1642 1634
q20 2123 1902 1892 1892
q21 4800 4615 4634 4615
q22 500 415 433 415
Total cold run time: 63170 ms
Total hot run time: 53847 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "liaoxin01 (via GitHub)" <gi...@apache.org>.
liaoxin01 commented on PR #31317:
URL: https://github.com/apache/doris/pull/31317#issuecomment-1968626695
run buildall
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "doris-robot (via GitHub)" <gi...@apache.org>.
doris-robot commented on PR #31317:
URL: https://github.com/apache/doris/pull/31317#issuecomment-1968750741
<details>
<summary>ClickBench: <b>Total hot run time: 33.4 s</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/clickbench-tools
ClickBench test result on commit 79b90d0e6d1ee594376d460be1d722f84c24b1e8, data reload: false
query1 0.04 0.03 0.03
query2 0.06 0.02 0.02
query3 0.23 0.06 0.06
query4 1.67 0.10 0.10
query5 0.52 0.52 0.52
query6 1.24 0.68 0.68
query7 0.02 0.01 0.01
query8 0.04 0.02 0.02
query9 0.57 0.51 0.52
query10 0.56 0.56 0.57
query11 0.12 0.10 0.10
query12 0.13 0.10 0.10
query13 0.61 0.62 0.61
query14 0.81 0.79 0.80
query15 0.85 0.83 0.81
query16 0.37 0.36 0.40
query17 0.99 0.96 0.97
query18 0.25 0.27 0.25
query19 1.79 1.76 1.69
query20 0.01 0.01 0.01
query21 15.41 0.64 0.61
query22 3.35 3.70 3.43
query23 17.46 1.09 0.99
query24 2.02 0.55 0.56
query25 0.44 0.10 0.10
query26 0.16 0.13 0.13
query27 0.04 0.04 0.05
query28 11.95 0.84 0.84
query29 12.55 3.36 3.20
query30 0.60 0.58 0.53
query31 2.78 0.34 0.35
query32 3.36 0.48 0.49
query33 3.19 3.21 3.21
query34 15.36 4.75 4.72
query35 4.75 4.73 4.74
query36 1.13 1.06 1.07
query37 0.06 0.05 0.05
query38 0.05 0.03 0.02
query39 0.02 0.01 0.02
query40 0.17 0.13 0.14
query41 0.07 0.02 0.01
query42 0.03 0.01 0.02
query43 0.02 0.02 0.02
Total cold run time: 105.85 s
Total hot run time: 33.4 s
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "doris-robot (via GitHub)" <gi...@apache.org>.
doris-robot commented on PR #31317:
URL: https://github.com/apache/doris/pull/31317#issuecomment-1966737594
TeamCity be ut coverage result:
Function Coverage: 35.64% (8548/23984)
Line Coverage: 27.44% (69385/252832)
Region Coverage: 26.62% (35972/135152)
Branch Coverage: 23.43% (18386/78472)
Coverage Report: http://coverage.selectdb-in.cc/coverage/ee84a0fb4b4a31f62ca947631bdc7ceaf39e9833_ee84a0fb4b4a31f62ca947631bdc7ceaf39e9833/report/index.html
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "doris-robot (via GitHub)" <gi...@apache.org>.
doris-robot commented on PR #31317:
URL: https://github.com/apache/doris/pull/31317#issuecomment-1966802714
<details>
<summary>TPC-DS: <b>Total hot run time: 169525 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpcds-tools
TPC-DS sf100 test result on commit ee84a0fb4b4a31f62ca947631bdc7ceaf39e9833, data reload: false
query1 959 342 342 342
query2 6534 1694 1732 1694
query3 6703 205 205 205
query4 23352 20572 20583 20572
query5 4263 359 353 353
query6 262 171 172 171
query7 4603 301 294 294
query8 269 185 214 185
query9 8448 2175 2173 2173
query10 411 219 214 214
query11 14636 14153 14070 14070
query12 137 82 84 82
query13 1643 425 424 424
query14 8569 6298 6517 6298
query15 202 173 172 172
query16 7082 269 252 252
query17 1027 528 504 504
query18 1912 285 266 266
query19 193 150 152 150
query20 86 81 84 81
query21 203 129 128 128
query22 4701 4674 4509 4509
query23 30422 29868 29861 29861
query24 11828 3155 3077 3077
query25 659 347 344 344
query26 1880 163 162 162
query27 3042 314 312 312
query28 7045 1792 1769 1769
query29 1173 562 571 562
query30 289 131 143 131
query31 869 693 699 693
query32 92 60 58 58
query33 711 235 225 225
query34 1041 484 488 484
query35 880 744 745 744
query36 941 869 873 869
query37 153 62 63 62
query38 3115 2974 2982 2974
query39 1337 1265 1258 1258
query40 287 101 100 100
query41 37 37 35 35
query42 107 96 95 95
query43 459 423 417 417
query44 1057 681 700 681
query45 187 181 173 173
query46 1056 797 767 767
query47 1551 1452 1455 1452
query48 408 345 329 329
query49 1177 301 284 284
query50 771 373 370 370
query51 4391 4254 4298 4254
query52 102 88 90 88
query53 332 267 263 263
query54 289 229 240 229
query55 82 81 78 78
query56 222 201 204 201
query57 1035 920 922 920
query58 212 196 194 194
query59 2242 2074 2159 2074
query60 242 208 218 208
query61 88 82 88 82
query62 594 367 361 361
query63 278 260 255 255
query64 6582 3073 3121 3073
query65 3238 3250 3221 3221
query66 1438 321 315 315
query67 14651 14335 14244 14244
query68 4934 559 559 559
query69 545 387 385 385
query70 1281 1259 1237 1237
query71 402 255 251 251
query72 6271 2838 2709 2709
query73 704 324 328 324
query74 6242 6055 5996 5996
query75 3040 2425 2416 2416
query76 2845 1014 1159 1014
query77 351 240 234 234
query78 9240 8599 8515 8515
query79 1041 533 550 533
query80 667 361 340 340
query81 469 198 195 195
query82 1314 88 85 85
query83 247 131 116 116
query84 283 82 76 76
query85 1173 350 342 342
query86 317 308 309 308
query87 3299 3082 3095 3082
query88 2792 2356 2340 2340
query89 399 341 344 341
query90 2045 165 163 163
query91 149 125 124 124
query92 59 48 52 48
query93 1046 525 504 504
query94 1125 178 182 178
query95 437 340 344 340
query96 571 268 265 265
query97 4278 4140 4138 4138
query98 222 204 201 201
query99 1033 679 707 679
Total cold run time: 263608 ms
Total hot run time: 169525 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "doris-robot (via GitHub)" <gi...@apache.org>.
doris-robot commented on PR #31317:
URL: https://github.com/apache/doris/pull/31317#issuecomment-1971266769
<details>
<summary>TPC-DS: <b>Total hot run time: 168467 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpcds-tools
TPC-DS sf100 test result on commit 4b84f9d7a277eea6e617ded20cd5b7b82ecbe1a9, data reload: false
query1 947 345 345 345
query2 6519 1778 1769 1769
query3 6708 219 206 206
query4 23531 20533 20498 20498
query5 4311 366 368 366
query6 257 173 163 163
query7 4603 297 294 294
query8 253 188 201 188
query9 8441 2243 2238 2238
query10 421 209 212 209
query11 14604 14238 14190 14190
query12 140 83 82 82
query13 1641 425 436 425
query14 8902 6574 6580 6574
query15 211 175 173 173
query16 7093 264 265 264
query17 1021 513 496 496
query18 1892 274 262 262
query19 193 156 147 147
query20 86 83 84 83
query21 191 126 116 116
query22 4683 4544 4506 4506
query23 30569 29956 29780 29780
query24 11064 3108 3031 3031
query25 626 346 342 342
query26 1628 154 159 154
query27 2975 318 312 312
query28 6934 1798 1796 1796
query29 1117 552 548 548
query30 281 131 141 131
query31 858 669 692 669
query32 96 57 58 57
query33 728 233 228 228
query34 1046 484 500 484
query35 885 746 734 734
query36 960 867 870 867
query37 127 60 59 59
query38 3121 2989 2973 2973
query39 1301 1264 1241 1241
query40 280 99 98 98
query41 38 39 35 35
query42 102 98 95 95
query43 461 428 440 428
query44 1067 697 722 697
query45 190 177 171 171
query46 1057 773 787 773
query47 1536 1470 1440 1440
query48 426 342 351 342
query49 1174 303 291 291
query50 766 376 377 376
query51 4423 4288 4256 4256
query52 112 89 90 89
query53 333 260 256 256
query54 297 225 226 225
query55 86 81 81 81
query56 218 214 201 201
query57 980 853 866 853
query58 215 203 191 191
query59 2147 2146 1911 1911
query60 242 210 216 210
query61 86 91 83 83
query62 596 378 356 356
query63 281 262 255 255
query64 6558 3040 3128 3040
query65 3228 3244 3215 3215
query66 1444 315 308 308
query67 14602 13698 14070 13698
query68 5070 563 577 563
query69 546 378 381 378
query70 1252 1134 1275 1134
query71 362 254 247 247
query72 6244 2824 2648 2648
query73 725 322 338 322
query74 6257 5951 5890 5890
query75 3074 2420 2389 2389
query76 2867 1033 1152 1033
query77 360 229 238 229
query78 9167 8399 8540 8399
query79 1020 516 528 516
query80 671 363 355 355
query81 460 197 199 197
query82 1286 86 84 84
query83 238 121 118 118
query84 282 82 77 77
query85 1128 348 359 348
query86 313 286 303 286
query87 3239 3064 3105 3064
query88 2759 2358 2321 2321
query89 425 324 343 324
query90 1972 167 167 167
query91 154 122 125 122
query92 54 47 50 47
query93 1037 521 501 501
query94 1044 186 179 179
query95 442 342 348 342
query96 581 269 262 262
query97 4271 4139 4135 4135
query98 218 212 191 191
query99 1052 625 636 625
Total cold run time: 262479 ms
Total hot run time: 168467 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "doris-robot (via GitHub)" <gi...@apache.org>.
doris-robot commented on PR #31317:
URL: https://github.com/apache/doris/pull/31317#issuecomment-1971403127
<details>
<summary>TPC-H: <b>Total hot run time: 37820 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpch-tools
Tpch sf100 test result on commit d9c2af4561e15f527a03a81ce52c99aef371d66e, data reload: false
------ Round 1 ----------------------------------
q1 17693 4015 3992 3992
q2 2066 150 140 140
q3 10568 943 920 920
q4 4666 933 927 927
q5 7639 2986 3027 2986
q6 185 123 129 123
q7 1295 822 806 806
q8 9488 2062 2052 2052
q9 7358 6439 6428 6428
q10 8218 2543 2572 2543
q11 430 223 217 217
q12 766 315 304 304
q13 17973 2934 2958 2934
q14 284 273 258 258
q15 482 438 433 433
q16 465 388 394 388
q17 936 844 809 809
q18 6611 5919 5811 5811
q19 1563 1500 1509 1500
q20 551 284 271 271
q21 7564 3678 3730 3678
q22 813 337 300 300
Total cold run time: 107614 ms
Total hot run time: 37820 ms
----- Round 2, with runtime_filter_mode=off -----
q1 3960 3951 3962 3951
q2 319 219 220 219
q3 2946 2898 2930 2898
q4 1832 1782 1811 1782
q5 5235 5244 5225 5225
q6 198 115 114 114
q7 2233 1793 1777 1777
q8 3176 3244 3249 3244
q9 8533 8529 8550 8529
q10 6175 3736 3697 3697
q11 517 429 450 429
q12 671 526 502 502
q13 14819 2827 2803 2803
q14 293 254 247 247
q15 467 432 433 432
q16 461 401 411 401
q17 1678 1677 1652 1652
q18 7921 7402 7244 7244
q19 1628 1600 1593 1593
q20 1943 1710 1718 1710
q21 4986 4816 4854 4816
q22 549 412 410 410
Total cold run time: 70540 ms
Total hot run time: 53675 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "doris-robot (via GitHub)" <gi...@apache.org>.
doris-robot commented on PR #31317:
URL: https://github.com/apache/doris/pull/31317#issuecomment-1973169566
TeamCity be ut coverage result:
Function Coverage: 35.44% (8554/24136)
Line Coverage: 27.21% (69408/255076)
Region Coverage: 26.43% (36017/136286)
Branch Coverage: 23.26% (18394/79070)
Coverage Report: http://coverage.selectdb-in.cc/coverage/0bf659efd6e2f1bc3fd4d820801956e27990eb21_0bf659efd6e2f1bc3fd4d820801956e27990eb21/report/index.html
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "github-actions[bot] (via GitHub)" <gi...@apache.org>.
github-actions[bot] commented on code in PR #31317:
URL: https://github.com/apache/doris/pull/31317#discussion_r1500271881
##########
be/src/cloud/cloud_tablet.cpp:
##########
@@ -443,4 +491,128 @@
_cumulative_point = new_point;
}
+Status CloudTablet::update_delete_bitmap(const std::shared_ptr<CloudTablet>& self,
Review Comment:
warning: function 'update_delete_bitmap' exceeds recommended size/complexity thresholds [readability-function-size]
```cpp
Status CloudTablet::update_delete_bitmap(const std::shared_ptr<CloudTablet>& self,
^
```
<details>
<summary>Additional context</summary>
**be/src/cloud/cloud_tablet.cpp:493:** 117 lines including whitespace and comments (threshold 80)
```cpp
Status CloudTablet::update_delete_bitmap(const std::shared_ptr<CloudTablet>& self,
^
```
</details>
##########
be/src/cloud/delete_bitmap_txn_manager.cpp:
##########
@@ -0,0 +1,193 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "cloud/delete_bitmap_txn_manager.h"
+
+#include <fmt/core.h>
+
+#include <chrono>
+#include <memory>
+#include <shared_mutex>
+
+#include "common/status.h"
+#include "common/sync_point.h"
+#include "olap/olap_common.h"
+#include "olap/tablet_meta.h"
+
+namespace doris {
+
+// TODO sweep time
+DeleteBitmapTxnManager::DeleteBitmapTxnManager(size_t size_in_bytes)
+ : LRUCachePolicy(CachePolicy::CacheType::CLOUD_DELETE_BITMAP_TXN_MANAGER, size_in_bytes,
+ LRUCacheType::SIZE, config::pk_index_page_cache_stale_sweep_time_sec, 4),
+ _stop_latch(1) {}
+
+DeleteBitmapTxnManager::~DeleteBitmapTxnManager() {
+ _stop_latch.count_down();
+ _clean_thread->join();
+}
+
+Status DeleteBitmapTxnManager::init() {
Review Comment:
warning: method 'init' can be made static [readability-convert-member-functions-to-static]
be/src/cloud/delete_bitmap_txn_manager.h:36:
```diff
- Status init();
+ static Status init();
```
##########
be/src/cloud/cloud_tablet.cpp:
##########
@@ -443,4 +491,128 @@ void CloudTablet::set_cumulative_layer_point(int64_t new_point) {
_cumulative_point = new_point;
}
+Status CloudTablet::update_delete_bitmap(const std::shared_ptr<CloudTablet>& self,
Review Comment:
warning: method 'update_delete_bitmap' can be made static [readability-convert-member-functions-to-static]
```suggestion
static Status CloudTablet::update_delete_bitmap(const std::shared_ptr<CloudTablet>& self,
```
##########
be/src/olap/task/engine_calc_delete_bitmap_task.cpp:
##########
@@ -0,0 +1,162 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "olap/task/engine_calc_delete_bitmap_task.h"
+
+#include <memory>
+
+#include "cloud/cloud_meta_mgr.h"
+#include "cloud/cloud_tablet.h"
+#include "common/status.h"
+#include "olap/base_tablet.h"
+#include "olap/olap_common.h"
+#include "olap/rowset/rowset.h"
+#include "olap/tablet_fwd.h"
+#include "olap/tablet_meta.h"
+#include "olap/utils.h"
+
+namespace doris {
+
+EngineCalcDeleteBitmapTask::EngineCalcDeleteBitmapTask(
+ CloudStorageEngine& engine, const TCalcDeleteBitmapRequest& cal_delete_bitmap_req,
+ std::vector<TTabletId>* error_tablet_ids, std::vector<TTabletId>* succ_tablet_ids)
+ : _engine(engine),
+ _cal_delete_bitmap_req(cal_delete_bitmap_req),
+ _error_tablet_ids(error_tablet_ids),
+ _succ_tablet_ids(succ_tablet_ids) {}
+
+void EngineCalcDeleteBitmapTask::add_error_tablet_id(int64_t tablet_id, const Status& err) {
Review Comment:
warning: method 'add_error_tablet_id' can be made static [readability-convert-member-functions-to-static]
be/src/olap/task/engine_calc_delete_bitmap_task.h:57:
```diff
- void add_error_tablet_id(int64_t tablet_id, const Status& err);
+ static void add_error_tablet_id(int64_t tablet_id, const Status& err);
```
##########
be/src/cloud/delete_bitmap_txn_manager.cpp:
##########
@@ -0,0 +1,193 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "cloud/delete_bitmap_txn_manager.h"
+
+#include <fmt/core.h>
+
+#include <chrono>
+#include <memory>
+#include <shared_mutex>
+
+#include "common/status.h"
+#include "common/sync_point.h"
+#include "olap/olap_common.h"
+#include "olap/tablet_meta.h"
+
+namespace doris {
+
+// TODO sweep time
+DeleteBitmapTxnManager::DeleteBitmapTxnManager(size_t size_in_bytes)
+ : LRUCachePolicy(CachePolicy::CacheType::CLOUD_DELETE_BITMAP_TXN_MANAGER, size_in_bytes,
+ LRUCacheType::SIZE, config::pk_index_page_cache_stale_sweep_time_sec, 4),
+ _stop_latch(1) {}
+
+DeleteBitmapTxnManager::~DeleteBitmapTxnManager() {
Review Comment:
warning: use '= default' to define a trivial destructor [modernize-use-equals-default]
```cpp
DeleteBitmapTxnManager::~DeleteBitmapTxnManager() {
^
```
##########
be/src/cloud/delete_bitmap_txn_manager.h:
##########
@@ -0,0 +1,94 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <mutex>
+
+#include "olap/lru_cache.h"
+#include "olap/olap_common.h"
+#include "olap/partial_update_info.h"
+#include "olap/rowset/rowset.h"
+#include "olap/tablet_meta.h"
+#include "util/countdown_latch.h"
+
+namespace doris {
+
+class DeleteBitmapTxnManager : public LRUCachePolicy {
+public:
+ DeleteBitmapTxnManager(size_t size_in_bytes);
+
+ ~DeleteBitmapTxnManager();
Review Comment:
warning: annotate this function with 'override' or (rarely) 'final' [modernize-use-override]
```suggestion
~DeleteBitmapTxnManager() override;
```
##########
be/src/cloud/cloud_rowset_builder.cpp:
##########
@@ -104,4 +107,24 @@ const RowsetMetaSharedPtr& CloudRowsetBuilder::rowset_meta() {
return _rowset_writer->rowset_meta();
}
+Status CloudRowsetBuilder::set_txn_related_delete_bitmap() {
Review Comment:
warning: method 'set_txn_related_delete_bitmap' can be made static [readability-convert-member-functions-to-static]
be/src/cloud/cloud_rowset_builder.h:39:
```diff
- Status set_txn_related_delete_bitmap();
+ static Status set_txn_related_delete_bitmap();
```
##########
be/src/olap/rowset_builder.cpp:
##########
@@ -119,7 +119,7 @@ void RowsetBuilder::_garbage_collection() {
}
}
-Status RowsetBuilder::init_mow_context(std::shared_ptr<MowContext>& mow_context) {
+Status BaseRowsetBuilder::init_mow_context(std::shared_ptr<MowContext>& mow_context) {
Review Comment:
warning: method 'init_mow_context' can be made static [readability-convert-member-functions-to-static]
be/src/olap/rowset_builder.h:84:
```diff
- Status init_mow_context(std::shared_ptr<MowContext>& mow_context);
+ static Status init_mow_context(std::shared_ptr<MowContext>& mow_context);
```
##########
be/src/olap/base_tablet.cpp:
##########
@@ -1059,4 +1060,82 @@ Status BaseTablet::_capture_consistent_rowsets_unlocked(
return Status::OK();
}
+Status BaseTablet::check_delete_bitmap_correctness(DeleteBitmapPtr delete_bitmap,
Review Comment:
warning: method 'check_delete_bitmap_correctness' can be made static [readability-convert-member-functions-to-static]
```suggestion
static Status BaseTablet::check_delete_bitmap_correctness(DeleteBitmapPtr delete_bitmap,
```
##########
be/src/olap/task/engine_calc_delete_bitmap_task.cpp:
##########
@@ -0,0 +1,162 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "olap/task/engine_calc_delete_bitmap_task.h"
+
+#include <memory>
+
+#include "cloud/cloud_meta_mgr.h"
+#include "cloud/cloud_tablet.h"
+#include "common/status.h"
+#include "olap/base_tablet.h"
+#include "olap/olap_common.h"
+#include "olap/rowset/rowset.h"
+#include "olap/tablet_fwd.h"
+#include "olap/tablet_meta.h"
+#include "olap/utils.h"
+
+namespace doris {
+
+EngineCalcDeleteBitmapTask::EngineCalcDeleteBitmapTask(
+ CloudStorageEngine& engine, const TCalcDeleteBitmapRequest& cal_delete_bitmap_req,
+ std::vector<TTabletId>* error_tablet_ids, std::vector<TTabletId>* succ_tablet_ids)
+ : _engine(engine),
+ _cal_delete_bitmap_req(cal_delete_bitmap_req),
+ _error_tablet_ids(error_tablet_ids),
+ _succ_tablet_ids(succ_tablet_ids) {}
+
+void EngineCalcDeleteBitmapTask::add_error_tablet_id(int64_t tablet_id, const Status& err) {
+ std::lock_guard<std::mutex> lck(_mutex);
+ _error_tablet_ids->push_back(tablet_id);
+ if (_res.ok() || _res.is<ErrorCode::DELETE_BITMAP_LOCK_ERROR>()) {
+ _res = err;
+ }
+}
+
+void EngineCalcDeleteBitmapTask::add_succ_tablet_id(int64_t tablet_id) {
+ std::lock_guard<std::mutex> lck(_mutex);
+ _succ_tablet_ids->push_back(tablet_id);
+}
+
+Status EngineCalcDeleteBitmapTask::execute() {
+ int64_t transaction_id = _cal_delete_bitmap_req.transaction_id;
+ OlapStopWatch watch;
+ VLOG_NOTICE << "begin to calculate delete bitmap. transaction_id=" << transaction_id;
+ std::unique_ptr<ThreadPoolToken> token =
+ _engine.calc_tablet_delete_bitmap_task_thread_pool()->new_token(
+ ThreadPool::ExecutionMode::CONCURRENT);
+
+ for (const auto& partition : _cal_delete_bitmap_req.partitions) {
+ int64_t version = partition.version;
+ for (auto tablet_id : partition.tablet_ids) {
+ auto base_tablet = DORIS_TRY(_engine.get_tablet(tablet_id));
+ std::shared_ptr<CloudTablet> tablet =
+ std::dynamic_pointer_cast<CloudTablet>(base_tablet);
+ if (tablet == nullptr) {
+ LOG(WARNING) << "can't get tablet when calculate delete bitmap. tablet_id="
+ << tablet_id;
+ _error_tablet_ids->push_back(tablet_id);
+ _res = Status::Error<ErrorCode::PUSH_TABLE_NOT_EXIST>(
+ "can't get tablet when calculate delete bitmap. tablet_id={}", tablet_id);
+ break;
+ }
+
+ Status st = tablet->sync_rowsets();
+ if (!st.ok() && !st.is<ErrorCode::INVALID_TABLET_STATE>()) {
+ return st;
+ }
+ if (st.is<ErrorCode::INVALID_TABLET_STATE>()) [[unlikely]] {
+ add_succ_tablet_id(tablet->tablet_id());
+ LOG(INFO)
+ << "tablet is under alter process, delete bitmap will be calculated later, "
+ "tablet_id: "
+ << tablet->tablet_id() << " txn_id: " << transaction_id
+ << ", request_version=" << version;
+ continue;
+ }
+ int64_t max_version = tablet->max_version_unlocked();
+ if (version != max_version + 1) {
+ _error_tablet_ids->push_back(tablet_id);
+ _res = Status::Error<ErrorCode::DELETE_BITMAP_LOCK_ERROR, false>(
+ "version not continuous");
+ LOG(WARNING) << "version not continuous, current max version=" << max_version
+ << ", request_version=" << version
+ << " tablet_id=" << tablet->tablet_id();
+ break;
+ }
+
+ auto tablet_calc_delete_bitmap_ptr = std::make_shared<TabletCalcDeleteBitmapTask>(
+ _engine, this, tablet, transaction_id, version);
+ auto submit_st = token->submit_func([=]() { tablet_calc_delete_bitmap_ptr->handle(); });
+ CHECK(submit_st.ok());
+ }
+ }
+ // wait for all finished
+ token->wait();
+
+ LOG(INFO) << "finish to calculate delete bitmap on transaction."
+ << "transaction_id=" << transaction_id << ", cost(us): " << watch.get_elapse_time_us()
+ << ", error_tablet_size=" << _error_tablet_ids->size()
+ << ", res=" << _res.to_string();
+ return _res;
+}
+
+TabletCalcDeleteBitmapTask::TabletCalcDeleteBitmapTask(CloudStorageEngine& engine,
+ EngineCalcDeleteBitmapTask* engine_task,
+ std::shared_ptr<CloudTablet> tablet,
+ int64_t transaction_id, int64_t version)
+ : _engine(engine),
+ _engine_calc_delete_bitmap_task(engine_task),
+ _tablet(tablet),
+ _transaction_id(transaction_id),
+ _version(version) {}
+
+void TabletCalcDeleteBitmapTask::handle() {
Review Comment:
warning: method 'handle' can be made const [readability-make-member-function-const]
```suggestion
void TabletCalcDeleteBitmapTask::handle() const {
```
be/src/olap/task/engine_calc_delete_bitmap_task.h:38:
```diff
- void handle();
+ void handle() const;
```
##########
be/src/olap/task/engine_calc_delete_bitmap_task.cpp:
##########
@@ -0,0 +1,162 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "olap/task/engine_calc_delete_bitmap_task.h"
+
+#include <memory>
+
+#include "cloud/cloud_meta_mgr.h"
+#include "cloud/cloud_tablet.h"
+#include "common/status.h"
+#include "olap/base_tablet.h"
+#include "olap/olap_common.h"
+#include "olap/rowset/rowset.h"
+#include "olap/tablet_fwd.h"
+#include "olap/tablet_meta.h"
+#include "olap/utils.h"
+
+namespace doris {
+
+EngineCalcDeleteBitmapTask::EngineCalcDeleteBitmapTask(
+ CloudStorageEngine& engine, const TCalcDeleteBitmapRequest& cal_delete_bitmap_req,
+ std::vector<TTabletId>* error_tablet_ids, std::vector<TTabletId>* succ_tablet_ids)
+ : _engine(engine),
+ _cal_delete_bitmap_req(cal_delete_bitmap_req),
+ _error_tablet_ids(error_tablet_ids),
+ _succ_tablet_ids(succ_tablet_ids) {}
+
+void EngineCalcDeleteBitmapTask::add_error_tablet_id(int64_t tablet_id, const Status& err) {
+ std::lock_guard<std::mutex> lck(_mutex);
+ _error_tablet_ids->push_back(tablet_id);
+ if (_res.ok() || _res.is<ErrorCode::DELETE_BITMAP_LOCK_ERROR>()) {
+ _res = err;
+ }
+}
+
+void EngineCalcDeleteBitmapTask::add_succ_tablet_id(int64_t tablet_id) {
Review Comment:
warning: method 'add_succ_tablet_id' can be made static [readability-convert-member-functions-to-static]
be/src/olap/task/engine_calc_delete_bitmap_task.h:58:
```diff
- void add_succ_tablet_id(int64_t tablet_id);
+ static void add_succ_tablet_id(int64_t tablet_id);
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "doris-robot (via GitHub)" <gi...@apache.org>.
doris-robot commented on PR #31317:
URL: https://github.com/apache/doris/pull/31317#issuecomment-1960906374
TeamCity be ut coverage result:
Function Coverage: 35.66% (8548/23969)
Line Coverage: 27.49% (69356/252339)
Region Coverage: 26.66% (35975/134959)
Branch Coverage: 23.47% (18391/78358)
Coverage Report: http://coverage.selectdb-in.cc/coverage/f4f246d4ac5209b17373b1d7d25eca18329f1ea8_f4f246d4ac5209b17373b1d7d25eca18329f1ea8/report/index.html
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "liaoxin01 (via GitHub)" <gi...@apache.org>.
liaoxin01 commented on PR #31317:
URL: https://github.com/apache/doris/pull/31317#issuecomment-1960815677
run buildall
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "doris-robot (via GitHub)" <gi...@apache.org>.
doris-robot commented on PR #31317:
URL: https://github.com/apache/doris/pull/31317#issuecomment-1961201529
TeamCity be ut coverage result:
Function Coverage: 35.66% (8548/23969)
Line Coverage: 27.49% (69375/252334)
Region Coverage: 26.67% (35985/134950)
Branch Coverage: 23.48% (18395/78352)
Coverage Report: http://coverage.selectdb-in.cc/coverage/414cf06f8598a980da120a66191f071fab1c4d2a_414cf06f8598a980da120a66191f071fab1c4d2a/report/index.html
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "doris-robot (via GitHub)" <gi...@apache.org>.
doris-robot commented on PR #31317:
URL: https://github.com/apache/doris/pull/31317#issuecomment-1964486472
TeamCity be ut coverage result:
Function Coverage: 35.66% (8548/23968)
Line Coverage: 27.48% (69389/252516)
Region Coverage: 26.65% (35984/135032)
Branch Coverage: 23.47% (18401/78398)
Coverage Report: http://coverage.selectdb-in.cc/coverage/a6744cda842c2b05e73edbbe3e227234a3e0f785_a6744cda842c2b05e73edbbe3e227234a3e0f785/report/index.html
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "liaoxin01 (via GitHub)" <gi...@apache.org>.
liaoxin01 commented on PR #31317:
URL: https://github.com/apache/doris/pull/31317#issuecomment-1970314348
run buildall
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "liaoxin01 (via GitHub)" <gi...@apache.org>.
liaoxin01 commented on PR #31317:
URL: https://github.com/apache/doris/pull/31317#issuecomment-1968584637
run buildall
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "doris-robot (via GitHub)" <gi...@apache.org>.
doris-robot commented on PR #31317:
URL: https://github.com/apache/doris/pull/31317#issuecomment-1968722960
<details>
<summary>TPC-H: <b>Total hot run time: 38282 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpch-tools
Tpch sf100 test result on commit 79b90d0e6d1ee594376d460be1d722f84c24b1e8, data reload: false
------ Round 1 ----------------------------------
q1 18929 4301 4174 4174
q2 2384 152 140 140
q3 11459 973 978 973
q4 5076 975 970 970
q5 8295 2986 3011 2986
q6 185 131 128 128
q7 1250 773 754 754
q8 9351 1958 2047 1958
q9 7199 6354 6385 6354
q10 8935 2564 2507 2507
q11 412 202 220 202
q12 788 318 313 313
q13 17966 3209 3215 3209
q14 294 256 258 256
q15 534 514 495 495
q16 485 421 417 417
q17 963 899 866 866
q18 6414 5954 6094 5954
q19 1736 1523 1533 1523
q20 500 284 275 275
q21 5648 3579 3542 3542
q22 613 304 286 286
Total cold run time: 109416 ms
Total hot run time: 38282 ms
----- Round 2, with runtime_filter_mode=off -----
q1 4042 4071 4067 4067
q2 308 222 222 222
q3 2990 2955 2945 2945
q4 1866 1811 1830 1811
q5 5105 5159 5136 5136
q6 200 116 118 116
q7 2177 1774 1730 1730
q8 3167 3218 3225 3218
q9 8315 8265 8284 8265
q10 4487 3658 3647 3647
q11 521 418 429 418
q12 710 526 530 526
q13 3738 3095 3066 3066
q14 269 260 267 260
q15 534 499 496 496
q16 519 496 467 467
q17 1705 1705 1695 1695
q18 7845 7603 7464 7464
q19 1673 1678 1662 1662
q20 2092 1912 1884 1884
q21 4812 4714 4648 4648
q22 514 450 422 422
Total cold run time: 57589 ms
Total hot run time: 54165 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "doris-robot (via GitHub)" <gi...@apache.org>.
doris-robot commented on PR #31317:
URL: https://github.com/apache/doris/pull/31317#issuecomment-1969063099
<details>
<summary>TPC-DS: <b>Total hot run time: 168791 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpcds-tools
TPC-DS sf100 test result on commit 0a23e751291ddf15970f07c0384db0ab6c5606aa, data reload: false
query1 945 351 349 349
query2 6523 1777 1818 1777
query3 6698 205 207 205
query4 23322 20579 20664 20579
query5 4293 362 350 350
query6 268 176 169 169
query7 4603 309 303 303
query8 265 191 197 191
query9 8446 2201 2202 2201
query10 413 224 213 213
query11 14517 14222 14128 14128
query12 146 78 86 78
query13 1638 418 424 418
query14 8659 6281 6338 6281
query15 201 172 174 172
query16 7086 261 249 249
query17 1047 518 495 495
query18 1899 271 270 270
query19 190 151 149 149
query20 88 82 85 82
query21 190 119 124 119
query22 4729 4622 4546 4546
query23 30554 29838 29937 29838
query24 11815 3122 3092 3092
query25 650 349 351 349
query26 1836 148 156 148
query27 3067 334 313 313
query28 7050 1796 1778 1778
query29 1170 546 557 546
query30 285 132 140 132
query31 856 681 704 681
query32 101 58 55 55
query33 718 218 222 218
query34 1046 477 494 477
query35 908 753 759 753
query36 960 881 829 829
query37 162 62 62 62
query38 3107 3002 3027 3002
query39 1314 1253 1240 1240
query40 286 100 101 100
query41 38 36 35 35
query42 101 107 98 98
query43 449 456 469 456
query44 1054 688 693 688
query45 187 181 171 171
query46 1028 780 766 766
query47 1541 1437 1448 1437
query48 420 352 344 344
query49 1142 293 298 293
query50 772 375 372 372
query51 4372 4373 4327 4327
query52 101 86 95 86
query53 339 265 263 263
query54 278 232 233 232
query55 86 84 88 84
query56 210 199 198 198
query57 999 895 893 893
query58 207 188 192 188
query59 2243 2198 1968 1968
query60 234 211 239 211
query61 88 102 82 82
query62 589 369 360 360
query63 279 248 261 248
query64 6592 2987 3113 2987
query65 3262 3211 3203 3203
query66 1448 316 308 308
query67 14567 14093 13687 13687
query68 4033 568 563 563
query69 513 369 387 369
query70 1243 1232 1228 1228
query71 320 246 253 246
query72 6109 2810 2652 2652
query73 717 327 320 320
query74 6227 6014 6020 6014
query75 3094 2437 2422 2422
query76 2340 1049 1157 1049
query77 343 237 229 229
query78 8921 8571 8508 8508
query79 1291 517 514 514
query80 1006 357 359 357
query81 468 195 195 195
query82 947 87 81 81
query83 226 125 117 117
query84 274 79 79 79
query85 1155 355 354 354
query86 362 271 297 271
query87 3265 3139 3084 3084
query88 2723 2363 2341 2341
query89 394 329 323 323
query90 1920 163 163 163
query91 149 123 123 123
query92 59 52 50 50
query93 1224 502 502 502
query94 1266 180 180 180
query95 439 339 343 339
query96 571 267 271 267
query97 4275 4110 4103 4103
query98 226 202 194 194
query99 1055 691 632 632
Total cold run time: 261832 ms
Total hot run time: 168791 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "doris-robot (via GitHub)" <gi...@apache.org>.
doris-robot commented on PR #31317:
URL: https://github.com/apache/doris/pull/31317#issuecomment-1971243316
<details>
<summary>TPC-H: <b>Total hot run time: 38441 ms</b></summary>
```
machine: 'aliyun_ecs.c7a.8xlarge_32C64G'
scripts: https://github.com/apache/doris/tree/master/tools/tpch-tools
Tpch sf100 test result on commit 4b84f9d7a277eea6e617ded20cd5b7b82ecbe1a9, data reload: false
------ Round 1 ----------------------------------
q1 18979 4203 4209 4203
q2 2564 155 140 140
q3 11071 980 974 974
q4 5684 977 984 977
q5 7897 2967 3104 2967
q6 187 129 128 128
q7 1291 791 783 783
q8 9640 1973 2037 1973
q9 7217 6372 6303 6303
q10 8193 2513 2498 2498
q11 419 205 213 205
q12 758 311 313 311
q13 18025 3209 3241 3209
q14 287 251 255 251
q15 541 498 496 496
q16 476 410 402 402
q17 957 912 895 895
q18 6610 6077 6017 6017
q19 1584 1530 1524 1524
q20 479 285 292 285
q21 6257 3641 3616 3616
q22 805 290 284 284
Total cold run time: 109921 ms
Total hot run time: 38441 ms
----- Round 2, with runtime_filter_mode=off -----
q1 4061 4041 4046 4041
q2 311 213 214 213
q3 2991 2978 2912 2912
q4 1867 1853 1841 1841
q5 5133 5136 5093 5093
q6 200 116 119 116
q7 2142 1709 1739 1709
q8 3133 3205 3223 3205
q9 8310 8287 8224 8224
q10 6235 3662 3690 3662
q11 521 434 432 432
q12 691 524 530 524
q13 3727 3087 3095 3087
q14 277 252 240 240
q15 545 494 500 494
q16 526 447 472 447
q17 1701 1682 1682 1682
q18 7861 7688 7459 7459
q19 1646 1655 1651 1651
q20 2103 1909 1927 1909
q21 4817 4653 4720 4653
q22 501 432 444 432
Total cold run time: 59299 ms
Total hot run time: 54026 ms
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
Re: [PR] [feature](merge-cloud) support MoW table in cloud mode [doris]
Posted by "github-actions[bot] (via GitHub)" <gi...@apache.org>.
github-actions[bot] commented on PR #31317:
URL: https://github.com/apache/doris/pull/31317#issuecomment-1971224396
PR approved by anyone and no changes requested.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org