You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by GitBox <gi...@apache.org> on 2020/08/10 14:03:32 UTC
[GitHub] [incubator-doris] morningman commented on a change in pull request #4312: Manually trigger compaction restapi interface
morningman commented on a change in pull request #4312:
URL: https://github.com/apache/incubator-doris/pull/4312#discussion_r467914921
##########
File path: be/src/common/config.h
##########
@@ -256,6 +256,9 @@ namespace config {
CONF_Bool(disable_storage_page_cache, "false");
// be policy
+ // whether open automatic compaction task
+ CONF_Bool(compaction_automatic_switch, "true");
Review comment:
better called it `disable_auto_compaction`
a `switch` usually corresponds to `on` and `off`
##########
File path: be/src/olap/olap_server.cpp
##########
@@ -91,35 +92,38 @@ Status StorageEngine::start_bg_threads() {
// calc the max concurrency of compaction tasks
int32_t max_compaction_concurrency = config::max_compaction_concurrency;
if (max_compaction_concurrency < 0
- || max_compaction_concurrency > base_compaction_num_threads + cumulative_compaction_num_threads) {
- max_compaction_concurrency = base_compaction_num_threads + cumulative_compaction_num_threads;
+ || max_compaction_concurrency > base_compaction_num_threads + cumulative_compaction_num_threads + 1) {
+ // reserve 1 thread for manual execution
+ max_compaction_concurrency = base_compaction_num_threads + cumulative_compaction_num_threads + 1;
}
Compaction::init(max_compaction_concurrency);
- _base_compaction_threads.reserve(base_compaction_num_threads);
- for (uint32_t i = 0; i < base_compaction_num_threads; ++i) {
- _base_compaction_threads.emplace_back(
- [this, data_dir_num, data_dirs, i] {
- _base_compaction_thread_callback(nullptr, data_dirs[i % data_dir_num]);
- });
- }
- for (auto& thread : _base_compaction_threads) {
- thread.detach();
- }
- LOG(INFO) << "base compaction threads started. number: " << base_compaction_num_threads;
-
- _cumulative_compaction_threads.reserve(cumulative_compaction_num_threads);
- for (uint32_t i = 0; i < cumulative_compaction_num_threads; ++i) {
- _cumulative_compaction_threads.emplace_back(
- [this, data_dir_num, data_dirs, i] {
- _cumulative_compaction_thread_callback(nullptr, data_dirs[i % data_dir_num]);
- });
- }
- for (auto& thread : _cumulative_compaction_threads) {
- thread.detach();
+ // check whether automatic switch is on
+ if (config::compaction_automatic_switch) {
Review comment:
This config `compaction_automatic_switch` can be modified at runtime.
So I suggest to use this config to disable the compaction thread from running. But the thread will be created here, just not doing anything.
##########
File path: docs/zh-CN/administrator-guide/http-actions/compaction-action.md
##########
@@ -95,5 +95,68 @@ curl -X GET http://192.168.10.24:8040/api/compaction/show?tablet_id=10015\&schem
## 手动触发 Compaction
-(TODO)
+```
+curl -X GET http://be_host:webserver_port/api/compaction/run?tablet_id=xxxx\&schema_hash=yyyy\&compact_type=cumulative
+```
+
+当前仅能执行一个手动compaction任务，其中compact_type取值为base或cumulative
+
+若 tablet 不存在，返回 JSON 格式的错误：
+
+```
+{
+ "status": "Fail",
+ "msg": "Tablet not found"
+}
+```
+若 compaction 执行任务触发失败时，返回 JSON 格式的错误：
+
+```
+{
+ "status": "Fail",
+ "msg": "fail to execute compaction, error = -2000"
+}
+```
+
+若 compaction 执行触发成功时，则返回 JSON 格式的结果:
+
+```
+{
+ "status": "Success",
+ "msg": "compaction task is successfully trigged."
+}
+```
+
+结果说明：
+
+* status：触发任务状态，当成功触发时为Success；当因某些原因（比如，没有获取到合适的版本）时，返回Fail。
+* msg：给出具体的成功或失败的信息。
+### 示例
+
+```
+curl -X GET http://192.168.10.24:8040/api/compaction/run?tablet_id=10015\&schema_hash=1294206575\&compact_type=cumulative
+```
+
+## 手动 Compaction 执行状态
+
+```
+curl -X GET http://be_host:webserver_port/api/compaction/run_status
Review comment:
Missing `tablet_id` and `schema_hash`?
##########
File path: be/src/http/action/compaction_action.cpp
##########
@@ -67,6 +72,127 @@ Status CompactionAction::_handle_show_compaction(HttpRequest* req, std::string*
return Status::OK();
}
+Status CompactionAction::_handle_run_compaction(HttpRequest *req, std::string* json_result) {
+
+ std::string req_tablet_id = req->param(TABLET_ID_KEY);
+ std::string req_schema_hash = req->param(TABLET_SCHEMA_HASH_KEY);
+ std::string compaction_type = req->param(PARAM_COMPACTION_TYPE);
+
+ // 1. param check
+ // check req_tablet_id and req_schema_hash is not empty
+ if (req_tablet_id == "" && req_schema_hash == "") {
+ return Status::NotSupported("The overall compaction status is not supported yet");
+ }
+
+ // check compaction_type is not empty and equals base or cumulative
+ if (compaction_type == "" && !(compaction_type == PARAM_COMPACTION_BASE || compaction_type == PARAM_COMPACTION_CUMULATIVE)) {
+ return Status::NotSupported("The compaction type is not supported");
+ }
+
+ // convert req_tablet_id amd req_schema_hash to int
+ uint64_t tablet_id = 0;
+ uint32_t schema_hash = 0;
+ try {
+ tablet_id = std::stoull(req_tablet_id);
+ schema_hash = std::stoul(req_schema_hash);
+ } catch (const std::exception& e) {
+ LOG(WARNING) << "invalid argument.tablet_id:" << req_tablet_id
+ << ", schema_hash:" << req_schema_hash;
+ return Status::InternalError(strings::Substitute("convert failed, $0", e.what()));
+ }
+
+ // 2. fetch the tablet by tablet_id and schema_hash
+ TabletSharedPtr tablet =
+ StorageEngine::instance()->tablet_manager()->get_tablet(tablet_id, schema_hash);
+
+ if (tablet == nullptr) {
+ LOG(WARNING) << "invalid argument.tablet_id:" << req_tablet_id
+ << ", schema_hash:" << req_schema_hash;
+ return Status::InternalError(
+ strings::Substitute("fail to get $0, $1", req_tablet_id, req_schema_hash));
+ }
+
+ // 3. execute compaction task
+ std::packaged_task<OLAPStatus()> task([this, tablet, compaction_type]() {
+ return _execute_compaction_callback(tablet, compaction_type);
+ });
+ std::future<OLAPStatus> future_obj = task.get_future();
+
+ {
+ // 3.1 check is there compaction running
+ std::lock_guard<std::mutex> lock(_compaction_running_mutex);
+ if (_is_compaction_running) {
+ return Status::TooManyTasks("Manual compaction task is running");
+ } else {
+ // 3.2 execute the compaction task and set compaction task running
+ _is_compaction_running = true;
+ std::thread(std::move(task)).detach();
+ }
+ }
+
+ // 4. wait for result for 2 seconds by async
+ std::future_status status = future_obj.wait_for(std::chrono::seconds(2));
+ if (status == std::future_status::ready) {
+ // fetch execute result
+ OLAPStatus olap_status = future_obj.get();
+ if (olap_status != OLAP_SUCCESS) {
+ return Status::InternalError(
+ strings::Substitute("fail to execute compaction, error = $0", olap_status));
+ }
+ } else {
+ LOG(INFO) << "Manual compaction task is timeout for waiting " << (status == std::future_status::timeout);
+ }
+
+ LOG(INFO) << "Manual compaction task is successfully trigged";
+ *json_result = "{\"status\": \"Success\", \"msg\": \"compaction task is successfully trigged.\"}";
Review comment:
trigged -> triggered
##########
File path: docs/zh-CN/administrator-guide/http-actions/compaction-action.md
##########
@@ -95,5 +95,68 @@ curl -X GET http://192.168.10.24:8040/api/compaction/show?tablet_id=10015\&schem
## 手动触发 Compaction
-(TODO)
+```
+curl -X GET http://be_host:webserver_port/api/compaction/run?tablet_id=xxxx\&schema_hash=yyyy\&compact_type=cumulative
+```
+
+当前仅能执行一个手动compaction任务，其中compact_type取值为base或cumulative
+
+若 tablet 不存在，返回 JSON 格式的错误：
+
+```
+{
+ "status": "Fail",
+ "msg": "Tablet not found"
+}
+```
+若 compaction 执行任务触发失败时，返回 JSON 格式的错误：
+
+```
+{
+ "status": "Fail",
+ "msg": "fail to execute compaction, error = -2000"
+}
+```
+
+若 compaction 执行触发成功时，则返回 JSON 格式的结果:
+
+```
+{
+ "status": "Success",
+ "msg": "compaction task is successfully trigged."
Review comment:
```suggestion
"msg": "compaction task is successfully triggered."
```
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org