You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by da...@apache.org on 2024/02/27 13:00:01 UTC
(doris) branch master updated: (merge-cloud) [Feature] Support to do schema change in cloud mode (#31055)
This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 73184b94323 (merge-cloud) [Feature] Support to do schema change in cloud mode (#31055)
73184b94323 is described below
commit 73184b943232195fcb5b98ffc10ea7bd45720241
Author: Lightman <31...@users.noreply.github.com>
AuthorDate: Tue Feb 27 20:59:54 2024 +0800
(merge-cloud) [Feature] Support to do schema change in cloud mode (#31055)
---
be/src/agent/agent_server.cpp | 5 +-
be/src/agent/task_worker_pool.cpp | 126 ++++-
be/src/agent/task_worker_pool.h | 2 +
be/src/cloud/cloud_meta_mgr.cpp | 1 -
be/src/cloud/cloud_rowset_writer.cpp | 7 +-
be/src/cloud/cloud_schema_change_job.cpp | 361 +++++++++++++++
be/src/cloud/cloud_schema_change_job.h | 55 +++
be/src/cloud/pb_convert.cpp | 8 +-
be/src/http/action/cloud_compaction_action.cpp | 41 ++
.../action/cloud_compaction_action.h} | 33 +-
be/src/io/cache/block/block_file_cache_factory.h | 6 +
be/src/olap/base_tablet.h | 4 +
be/src/olap/merger.cpp | 1 +
be/src/olap/schema_change.cpp | 511 ++++++++++-----------
be/src/olap/schema_change.h | 164 ++++---
be/src/olap/tablet.h | 3 -
be/src/olap/task/engine_alter_tablet_task.cpp | 62 ---
be/src/service/http_service.cpp | 19 +
.../java/org/apache/doris/alter/AlterHandler.java | 5 +-
.../java/org/apache/doris/alter/AlterJobV2.java | 28 ++
.../org/apache/doris/alter/AlterJobV2Factory.java | 76 +++
.../org/apache/doris/alter/CloudRollupJobV2.java | 220 +++++++++
.../apache/doris/alter/CloudSchemaChangeJobV2.java | 229 +++++++++
.../doris/alter/MaterializedViewHandler.java | 26 +-
.../java/org/apache/doris/alter/RollupJobV2.java | 98 ++--
.../apache/doris/alter/SchemaChangeHandler.java | 37 +-
.../org/apache/doris/alter/SchemaChangeJobV2.java | 84 ++--
.../main/java/org/apache/doris/catalog/Env.java | 6 +-
.../java/org/apache/doris/catalog/EnvFactory.java | 4 +
.../java/org/apache/doris/catalog/Replica.java | 17 +
.../doris/cloud/catalog/CloudEnvFactory.java | 5 +
.../apache/doris/cloud/catalog/CloudReplica.java | 10 +-
.../cloud/datasource/CloudInternalCatalog.java | 15 +-
.../org/apache/doris/persist/gson/GsonUtils.java | 6 +-
.../java/org/apache/doris/qe/SessionVariable.java | 2 +-
.../org/apache/doris/task/AlterReplicaTask.java | 7 +-
.../org/apache/doris/alter/RollupJobV2Test.java | 3 +-
.../apache/doris/alter/SchemaChangeJobV2Test.java | 3 +-
.../cloud_p0/conf/regression-conf-custom.groovy | 2 +-
39 files changed, 1756 insertions(+), 536 deletions(-)
diff --git a/be/src/agent/agent_server.cpp b/be/src/agent/agent_server.cpp
index 7a462f4824c..269ee490224 100644
--- a/be/src/agent/agent_server.cpp
+++ b/be/src/agent/agent_server.cpp
@@ -192,10 +192,13 @@ void AgentServer::cloud_start_workers(CloudStorageEngine& engine, ExecEnv* exec_
_workers[TTaskType::PUSH] = std::make_unique<TaskWorkerPool>(
"PUSH", config::delete_worker_count,
[&engine](auto&& task) { cloud_push_callback(engine, task); });
- // TODO(plat1ko): ALTER worker
// TODO(plat1ko): SUBMIT_TABLE_COMPACTION worker
// TODO(plat1ko): CALCULATE_DELETE_BITMAP worker
+ _workers[TTaskType::ALTER] = std::make_unique<TaskWorkerPool>(
+ "ALTER_TABLE", config::alter_tablet_worker_count,
+ [&engine](auto&& task) { return alter_cloud_tablet_callback(engine, task); });
+
_report_workers.push_back(std::make_unique<ReportWorker>(
"REPORT_TASK", _master_info, config::report_task_interval_seconds,
[&master_info = _master_info] { report_task_callback(master_info); }));
diff --git a/be/src/agent/task_worker_pool.cpp b/be/src/agent/task_worker_pool.cpp
index 2b327f360fa..35d9be268f1 100644
--- a/be/src/agent/task_worker_pool.cpp
+++ b/be/src/agent/task_worker_pool.cpp
@@ -45,6 +45,7 @@
#include "agent/utils.h"
#include "cloud/cloud_delete_task.h"
+#include "cloud/cloud_schema_change_job.h"
#include "common/config.h"
#include "common/logging.h"
#include "common/status.h"
@@ -63,7 +64,6 @@
#include "olap/tablet_manager.h"
#include "olap/tablet_meta.h"
#include "olap/tablet_schema.h"
-#include "olap/task/engine_alter_tablet_task.h"
#include "olap/task/engine_batch_load_task.h"
#include "olap/task/engine_checksum_task.h"
#include "olap/task/engine_clone_task.h"
@@ -166,20 +166,7 @@ void alter_tablet(StorageEngine& engine, const TAgentTaskRequest& agent_task_req
const TTaskType::type task_type, TFinishTaskRequest* finish_task_request) {
Status status;
- std::string_view process_name;
- switch (task_type) {
- case TTaskType::ALTER:
- process_name = "alter tablet";
- break;
- default:
- std::string task_name;
- EnumToString(TTaskType, task_type, task_name);
- LOG(WARNING) << "schema change type invalid. type: " << task_name
- << ", signature: " << signature;
- status = Status::NotSupported("Schema change type invalid");
- break;
- }
-
+ std::string_view process_name = "alter tablet";
// Check last schema change status, if failed delete tablet file
// Do not need to adjust delete success or not
// Because if delete failed create rollup will failed
@@ -188,8 +175,25 @@ void alter_tablet(StorageEngine& engine, const TAgentTaskRequest& agent_task_req
if (status.ok()) {
new_tablet_id = agent_task_req.alter_tablet_req_v2.new_tablet_id;
new_schema_hash = agent_task_req.alter_tablet_req_v2.new_schema_hash;
- EngineAlterTabletTask engine_task(agent_task_req.alter_tablet_req_v2);
- status = engine_task.execute();
+ auto mem_tracker = std::make_shared<MemTrackerLimiter>(
+ MemTrackerLimiter::Type::SCHEMA_CHANGE,
+ fmt::format("EngineAlterTabletTask#baseTabletId={}:newTabletId={}",
+ std::to_string(agent_task_req.alter_tablet_req_v2.base_tablet_id),
+ std::to_string(agent_task_req.alter_tablet_req_v2.new_tablet_id),
+ config::memory_limitation_per_thread_for_schema_change_bytes));
+ SCOPED_ATTACH_TASK(mem_tracker);
+ DorisMetrics::instance()->create_rollup_requests_total->increment(1);
+ Status res = Status::OK();
+ try {
+ DCHECK(agent_task_req.alter_tablet_req_v2.__isset.job_id);
+ SchemaChangeJob job(engine, agent_task_req.alter_tablet_req_v2);
+ status = job.process_alter_tablet(agent_task_req.alter_tablet_req_v2);
+ } catch (const Exception& e) {
+ status = e.to_status();
+ }
+ if (!status.ok()) {
+ DorisMetrics::instance()->create_rollup_requests_failed->increment(1);
+ }
}
if (status.ok()) {
@@ -227,6 +231,66 @@ void alter_tablet(StorageEngine& engine, const TAgentTaskRequest& agent_task_req
finish_task_request->__set_task_status(status.to_thrift());
}
+void alter_cloud_tablet(CloudStorageEngine& engine, const TAgentTaskRequest& agent_task_req,
+ int64_t signature, const TTaskType::type task_type,
+ TFinishTaskRequest* finish_task_request) {
+ Status status;
+
+ std::string_view process_name = "alter tablet";
+ // Check last schema change status, if failed delete tablet file
+ // Do not need to adjust delete success or not
+ // Because if delete failed create rollup will failed
+ TTabletId new_tablet_id = 0;
+ if (status.ok()) {
+ new_tablet_id = agent_task_req.alter_tablet_req_v2.new_tablet_id;
+ auto mem_tracker = std::make_shared<MemTrackerLimiter>(
+ MemTrackerLimiter::Type::SCHEMA_CHANGE,
+ fmt::format("EngineAlterTabletTask#baseTabletId={}:newTabletId={}",
+ std::to_string(agent_task_req.alter_tablet_req_v2.base_tablet_id),
+ std::to_string(agent_task_req.alter_tablet_req_v2.new_tablet_id),
+ config::memory_limitation_per_thread_for_schema_change_bytes));
+ SCOPED_ATTACH_TASK(mem_tracker);
+ DorisMetrics::instance()->create_rollup_requests_total->increment(1);
+ Status res = Status::OK();
+ try {
+ DCHECK(agent_task_req.alter_tablet_req_v2.__isset.job_id);
+ CloudSchemaChangeJob job(engine,
+ std::to_string(agent_task_req.alter_tablet_req_v2.job_id),
+ agent_task_req.alter_tablet_req_v2.expiration);
+ status = job.process_alter_tablet(agent_task_req.alter_tablet_req_v2);
+ } catch (const Exception& e) {
+ status = e.to_status();
+ }
+ if (!status.ok()) {
+ DorisMetrics::instance()->create_rollup_requests_failed->increment(1);
+ }
+ }
+
+ if (status.ok()) {
+ s_report_version.fetch_add(1, std::memory_order_relaxed);
+ }
+
+ // Return result to fe
+ finish_task_request->__set_backend(BackendOptions::get_local_backend());
+ finish_task_request->__set_report_version(s_report_version);
+ finish_task_request->__set_task_type(task_type);
+ finish_task_request->__set_signature(signature);
+
+ if (!status.ok() && !status.is<NOT_IMPLEMENTED_ERROR>()) {
+ LOG_WARNING("failed to {}", process_name)
+ .tag("signature", agent_task_req.signature)
+ .tag("base_tablet_id", agent_task_req.alter_tablet_req_v2.base_tablet_id)
+ .tag("new_tablet_id", new_tablet_id)
+ .error(status);
+ } else {
+ LOG_INFO("successfully {}", process_name)
+ .tag("signature", agent_task_req.signature)
+ .tag("base_tablet_id", agent_task_req.alter_tablet_req_v2.base_tablet_id)
+ .tag("new_tablet_id", new_tablet_id);
+ }
+ finish_task_request->__set_task_status(status.to_thrift());
+}
+
Status check_migrate_request(StorageEngine& engine, const TStorageMediumMigrateReq& req,
TabletSharedPtr& tablet, DataDir** dest_store) {
int64_t tablet_id = req.tablet_id;
@@ -1681,14 +1745,28 @@ void alter_tablet_callback(StorageEngine& engine, const TAgentTaskRequest& req)
if (!is_task_timeout) {
TFinishTaskRequest finish_task_request;
TTaskType::type task_type = req.task_type;
- switch (task_type) {
- case TTaskType::ALTER:
- alter_tablet(engine, req, signature, task_type, &finish_task_request);
- break;
- default:
- // pass
- break;
+ alter_tablet(engine, req, signature, task_type, &finish_task_request);
+ finish_task(finish_task_request);
+ }
+ remove_task_info(req.task_type, req.signature);
+}
+
+void alter_cloud_tablet_callback(CloudStorageEngine& engine, const TAgentTaskRequest& req) {
+ int64_t signature = req.signature;
+ LOG(INFO) << "get alter table task, signature: " << signature;
+ bool is_task_timeout = false;
+ if (req.__isset.recv_time) {
+ int64_t time_elapsed = time(nullptr) - req.recv_time;
+ if (time_elapsed > config::report_task_interval_seconds * 20) {
+ LOG(INFO) << "task elapsed " << time_elapsed
+ << " seconds since it is inserted to queue, it is timeout";
+ is_task_timeout = true;
}
+ }
+ if (!is_task_timeout) {
+ TFinishTaskRequest finish_task_request;
+ TTaskType::type task_type = req.task_type;
+ alter_cloud_tablet(engine, req, signature, task_type, &finish_task_request);
finish_task(finish_task_request);
}
remove_task_info(req.task_type, req.signature);
diff --git a/be/src/agent/task_worker_pool.h b/be/src/agent/task_worker_pool.h
index 5f090539d4f..531a64655a1 100644
--- a/be/src/agent/task_worker_pool.h
+++ b/be/src/agent/task_worker_pool.h
@@ -165,6 +165,8 @@ void update_tablet_meta_callback(StorageEngine& engine, const TAgentTaskRequest&
void alter_tablet_callback(StorageEngine& engine, const TAgentTaskRequest& req);
+void alter_cloud_tablet_callback(CloudStorageEngine& engine, const TAgentTaskRequest& req);
+
void clone_callback(StorageEngine& engine, const TMasterInfo& master_info,
const TAgentTaskRequest& req);
diff --git a/be/src/cloud/cloud_meta_mgr.cpp b/be/src/cloud/cloud_meta_mgr.cpp
index f3fdc1e3687..c3eb63bdb01 100644
--- a/be/src/cloud/cloud_meta_mgr.cpp
+++ b/be/src/cloud/cloud_meta_mgr.cpp
@@ -620,7 +620,6 @@ Status CloudMetaMgr::prepare_rowset(const RowsetMeta& rs_meta, bool is_tmp,
req.set_temporary(is_tmp);
RowsetMetaPB doris_rs_meta = rs_meta.get_rowset_pb(/*skip_schema=*/true);
- rs_meta.to_rowset_pb(&doris_rs_meta, true);
doris_rowset_meta_to_cloud(req.mutable_rowset_meta(), std::move(doris_rs_meta));
Status st = retry_rpc("prepare rowset", req, &resp, &MetaService_Stub::prepare_rowset);
diff --git a/be/src/cloud/cloud_rowset_writer.cpp b/be/src/cloud/cloud_rowset_writer.cpp
index 410614e52fe..f7e4f006194 100644
--- a/be/src/cloud/cloud_rowset_writer.cpp
+++ b/be/src/cloud/cloud_rowset_writer.cpp
@@ -17,6 +17,7 @@
#include "cloud/cloud_rowset_writer.h"
+#include "io/cache/block/block_file_cache_factory.h"
#include "olap/rowset/rowset_factory.h"
namespace doris {
@@ -31,9 +32,9 @@ Status CloudRowsetWriter::init(const RowsetWriterContext& rowset_writer_context)
if (_context.fs) {
_rowset_meta->set_fs(_context.fs);
} else {
- // TODO(plat1ko):
// In cloud mode, this branch implies it is an intermediate rowset for external merge sort,
// we use `global_local_filesystem` to write data to `tmp_file_dir`(see `BetaRowset::segment_file_path`).
+ _context.rowset_dir = io::FileCacheFactory::instance()->get_cache_path();
}
_rowset_meta->set_rowset_id(_context.rowset_id);
_rowset_meta->set_partition_id(_context.partition_id);
@@ -100,6 +101,10 @@ Status CloudRowsetWriter::build(RowsetSharedPtr& rowset) {
_rowset_meta->set_tablet_schema(_context.tablet_schema);
}
+ if (_rowset_meta->newest_write_timestamp() == -1) {
+ _rowset_meta->set_newest_write_timestamp(UnixSeconds());
+ }
+
// TODO(plat1ko): Record segment file size in rowset meta
RETURN_NOT_OK_STATUS_WITH_WARN(
diff --git a/be/src/cloud/cloud_schema_change_job.cpp b/be/src/cloud/cloud_schema_change_job.cpp
new file mode 100644
index 00000000000..e74c21a5501
--- /dev/null
+++ b/be/src/cloud/cloud_schema_change_job.cpp
@@ -0,0 +1,361 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "cloud/cloud_schema_change_job.h"
+
+#include <gen_cpp/cloud.pb.h>
+
+#include <memory>
+
+#include "cloud/cloud_meta_mgr.h"
+#include "cloud/cloud_tablet_mgr.h"
+#include "common/status.h"
+#include "olap/delete_handler.h"
+#include "olap/rowset/beta_rowset.h"
+#include "olap/rowset/rowset_factory.h"
+#include "olap/rowset/segment_v2/inverted_index_desc.h"
+#include "olap/storage_engine.h"
+#include "olap/tablet.h"
+#include "olap/tablet_fwd.h"
+#include "olap/tablet_meta.h"
+#include "service/backend_options.h"
+
+namespace doris {
+using namespace ErrorCode;
+
+static constexpr int ALTER_TABLE_BATCH_SIZE = 4096;
+
+static std::unique_ptr<SchemaChange> get_sc_procedure(const BlockChanger& changer,
+ bool sc_sorting) {
+ if (sc_sorting) {
+ return std::make_unique<VBaseSchemaChangeWithSorting>(
+ changer, config::memory_limitation_per_thread_for_schema_change_bytes);
+ }
+ // else sc_directly
+ return std::make_unique<VSchemaChangeDirectly>(changer);
+}
+
+CloudSchemaChangeJob::CloudSchemaChangeJob(CloudStorageEngine& cloud_storage_engine,
+ std::string job_id, int64_t expiration)
+ : _cloud_storage_engine(cloud_storage_engine),
+ _job_id(std::move(job_id)),
+ _expiration(expiration) {}
+
+CloudSchemaChangeJob::~CloudSchemaChangeJob() = default;
+
+Status CloudSchemaChangeJob::process_alter_tablet(const TAlterTabletReqV2& request) {
+ LOG(INFO) << "Begin to alter tablet. base_tablet_id=" << request.base_tablet_id
+ << ", new_tablet_id=" << request.new_tablet_id
+ << ", alter_version=" << request.alter_version << ", job_id=" << _job_id;
+
+ // new tablet has to exist
+ _new_tablet = DORIS_TRY(_cloud_storage_engine.tablet_mgr().get_tablet(request.new_tablet_id));
+ if (_new_tablet->tablet_state() == TABLET_RUNNING) {
+ LOG(INFO) << "schema change job has already finished. base_tablet_id="
+ << request.base_tablet_id << ", new_tablet_id=" << request.new_tablet_id
+ << ", alter_version=" << request.alter_version << ", job_id=" << _job_id;
+ return Status::OK();
+ }
+
+ _base_tablet = DORIS_TRY(_cloud_storage_engine.tablet_mgr().get_tablet(request.base_tablet_id));
+
+ std::unique_lock<std::mutex> schema_change_lock(_base_tablet->get_schema_change_lock(),
+ std::try_to_lock);
+ if (!schema_change_lock.owns_lock()) {
+ LOG(WARNING) << "Failed to obtain schema change lock. base_tablet="
+ << request.base_tablet_id;
+ return Status::Error<TRY_LOCK_FAILED>("Failed to obtain schema change lock. base_tablet={}",
+ request.base_tablet_id);
+ }
+ // MUST sync rowsets before capturing rowset readers and building DeleteHandler
+ RETURN_IF_ERROR(_base_tablet->sync_rowsets(request.alter_version));
+ // ATTN: Only convert rowsets of version larger than 1, MUST let the new tablet cache have rowset [0-1]
+ _output_cumulative_point = _base_tablet->cumulative_layer_point();
+
+ std::vector<RowSetSplits> rs_splits;
+ int64_t base_max_version = _base_tablet->max_version_unlocked();
+ if (request.alter_version > 1) {
+ // [0-1] is a placeholder rowset, no need to convert
+ RETURN_IF_ERROR(_base_tablet->capture_rs_readers({2, base_max_version}, &rs_splits, false));
+ }
+ // FIXME(cyx): Should trigger compaction on base_tablet if there are too many rowsets to convert.
+
+ // Create a new tablet schema, should merge with dropped columns in light weight schema change
+ _base_tablet_schema = std::make_shared<TabletSchema>();
+ _base_tablet_schema->update_tablet_columns(*_base_tablet->tablet_schema(), request.columns);
+ _new_tablet_schema = _new_tablet->tablet_schema();
+
+ // delete handlers to filter out deleted rows
+ DeleteHandler delete_handler;
+ std::vector<RowsetMetaSharedPtr> delete_predicates;
+ for (auto& split : rs_splits) {
+ auto& rs_meta = split.rs_reader->rowset()->rowset_meta();
+ if (rs_meta->has_delete_predicate()) {
+ _base_tablet_schema->merge_dropped_columns(*rs_meta->tablet_schema());
+ delete_predicates.push_back(rs_meta);
+ }
+ }
+ RETURN_IF_ERROR(delete_handler.init(_base_tablet_schema, delete_predicates, base_max_version));
+
+ std::vector<ColumnId> return_columns;
+ return_columns.resize(_base_tablet_schema->num_columns());
+ std::iota(return_columns.begin(), return_columns.end(), 0);
+
+ // reader_context is stack variables, it's lifetime MUST keep the same with rs_readers
+ RowsetReaderContext reader_context;
+ reader_context.reader_type = ReaderType::READER_ALTER_TABLE;
+ reader_context.tablet_schema = _base_tablet_schema;
+ reader_context.need_ordered_result = true;
+ reader_context.delete_handler = &delete_handler;
+ reader_context.return_columns = &return_columns;
+ reader_context.sequence_id_idx = reader_context.tablet_schema->sequence_col_idx();
+ reader_context.is_unique = _base_tablet->keys_type() == UNIQUE_KEYS;
+ reader_context.batch_size = ALTER_TABLE_BATCH_SIZE;
+ reader_context.delete_bitmap = &_base_tablet->tablet_meta()->delete_bitmap();
+ reader_context.version = Version(0, base_max_version);
+
+ for (auto& split : rs_splits) {
+ RETURN_IF_ERROR(split.rs_reader->init(&reader_context));
+ }
+
+ SchemaChangeParams sc_params;
+
+ RETURN_IF_ERROR(DescriptorTbl::create(&sc_params.pool, request.desc_tbl, &sc_params.desc_tbl));
+ sc_params.ref_rowset_readers.reserve(rs_splits.size());
+ for (RowSetSplits& split : rs_splits) {
+ sc_params.ref_rowset_readers.emplace_back(std::move(split.rs_reader));
+ }
+ sc_params.delete_handler = &delete_handler;
+ sc_params.be_exec_version = request.be_exec_version;
+ DCHECK(request.__isset.alter_tablet_type);
+ switch (request.alter_tablet_type) {
+ case TAlterTabletType::SCHEMA_CHANGE:
+ sc_params.alter_tablet_type = AlterTabletType::SCHEMA_CHANGE;
+ break;
+ case TAlterTabletType::ROLLUP:
+ sc_params.alter_tablet_type = AlterTabletType::ROLLUP;
+ break;
+ case TAlterTabletType::MIGRATION:
+ sc_params.alter_tablet_type = AlterTabletType::MIGRATION;
+ break;
+ }
+ if (!request.__isset.materialized_view_params) {
+ return _convert_historical_rowsets(sc_params);
+ }
+ for (auto item : request.materialized_view_params) {
+ AlterMaterializedViewParam mv_param;
+ mv_param.column_name = item.column_name;
+ /*
+ * origin_column_name is always be set now,
+ * but origin_column_name may be not set in some materialized view function. eg:count(1)
+ */
+ if (item.__isset.origin_column_name) {
+ mv_param.origin_column_name = item.origin_column_name;
+ }
+
+ if (item.__isset.mv_expr) {
+ mv_param.expr = std::make_shared<TExpr>(item.mv_expr);
+ }
+ sc_params.materialized_params_map.insert(
+ std::make_pair(to_lower(item.column_name), mv_param));
+ }
+ sc_params.enable_unique_key_merge_on_write = _new_tablet->enable_unique_key_merge_on_write();
+ return _convert_historical_rowsets(sc_params);
+}
+
+Status CloudSchemaChangeJob::_convert_historical_rowsets(const SchemaChangeParams& sc_params) {
+ LOG(INFO) << "Begin to convert historical rowsets for new_tablet from base_tablet. base_tablet="
+ << _base_tablet->tablet_id() << ", new_tablet=" << _new_tablet->tablet_id()
+ << ", job_id=" << _job_id;
+
+ // Add filter information in change, and filter column information will be set in _parse_request
+ // And filter some data every time the row block changes
+ BlockChanger changer(_new_tablet->tablet_schema(), *sc_params.desc_tbl);
+
+ bool sc_sorting = false;
+ bool sc_directly = false;
+
+ // 1. Parse the Alter request and convert it into an internal representation
+ RETURN_IF_ERROR(SchemaChangeJob::parse_request(sc_params, _base_tablet_schema.get(),
+ _new_tablet_schema.get(), &changer, &sc_sorting,
+ &sc_directly));
+ if (!sc_sorting && !sc_directly && sc_params.alter_tablet_type == AlterTabletType::ROLLUP) {
+ LOG(INFO) << "Don't support to add materialized view by linked schema change";
+ return Status::InternalError(
+ "Don't support to add materialized view by linked schema change");
+ }
+
+ // 2. Generate historical data converter
+ auto sc_procedure = get_sc_procedure(changer, sc_sorting);
+
+ cloud::TabletJobInfoPB job;
+ auto* idx = job.mutable_idx();
+ idx->set_tablet_id(_base_tablet->tablet_id());
+ idx->set_table_id(_base_tablet->table_id());
+ idx->set_index_id(_base_tablet->index_id());
+ idx->set_partition_id(_base_tablet->partition_id());
+ auto* sc_job = job.mutable_schema_change();
+ sc_job->set_id(_job_id);
+ sc_job->set_initiator(BackendOptions::get_localhost() + ':' +
+ std::to_string(config::heartbeat_service_port));
+ auto* new_tablet_idx = sc_job->mutable_new_tablet_idx();
+ new_tablet_idx->set_tablet_id(_new_tablet->tablet_id());
+ new_tablet_idx->set_table_id(_new_tablet->table_id());
+ new_tablet_idx->set_index_id(_new_tablet->index_id());
+ new_tablet_idx->set_partition_id(_new_tablet->partition_id());
+ cloud::StartTabletJobResponse start_resp;
+ auto st = _cloud_storage_engine.meta_mgr().prepare_tablet_job(job, &start_resp);
+ if (!st.ok()) {
+ if (start_resp.status().code() == cloud::JOB_ALREADY_SUCCESS) {
+ st = _new_tablet->sync_rowsets();
+ if (!st.ok()) {
+ LOG_WARNING("failed to sync new tablet")
+ .tag("tablet_id", _new_tablet->tablet_id())
+ .error(st);
+ }
+ return Status::OK();
+ }
+ return st;
+ }
+
+ // 3. Convert historical data
+ for (const auto& rs_reader : sc_params.ref_rowset_readers) {
+ VLOG_TRACE << "Begin to convert a history rowset. version=" << rs_reader->version();
+
+ RowsetWriterContext context;
+ context.txn_id = rs_reader->rowset()->txn_id();
+ context.txn_expiration = _expiration;
+ context.version = rs_reader->version();
+ context.rowset_state = VISIBLE;
+ context.segments_overlap = rs_reader->rowset()->rowset_meta()->segments_overlap();
+ context.tablet_schema = _new_tablet->tablet_schema();
+ context.newest_write_timestamp = rs_reader->newest_write_timestamp();
+ context.fs = _cloud_storage_engine.latest_fs();
+ auto rowset_writer = DORIS_TRY(_new_tablet->create_rowset_writer(context, false));
+
+ RowsetMetaSharedPtr existed_rs_meta;
+ auto st = _cloud_storage_engine.meta_mgr().prepare_rowset(*rowset_writer->rowset_meta(),
+ true, &existed_rs_meta);
+ if (!st.ok()) {
+ if (st.is<ALREADY_EXIST>()) {
+ LOG(INFO) << "Rowset " << rs_reader->version() << " has already existed in tablet "
+ << _new_tablet->tablet_id();
+ // Add already committed rowset to _output_rowsets.
+ DCHECK(existed_rs_meta != nullptr);
+ RowsetSharedPtr rowset;
+ // schema is nullptr implies using RowsetMeta.tablet_schema
+ RETURN_IF_ERROR(RowsetFactory::create_rowset(nullptr, _new_tablet->tablet_path(),
+ existed_rs_meta, &rowset));
+ _output_rowsets.push_back(std::move(rowset));
+ continue;
+ } else {
+ return st;
+ }
+ }
+
+ RETURN_IF_ERROR(sc_procedure->process(rs_reader, rowset_writer.get(), _new_tablet,
+ _base_tablet, _base_tablet_schema,
+ _new_tablet_schema));
+
+ RowsetSharedPtr new_rowset;
+ st = rowset_writer->build(new_rowset);
+ if (!st.ok()) {
+ return Status::InternalError("failed to build rowset, version=[{}-{}] status={}",
+ rs_reader->version().first, rs_reader->version().second,
+ st.to_string());
+ }
+
+ st = _cloud_storage_engine.meta_mgr().commit_rowset(*rowset_writer->rowset_meta(), true,
+ &existed_rs_meta);
+ if (!st.ok()) {
+ if (st.is<ALREADY_EXIST>()) {
+ LOG(INFO) << "Rowset " << rs_reader->version() << " has already existed in tablet "
+ << _new_tablet->tablet_id();
+ // Add already committed rowset to _output_rowsets.
+ DCHECK(existed_rs_meta != nullptr);
+ RowsetSharedPtr rowset;
+ // schema is nullptr implies using RowsetMeta.tablet_schema
+ RETURN_IF_ERROR(RowsetFactory::create_rowset(nullptr, _new_tablet->tablet_path(),
+ existed_rs_meta, &rowset));
+ _output_rowsets.push_back(std::move(rowset));
+ continue;
+ } else {
+ return st;
+ }
+ }
+ _output_rowsets.push_back(std::move(new_rowset));
+
+ VLOG_TRACE << "Successfully convert a history version " << rs_reader->version();
+ }
+
+ if (sc_params.ref_rowset_readers.empty()) {
+ sc_job->set_alter_version(1); // no rowset to convert implies alter_version == 1
+ } else {
+ int64_t num_output_rows = 0;
+ int64_t size_output_rowsets = 0;
+ int64_t num_output_segments = 0;
+ for (auto& rs : _output_rowsets) {
+ sc_job->add_txn_ids(rs->txn_id());
+ sc_job->add_output_versions(rs->end_version());
+ num_output_rows += rs->num_rows();
+ size_output_rowsets += rs->data_disk_size();
+ num_output_segments += rs->num_segments();
+ }
+ sc_job->set_num_output_rows(num_output_rows);
+ sc_job->set_size_output_rowsets(size_output_rowsets);
+ sc_job->set_num_output_segments(num_output_segments);
+ sc_job->set_num_output_rowsets(_output_rowsets.size());
+ sc_job->set_alter_version(_output_rowsets.back()->end_version());
+ }
+ _output_cumulative_point = std::min(_output_cumulative_point, sc_job->alter_version() + 1);
+ sc_job->set_output_cumulative_point(_output_cumulative_point);
+
+ // TODO(Lchangliang): process delete bitmap if the table is MOW
+
+ cloud::FinishTabletJobResponse finish_resp;
+ st = _cloud_storage_engine.meta_mgr().commit_tablet_job(job, &finish_resp);
+ if (!st.ok()) {
+ if (finish_resp.status().code() == cloud::JOB_ALREADY_SUCCESS) {
+ st = _new_tablet->sync_rowsets();
+ if (!st.ok()) {
+ LOG_WARNING("failed to sync new tablet")
+ .tag("tablet_id", _new_tablet->tablet_id())
+ .error(st);
+ }
+ return Status::OK();
+ }
+ return st;
+ }
+ const auto& stats = finish_resp.stats();
+ {
+ std::unique_lock wlock(_new_tablet->get_header_lock());
+ // new_tablet's state MUST be `TABLET_NOTREADY`, because we won't sync a new tablet in schema change job
+ DCHECK(_new_tablet->tablet_state() == TABLET_NOTREADY);
+ if (_new_tablet->tablet_state() != TABLET_NOTREADY) [[unlikely]] {
+ LOG(ERROR) << "invalid tablet state, tablet_id=" << _new_tablet->tablet_id();
+ return Status::InternalError("invalid tablet state, tablet_id={}",
+ _new_tablet->tablet_id());
+ }
+ _new_tablet->add_rowsets(std::move(_output_rowsets), true, wlock);
+ _new_tablet->set_cumulative_layer_point(_output_cumulative_point);
+ _new_tablet->reset_approximate_stats(stats.num_rowsets(), stats.num_segments(),
+ stats.num_rows(), stats.data_size());
+ RETURN_IF_ERROR(_new_tablet->set_tablet_state(TABLET_RUNNING));
+ }
+ return Status::OK();
+}
+} // namespace doris
diff --git a/be/src/cloud/cloud_schema_change_job.h b/be/src/cloud/cloud_schema_change_job.h
new file mode 100644
index 00000000000..7bb03fda12a
--- /dev/null
+++ b/be/src/cloud/cloud_schema_change_job.h
@@ -0,0 +1,55 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <memory>
+
+#include "cloud/cloud_storage_engine.h"
+#include "cloud/cloud_tablet.h"
+#include "olap/rowset/rowset.h"
+#include "olap/schema_change.h"
+#include "olap/tablet_fwd.h"
+
+namespace doris {
+
+class CloudSchemaChangeJob {
+public:
+ CloudSchemaChangeJob(CloudStorageEngine& cloud_storage_engine, std::string job_id,
+ int64_t expiration);
+ ~CloudSchemaChangeJob();
+
+ // This method is idempotent for a same request.
+ Status process_alter_tablet(const TAlterTabletReqV2& request);
+
+private:
+ Status _convert_historical_rowsets(const SchemaChangeParams& sc_params);
+
+private:
+ CloudStorageEngine& _cloud_storage_engine;
+ std::shared_ptr<CloudTablet> _base_tablet;
+ std::shared_ptr<CloudTablet> _new_tablet;
+ TabletSchemaSPtr _base_tablet_schema;
+ TabletSchemaSPtr _new_tablet_schema;
+ std::string _job_id;
+ std::vector<RowsetSharedPtr> _output_rowsets;
+ int64_t _output_cumulative_point = 0;
+ // absolute expiration time in second
+ int64_t _expiration;
+};
+
+} // namespace doris
diff --git a/be/src/cloud/pb_convert.cpp b/be/src/cloud/pb_convert.cpp
index f52ce9c2218..402b78449c2 100644
--- a/be/src/cloud/pb_convert.cpp
+++ b/be/src/cloud/pb_convert.cpp
@@ -56,7 +56,9 @@ void doris_rowset_meta_to_cloud(RowsetMetaCloudPB* out, const RowsetMetaPB& in)
out->mutable_delete_predicate()->CopyFrom(in.delete_predicate());
}
out->set_empty(in.empty());
- out->mutable_load_id()->CopyFrom(in.load_id());
+ if (in.has_load_id()) {
+ out->mutable_load_id()->CopyFrom(in.load_id());
+ }
out->set_delete_flag(in.delete_flag());
out->set_creation_time(in.creation_time());
if (in.has_tablet_uid()) {
@@ -102,7 +104,9 @@ void doris_rowset_meta_to_cloud(RowsetMetaCloudPB* out, RowsetMetaPB&& in) {
out->mutable_delete_predicate()->Swap(in.mutable_delete_predicate());
}
out->set_empty(in.empty());
- out->mutable_load_id()->CopyFrom(in.load_id());
+ if (in.has_load_id()) {
+ out->mutable_load_id()->CopyFrom(in.load_id());
+ }
out->set_delete_flag(in.delete_flag());
out->set_creation_time(in.creation_time());
if (in.has_tablet_uid()) {
diff --git a/be/src/http/action/cloud_compaction_action.cpp b/be/src/http/action/cloud_compaction_action.cpp
new file mode 100644
index 00000000000..f8ed8e089a2
--- /dev/null
+++ b/be/src/http/action/cloud_compaction_action.cpp
@@ -0,0 +1,41 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "http/action/cloud_compaction_action.h"
+
+#include "http/http_channel.h"
+#include "http/http_headers.h"
+#include "http/http_request.h"
+#include "http/http_status.h"
+
+namespace doris {
+using namespace ErrorCode;
+
+const static std::string HEADER_JSON = "application/json";
+
+CloudCompactionAction::CloudCompactionAction(CompactionActionType ctype, ExecEnv* exec_env,
+ CloudStorageEngine& engine, TPrivilegeHier::type hier,
+ TPrivilegeType::type ptype)
+ : HttpHandlerWithAuth(exec_env, hier, ptype), _engine(engine), _type(ctype) {}
+
+void CloudCompactionAction::handle(HttpRequest* req) {
+ std::string json_str = R"({"status" : "Success"})";
+ req->add_output_header(HttpHeaders::CONTENT_TYPE, HEADER_JSON.c_str());
+ HttpChannel::send_reply(req, HttpStatus::OK, json_str);
+}
+
+} // end namespace doris
diff --git a/be/src/olap/task/engine_alter_tablet_task.h b/be/src/http/action/cloud_compaction_action.h
similarity index 55%
rename from be/src/olap/task/engine_alter_tablet_task.h
rename to be/src/http/action/cloud_compaction_action.h
index 70f4a576daa..b9b9bed472b 100644
--- a/be/src/olap/task/engine_alter_tablet_task.h
+++ b/be/src/http/action/cloud_compaction_action.h
@@ -17,29 +17,28 @@
#pragma once
-#include <memory>
-
+#include "cloud/cloud_storage_engine.h"
#include "common/status.h"
-#include "olap/task/engine_task.h"
+#include "http/action/compaction_action.h"
+#include "http/http_handler_with_auth.h"
namespace doris {
-class MemTrackerLimiter;
-class TAlterInvertedIndexReq;
-class TAlterTabletReqV2;
+class HttpRequest;
-// base class for storage engine
-// add "Engine" as task prefix to prevent duplicate name with agent task
-class EngineAlterTabletTask final : public EngineTask {
+/// This action is used for viewing the compaction status.
+/// See compaction-action.md for details.
+class CloudCompactionAction : public HttpHandlerWithAuth {
public:
- Status execute() override;
+ CloudCompactionAction(CompactionActionType ctype, ExecEnv* exec_env, CloudStorageEngine& engine,
+ TPrivilegeHier::type hier, TPrivilegeType::type ptype);
- EngineAlterTabletTask(const TAlterTabletReqV2& alter_tablet_request);
- ~EngineAlterTabletTask() override = default;
+ ~CloudCompactionAction() override = default;
-private:
- const TAlterTabletReqV2& _alter_tablet_req;
+ void handle(HttpRequest* req) override;
- std::shared_ptr<MemTrackerLimiter> _mem_tracker;
-}; // EngineTask
+private:
+ [[maybe_unused]] CloudStorageEngine& _engine;
+ [[maybe_unused]] CompactionActionType _type;
+};
-} // namespace doris
+} // namespace doris
\ No newline at end of file
diff --git a/be/src/io/cache/block/block_file_cache_factory.h b/be/src/io/cache/block/block_file_cache_factory.h
index 0b6b6504c96..620e1b988aa 100644
--- a/be/src/io/cache/block/block_file_cache_factory.h
+++ b/be/src/io/cache/block/block_file_cache_factory.h
@@ -46,6 +46,11 @@ public:
size_t try_release(const std::string& base_path);
+ const std::string& get_cache_path() {
+ size_t cur_index = _next_index.fetch_add(1);
+ return _caches[cur_index % _caches.size()]->get_base_path();
+ }
+
CloudFileCachePtr get_by_path(const IFileCache::Key& key);
CloudFileCachePtr get_by_path(const std::string& cache_base_path);
std::vector<IFileCache::QueryFileCacheContextHolderPtr> get_query_context_holders(
@@ -59,6 +64,7 @@ private:
std::mutex _cache_mutex;
std::vector<std::unique_ptr<IFileCache>> _caches;
std::unordered_map<std::string, CloudFileCachePtr> _path_to_cache;
+ std::atomic_size_t _next_index {0}; // use for round-robin
};
} // namespace io
diff --git a/be/src/olap/base_tablet.h b/be/src/olap/base_tablet.h
index 5152f0a541d..943c257de64 100644
--- a/be/src/olap/base_tablet.h
+++ b/be/src/olap/base_tablet.h
@@ -58,6 +58,7 @@ public:
int32_t schema_hash() const { return _tablet_meta->schema_hash(); }
KeysType keys_type() const { return _tablet_meta->tablet_schema()->keys_type(); }
size_t num_key_columns() const { return _tablet_meta->tablet_schema()->num_key_columns(); }
+ std::mutex& get_schema_change_lock() { return _schema_change_lock; }
bool enable_unique_key_merge_on_write() const {
#ifdef BE_TEST
if (_tablet_meta == nullptr) {
@@ -238,6 +239,9 @@ protected:
// metrics of this tablet
std::shared_ptr<MetricEntity> _metric_entity;
+protected:
+ std::mutex _schema_change_lock;
+
public:
IntCounter* query_scan_bytes = nullptr;
IntCounter* query_scan_rows = nullptr;
diff --git a/be/src/olap/merger.cpp b/be/src/olap/merger.cpp
index 33dd99ca7b6..f3163720978 100644
--- a/be/src/olap/merger.cpp
+++ b/be/src/olap/merger.cpp
@@ -33,6 +33,7 @@
#include "common/config.h"
#include "common/logging.h"
+#include "olap/base_tablet.h"
#include "olap/olap_common.h"
#include "olap/olap_define.h"
#include "olap/rowid_conversion.h"
diff --git a/be/src/olap/schema_change.cpp b/be/src/olap/schema_change.cpp
index d3005c7cb3a..1e5a3584d19 100644
--- a/be/src/olap/schema_change.cpp
+++ b/be/src/olap/schema_change.cpp
@@ -20,19 +20,24 @@
#include <algorithm>
#include <exception>
#include <map>
+#include <memory>
#include <mutex>
#include <roaring/roaring.hh>
#include <tuple>
#include <utility>
+#include "cloud/cloud_schema_change_job.h"
+#include "cloud/config.h"
#include "common/logging.h"
#include "common/signal_handler.h"
#include "common/status.h"
+#include "exec/schema_scanner/schema_metadata_name_ids_scanner.h"
#include "gutil/hash/hash.h"
#include "gutil/integral_types.h"
#include "gutil/strings/numbers.h"
#include "io/fs/file_system.h"
#include "io/io_common.h"
+#include "olap/base_tablet.h"
#include "olap/data_dir.h"
#include "olap/delete_handler.h"
#include "olap/field.h"
@@ -60,6 +65,7 @@
#include "olap/types.h"
#include "olap/utils.h"
#include "olap/wrapper_field.h"
+#include "runtime/exec_env.h"
#include "runtime/memory/mem_tracker.h"
#include "runtime/runtime_state.h"
#include "util/debug_points.h"
@@ -85,7 +91,7 @@ constexpr int ALTER_TABLE_BATCH_SIZE = 4064;
class MultiBlockMerger {
public:
- MultiBlockMerger(TabletSharedPtr tablet) : _tablet(tablet), _cmp(tablet) {}
+ MultiBlockMerger(BaseTabletSPtr tablet) : _tablet(tablet), _cmp(*tablet) {}
Status merge(const std::vector<std::unique_ptr<vectorized::Block>>& blocks,
RowsetWriter* rowset_writer, uint64_t* merged_rows) {
@@ -224,7 +230,7 @@ private:
};
struct RowRefComparator {
- RowRefComparator(TabletSharedPtr tablet) : _num_columns(tablet->num_key_columns()) {}
+ RowRefComparator(const BaseTablet& tablet) : _num_columns(tablet.num_key_columns()) {}
int compare(const RowRef& lhs, const RowRef& rhs) const {
return lhs.block->compare_at(lhs.position, rhs.position, _num_columns, *rhs.block, -1);
@@ -237,7 +243,7 @@ private:
const size_t _num_columns;
};
- TabletSharedPtr _tablet;
+ BaseTabletSPtr _tablet;
RowRefComparator _cmp;
};
@@ -363,8 +369,7 @@ Status BlockChanger::change_block(vectorized::Block* ref_block,
// This check is to prevent schema-change from causing data loss
Status BlockChanger::_check_cast_valid(vectorized::ColumnPtr ref_column,
- vectorized::ColumnPtr new_column,
- AlterTabletType type) const {
+ vectorized::ColumnPtr new_column, AlterTabletType type) {
if (ref_column->size() != new_column->size()) {
return Status::InternalError(
"column size is changed, ref_column_size={}, new_column_size={}",
@@ -429,51 +434,36 @@ Status BlockChanger::_check_cast_valid(vectorized::ColumnPtr ref_column,
}
Status LinkedSchemaChange::process(RowsetReaderSharedPtr rowset_reader, RowsetWriter* rowset_writer,
- TabletSharedPtr new_tablet, TabletSharedPtr base_tablet,
+ BaseTabletSPtr new_tablet, BaseTabletSPtr base_tablet,
TabletSchemaSPtr base_tablet_schema,
TabletSchemaSPtr new_tablet_schema) {
- // In some cases, there may be more than one type of rowset in a tablet,
- // in which case the conversion cannot be done directly by linked schema change,
- // but requires direct schema change to rewrite the data.
- if (rowset_reader->type() != rowset_writer->type()) {
- LOG(INFO) << "the type of rowset " << rowset_reader->rowset()->rowset_id()
- << " in base tablet is not same as type " << rowset_writer->type()
- << ", use direct schema change.";
- return SchemaChangeHandler::get_sc_procedure(_changer, false, true)
- ->process(rowset_reader, rowset_writer, new_tablet, base_tablet, base_tablet_schema,
- new_tablet_schema);
- } else {
- Status status = rowset_writer->add_rowset_for_linked_schema_change(rowset_reader->rowset());
- if (!status) {
- LOG(WARNING) << "fail to convert rowset."
- << ", new_tablet=" << new_tablet->tablet_id()
- << ", version=" << rowset_writer->version().first << "-"
- << rowset_writer->version().second << ", error status " << status;
- return status;
- }
- // copy delete bitmap to new tablet.
- if (new_tablet->keys_type() == UNIQUE_KEYS &&
- new_tablet->enable_unique_key_merge_on_write()) {
- DeleteBitmap origin_delete_bitmap(base_tablet->tablet_id());
- base_tablet->tablet_meta()->delete_bitmap().subset(
- {rowset_reader->rowset()->rowset_id(), 0, 0},
- {rowset_reader->rowset()->rowset_id(), UINT32_MAX, INT64_MAX},
- &origin_delete_bitmap);
- for (auto& iter : origin_delete_bitmap.delete_bitmap) {
- int ret = new_tablet->tablet_meta()->delete_bitmap().set(
- {rowset_writer->rowset_id(), std::get<1>(iter.first),
- std::get<2>(iter.first)},
- iter.second);
- DCHECK(ret == 1);
- }
+ Status status = rowset_writer->add_rowset_for_linked_schema_change(rowset_reader->rowset());
+ if (!status) {
+ LOG(WARNING) << "fail to convert rowset."
+ << ", new_tablet=" << new_tablet->tablet_id()
+ << ", version=" << rowset_writer->version().first << "-"
+ << rowset_writer->version().second << ", error status " << status;
+ return status;
+ }
+ // copy delete bitmap to new tablet.
+ if (new_tablet->keys_type() == UNIQUE_KEYS && new_tablet->enable_unique_key_merge_on_write()) {
+ DeleteBitmap origin_delete_bitmap(base_tablet->tablet_id());
+ base_tablet->tablet_meta()->delete_bitmap().subset(
+ {rowset_reader->rowset()->rowset_id(), 0, 0},
+ {rowset_reader->rowset()->rowset_id(), UINT32_MAX, INT64_MAX},
+ &origin_delete_bitmap);
+ for (auto& iter : origin_delete_bitmap.delete_bitmap) {
+ int ret = new_tablet->tablet_meta()->delete_bitmap().set(
+ {rowset_writer->rowset_id(), std::get<1>(iter.first), std::get<2>(iter.first)},
+ iter.second);
+ DCHECK(ret == 1);
}
- return Status::OK();
}
+ return Status::OK();
}
Status VSchemaChangeDirectly::_inner_process(RowsetReaderSharedPtr rowset_reader,
- RowsetWriter* rowset_writer,
- TabletSharedPtr new_tablet,
+ RowsetWriter* rowset_writer, BaseTabletSPtr new_tablet,
TabletSchemaSPtr base_tablet_schema,
TabletSchemaSPtr new_tablet_schema) {
bool eof = false;
@@ -498,8 +488,8 @@ Status VSchemaChangeDirectly::_inner_process(RowsetReaderSharedPtr rowset_reader
return Status::OK();
}
-VSchemaChangeWithSorting::VSchemaChangeWithSorting(const BlockChanger& changer,
- size_t memory_limitation)
+VBaseSchemaChangeWithSorting::VBaseSchemaChangeWithSorting(const BlockChanger& changer,
+ size_t memory_limitation)
: _changer(changer),
_memory_limitation(memory_limitation),
_temp_delta_versions(Version::mock()) {
@@ -507,42 +497,29 @@ VSchemaChangeWithSorting::VSchemaChangeWithSorting(const BlockChanger& changer,
fmt::format("VSchemaChangeWithSorting:changer={}", std::to_string(int64(&changer))));
}
-Status VSchemaChangeWithSorting::_inner_process(RowsetReaderSharedPtr rowset_reader,
- RowsetWriter* rowset_writer,
- TabletSharedPtr new_tablet,
- TabletSchemaSPtr base_tablet_schema,
- TabletSchemaSPtr new_tablet_schema) {
+Status VBaseSchemaChangeWithSorting::_inner_process(RowsetReaderSharedPtr rowset_reader,
+ RowsetWriter* rowset_writer,
+ BaseTabletSPtr new_tablet,
+ TabletSchemaSPtr base_tablet_schema,
+ TabletSchemaSPtr new_tablet_schema) {
// for internal sorting
std::vector<std::unique_ptr<vectorized::Block>> blocks;
- // for external sorting
- // src_rowsets to store the rowset generated by internal sorting
- std::vector<RowsetSharedPtr> src_rowsets;
- std::vector<PendingRowsetGuard> pending_rs_guards;
-
- Defer defer {[&]() {
- // remove the intermediate rowsets generated by internal sorting
- for (auto& row_set : src_rowsets) {
- ExecEnv::GetInstance()->storage_engine().to_local().add_unused_rowset(row_set);
- }
- }};
-
RowsetSharedPtr rowset = rowset_reader->rowset();
SegmentsOverlapPB segments_overlap = rowset->rowset_meta()->segments_overlap();
int64_t newest_write_timestamp = rowset->newest_write_timestamp();
_temp_delta_versions.first = _temp_delta_versions.second;
-
+ _src_rowsets.clear(); // init _src_rowsets
auto create_rowset = [&]() -> Status {
if (blocks.empty()) {
return Status::OK();
}
- auto [rowset, guard] = DORIS_TRY(_internal_sorting(
+ auto rowset = DORIS_TRY(_internal_sorting(
blocks, Version(_temp_delta_versions.second, _temp_delta_versions.second),
newest_write_timestamp, new_tablet, BETA_ROWSET, segments_overlap,
new_tablet_schema));
- src_rowsets.push_back(std::move(rowset));
- pending_rs_guards.push_back(std::move(guard));
+ _src_rowsets.push_back(std::move(rowset));
for (auto& block : blocks) {
_mem_tracker->release(block->allocated_bytes());
}
@@ -592,19 +569,48 @@ Status VSchemaChangeWithSorting::_inner_process(RowsetReaderSharedPtr rowset_rea
RETURN_IF_ERROR(create_rowset());
- if (src_rowsets.empty()) {
+ if (_src_rowsets.empty()) {
RETURN_IF_ERROR(rowset_writer->flush());
} else {
RETURN_IF_ERROR(
- _external_sorting(src_rowsets, rowset_writer, new_tablet, new_tablet_schema));
+ _external_sorting(_src_rowsets, rowset_writer, new_tablet, new_tablet_schema));
}
return Status::OK();
}
-Result<std::pair<RowsetSharedPtr, PendingRowsetGuard>> VSchemaChangeWithSorting::_internal_sorting(
+Result<RowsetSharedPtr> VBaseSchemaChangeWithSorting::_internal_sorting(
+ const std::vector<std::unique_ptr<vectorized::Block>>& blocks, const Version& version,
+ int64_t newest_write_timestamp, BaseTabletSPtr new_tablet, RowsetTypePB new_rowset_type,
+ SegmentsOverlapPB segments_overlap, TabletSchemaSPtr new_tablet_schema) {
+ uint64_t merged_rows = 0;
+ MultiBlockMerger merger(new_tablet);
+ RowsetWriterContext context;
+ context.version = version;
+ context.rowset_state = VISIBLE;
+ context.segments_overlap = segments_overlap;
+ context.tablet_schema = new_tablet_schema;
+ context.original_tablet_schema = new_tablet_schema;
+ context.newest_write_timestamp = newest_write_timestamp;
+ context.write_type = DataWriteType::TYPE_SCHEMA_CHANGE;
+ std::unique_ptr<RowsetWriter> rowset_writer;
+ // TODO(plat1ko): Use monad op
+ if (auto result = new_tablet->create_rowset_writer(context, false); !result.has_value())
+ [[unlikely]] {
+ return unexpected(std::move(result).error());
+ } else {
+ rowset_writer = std::move(result).value();
+ }
+ RETURN_IF_ERROR_RESULT(merger.merge(blocks, rowset_writer.get(), &merged_rows));
+ _add_merged_rows(merged_rows);
+ RowsetSharedPtr rowset;
+ RETURN_IF_ERROR_RESULT(rowset_writer->build(rowset));
+ return rowset;
+}
+
+Result<RowsetSharedPtr> VLocalSchemaChangeWithSorting::_internal_sorting(
const std::vector<std::unique_ptr<vectorized::Block>>& blocks, const Version& version,
- int64_t newest_write_timestamp, TabletSharedPtr new_tablet, RowsetTypePB new_rowset_type,
+ int64_t newest_write_timestamp, BaseTabletSPtr new_tablet, RowsetTypePB new_rowset_type,
SegmentsOverlapPB segments_overlap, TabletSchemaSPtr new_tablet_schema) {
uint64_t merged_rows = 0;
MultiBlockMerger merger(new_tablet);
@@ -624,19 +630,19 @@ Result<std::pair<RowsetSharedPtr, PendingRowsetGuard>> VSchemaChangeWithSorting:
} else {
rowset_writer = std::move(result).value();
}
- auto guard = ExecEnv::GetInstance()->storage_engine().to_local().pending_local_rowsets().add(
- context.rowset_id);
+ auto guard = _local_storage_engine.pending_local_rowsets().add(context.rowset_id);
+ _pending_rs_guards.push_back(std::move(guard));
RETURN_IF_ERROR_RESULT(merger.merge(blocks, rowset_writer.get(), &merged_rows));
_add_merged_rows(merged_rows);
RowsetSharedPtr rowset;
RETURN_IF_ERROR_RESULT(rowset_writer->build(rowset));
- return std::make_pair(std::move(rowset), std::move(guard));
+ return rowset;
}
-Status VSchemaChangeWithSorting::_external_sorting(vector<RowsetSharedPtr>& src_rowsets,
- RowsetWriter* rowset_writer,
- TabletSharedPtr new_tablet,
- TabletSchemaSPtr new_tablet_schema) {
+Status VBaseSchemaChangeWithSorting::_external_sorting(vector<RowsetSharedPtr>& src_rowsets,
+ RowsetWriter* rowset_writer,
+ BaseTabletSPtr new_tablet,
+ TabletSchemaSPtr new_tablet_schema) {
std::vector<RowsetReaderSharedPtr> rs_readers;
for (auto& rowset : src_rowsets) {
RowsetReaderSharedPtr rs_reader;
@@ -652,97 +658,110 @@ Status VSchemaChangeWithSorting::_external_sorting(vector<RowsetSharedPtr>& src_
return Status::OK();
}
-Status SchemaChangeHandler::process_alter_tablet_v2(const TAlterTabletReqV2& request) {
+Status VLocalSchemaChangeWithSorting::_inner_process(RowsetReaderSharedPtr rowset_reader,
+ RowsetWriter* rowset_writer,
+ BaseTabletSPtr new_tablet,
+ TabletSchemaSPtr base_tablet_schema,
+ TabletSchemaSPtr new_tablet_schema) {
+ Defer defer {[&]() {
+ // remove the intermediate rowsets generated by internal sorting
+ for (auto& row_set : _src_rowsets) {
+ _local_storage_engine.add_unused_rowset(row_set);
+ }
+ }};
+ _pending_rs_guards.clear();
+ return VBaseSchemaChangeWithSorting::_inner_process(rowset_reader, rowset_writer, new_tablet,
+ base_tablet_schema, new_tablet_schema);
+}
+
+Status SchemaChangeJob::process_alter_tablet(const TAlterTabletReqV2& request) {
if (!request.__isset.desc_tbl) {
return Status::Error<INVALID_ARGUMENT>(
"desc_tbl is not set. Maybe the FE version is not equal to the BE "
"version.");
}
+ if (_base_tablet == nullptr) {
+ return Status::Error<TABLE_NOT_FOUND>("fail to find base tablet. base_tablet={}",
+ request.base_tablet_id);
+ }
+ if (_new_tablet == nullptr) {
+ return Status::Error<TABLE_NOT_FOUND>("fail to find new tablet. new_tablet={}",
+ request.new_tablet_id);
+ }
LOG(INFO) << "begin to do request alter tablet: base_tablet_id=" << request.base_tablet_id
<< ", new_tablet_id=" << request.new_tablet_id
<< ", alter_version=" << request.alter_version;
- TabletSharedPtr base_tablet =
- ExecEnv::GetInstance()->storage_engine().to_local().tablet_manager()->get_tablet(
- request.base_tablet_id);
- if (base_tablet == nullptr) {
- return Status::Error<TABLE_NOT_FOUND>("fail to find base tablet. base_tablet={}",
- request.base_tablet_id);
- }
// Lock schema_change_lock util schema change info is stored in tablet header
- std::unique_lock<std::mutex> schema_change_lock(base_tablet->get_schema_change_lock(),
+ std::unique_lock<std::mutex> schema_change_lock(_base_tablet->get_schema_change_lock(),
std::try_to_lock);
if (!schema_change_lock.owns_lock()) {
return Status::Error<TRY_LOCK_FAILED>("failed to obtain schema change lock. base_tablet={}",
request.base_tablet_id);
}
- Status res = _do_process_alter_tablet_v2(request);
+ Status res = _do_process_alter_tablet(request);
LOG(INFO) << "finished alter tablet process, res=" << res;
return res;
}
-std::shared_mutex SchemaChangeHandler::_mutex;
-std::unordered_set<int64_t> SchemaChangeHandler::_tablet_ids_in_converting;
+SchemaChangeJob::SchemaChangeJob(StorageEngine& local_storage_engine,
+ const TAlterTabletReqV2& request)
+ : _local_storage_engine(local_storage_engine) {
+ _base_tablet = _local_storage_engine.tablet_manager()->get_tablet(request.base_tablet_id);
+ _new_tablet = _local_storage_engine.tablet_manager()->get_tablet(request.new_tablet_id);
+ if (_base_tablet && _new_tablet) {
+ _base_tablet_schema = std::make_shared<TabletSchema>();
+ _base_tablet_schema->update_tablet_columns(*_base_tablet->tablet_schema(), request.columns);
+ // During a schema change, the extracted columns of a variant should not be included in the tablet schema.
+ // This is because the schema change for a variant needs to ignore the extracted columns.
+ // Otherwise, the schema types in different rowsets might be inconsistent. When performing a schema change,
+ // the complete variant is constructed by reading all the sub-columns of the variant.
+ _new_tablet_schema = _new_tablet->tablet_schema()->copy_without_extracted_columns();
+ }
+}
// In the past schema change and rollup will create new tablet and will wait for txns starting before the task to finished
// It will cost a lot of time to wait and the task is very difficult to understand.
// In alter task v2, FE will call BE to create tablet and send an alter task to BE to convert historical data.
// The admin should upgrade all BE and then upgrade FE.
// Should delete the old code after upgrade finished.
-Status SchemaChangeHandler::_do_process_alter_tablet_v2(const TAlterTabletReqV2& request) {
- Status res = Status::OK();
- TabletSharedPtr base_tablet =
- ExecEnv::GetInstance()->storage_engine().to_local().tablet_manager()->get_tablet(
- request.base_tablet_id);
- if (base_tablet == nullptr) {
- return Status::Error<TABLE_NOT_FOUND>("fail to find base tablet. base_tablet={}",
- request.base_tablet_id);
- }
-
- signal::tablet_id = base_tablet->get_table_id();
-
- // new tablet has to exist
- TabletSharedPtr new_tablet =
- ExecEnv::GetInstance()->storage_engine().to_local().tablet_manager()->get_tablet(
- request.new_tablet_id);
- if (new_tablet == nullptr) {
- return Status::Error<TABLE_NOT_FOUND>("fail to find new tablet. new_tablet={}",
- request.new_tablet_id);
- }
+Status SchemaChangeJob::_do_process_alter_tablet(const TAlterTabletReqV2& request) {
+ Status res;
+ signal::tablet_id = _base_tablet->get_table_id();
// check if tablet's state is not_ready, if it is ready, it means the tablet already finished
// check whether the tablet's max continuous version == request.version
- if (new_tablet->tablet_state() != TABLET_NOTREADY) {
- res = _validate_alter_result(new_tablet, request);
- LOG(INFO) << "tablet's state=" << new_tablet->tablet_state()
+ if (_new_tablet->tablet_state() != TABLET_NOTREADY) {
+ res = _validate_alter_result(request);
+ LOG(INFO) << "tablet's state=" << _new_tablet->tablet_state()
<< " the convert job already finished, check its version"
<< " res=" << res;
return res;
}
- new_tablet->set_alter_failed(false);
- Defer defer([&new_tablet] {
+ _new_tablet->set_alter_failed(false);
+ Defer defer([this] {
// if tablet state is not TABLET_RUNNING when return, indicates that alter has failed.
- if (new_tablet->tablet_state() != TABLET_RUNNING) {
- new_tablet->set_alter_failed(true);
+ if (_new_tablet->tablet_state() != TABLET_RUNNING) {
+ _new_tablet->set_alter_failed(true);
}
});
LOG(INFO) << "finish to validate alter tablet request. begin to convert data from base tablet "
"to new tablet"
- << " base_tablet=" << base_tablet->tablet_id()
- << " new_tablet=" << new_tablet->tablet_id();
+ << " base_tablet=" << _base_tablet->tablet_id()
+ << " new_tablet=" << _new_tablet->tablet_id();
- std::shared_lock base_migration_rlock(base_tablet->get_migration_lock(), std::try_to_lock);
+ std::shared_lock base_migration_rlock(_base_tablet->get_migration_lock(), std::try_to_lock);
if (!base_migration_rlock.owns_lock()) {
return Status::Error<TRY_LOCK_FAILED>(
- "SchemaChangeHandler::_do_process_alter_tablet_v2 get lock failed");
+ "SchemaChangeJob::_do_process_alter_tablet get lock failed");
}
- std::shared_lock new_migration_rlock(new_tablet->get_migration_lock(), std::try_to_lock);
+ std::shared_lock new_migration_rlock(_new_tablet->get_migration_lock(), std::try_to_lock);
if (!new_migration_rlock.owns_lock()) {
return Status::Error<TRY_LOCK_FAILED>(
- "SchemaChangeHandler::_do_process_alter_tablet_v2 get lock failed");
+ "SchemaChangeJob::_do_process_alter_tablet get lock failed");
}
std::vector<Version> versions_to_be_changed;
@@ -754,21 +773,13 @@ Status SchemaChangeHandler::_do_process_alter_tablet_v2(const TAlterTabletReqV2&
// delete handlers for new tablet
DeleteHandler delete_handler;
std::vector<ColumnId> return_columns;
- // Create a new tablet schema, should merge with dropped columns in light weight schema change
- TabletSchemaSPtr base_tablet_schema = std::make_shared<TabletSchema>();
- base_tablet_schema->copy_from(*base_tablet->tablet_schema());
- if (!request.columns.empty() && request.columns[0].col_unique_id >= 0) {
- base_tablet_schema->clear_columns();
- for (const auto& column : request.columns) {
- base_tablet_schema->append_column(TabletColumn(column));
- }
- }
+
// Use tablet schema directly from base tablet, they are the newest schema, not contain
// dropped column during light weight schema change.
// But the tablet schema in base tablet maybe not the latest from FE, so that if fe pass through
// a tablet schema, then use request schema.
- size_t num_cols = request.columns.empty() ? base_tablet->tablet_schema()->num_columns()
- : request.columns.size();
+ size_t num_cols =
+ request.columns.empty() ? _base_tablet_schema->num_columns() : request.columns.size();
return_columns.resize(num_cols);
for (int i = 0; i < num_cols; ++i) {
return_columns[i] = i;
@@ -777,16 +788,16 @@ Status SchemaChangeHandler::_do_process_alter_tablet_v2(const TAlterTabletReqV2&
// begin to find deltas to convert from base tablet to new tablet so that
// obtain base tablet and new tablet's push lock and header write lock to prevent loading data
{
- std::lock_guard<std::mutex> base_tablet_lock(base_tablet->get_push_lock());
- std::lock_guard<std::mutex> new_tablet_lock(new_tablet->get_push_lock());
- std::lock_guard<std::shared_mutex> base_tablet_wlock(base_tablet->get_header_lock());
+ std::lock_guard base_tablet_lock(_base_tablet->get_push_lock());
+ std::lock_guard new_tablet_lock(_new_tablet->get_push_lock());
+ std::lock_guard base_tablet_wlock(_base_tablet->get_header_lock());
SCOPED_SIMPLE_TRACE_IF_TIMEOUT(TRACE_TABLET_LOCK_THRESHOLD);
- std::lock_guard<std::shared_mutex> new_tablet_wlock(new_tablet->get_header_lock());
+ std::lock_guard<std::shared_mutex> new_tablet_wlock(_new_tablet->get_header_lock());
do {
RowsetSharedPtr max_rowset;
// get history data to be converted and it will check if there is hold in base tablet
- if (!_get_versions_to_be_changed(base_tablet, &versions_to_be_changed, &max_rowset)) {
+ if (!_get_versions_to_be_changed(&versions_to_be_changed, &max_rowset)) {
LOG(WARNING) << "fail to get version to be changed. res=" << res;
break;
}
@@ -802,10 +813,10 @@ Status SchemaChangeHandler::_do_process_alter_tablet_v2(const TAlterTabletReqV2&
// before calculating version_to_be_changed,
// remove all data from new tablet, prevent to rewrite data(those double pushed when wait)
LOG(INFO) << "begin to remove all data from new tablet to prevent rewrite."
- << " new_tablet=" << new_tablet->tablet_id();
+ << " new_tablet=" << _new_tablet->tablet_id();
std::vector<RowsetSharedPtr> rowsets_to_delete;
std::vector<std::pair<Version, RowsetSharedPtr>> version_rowsets;
- new_tablet->acquire_version_and_rowsets(&version_rowsets);
+ _new_tablet->acquire_version_and_rowsets(&version_rowsets);
std::sort(version_rowsets.begin(), version_rowsets.end(),
[](const std::pair<Version, RowsetSharedPtr>& l,
const std::pair<Version, RowsetSharedPtr>& r) {
@@ -825,15 +836,15 @@ Status SchemaChangeHandler::_do_process_alter_tablet_v2(const TAlterTabletReqV2&
}
}
std::vector<RowsetSharedPtr> empty_vec;
- RETURN_IF_ERROR(new_tablet->modify_rowsets(empty_vec, rowsets_to_delete));
+ RETURN_IF_ERROR(_new_tablet->modify_rowsets(empty_vec, rowsets_to_delete));
// inherit cumulative_layer_point from base_tablet
// check if new_tablet.ce_point > base_tablet.ce_point?
- new_tablet->set_cumulative_layer_point(-1);
+ _new_tablet->set_cumulative_layer_point(-1);
// save tablet meta
- new_tablet->save_meta();
+ _new_tablet->save_meta();
for (auto& rowset : rowsets_to_delete) {
// do not call rowset.remove directly, using gc thread to delete it
- ExecEnv::GetInstance()->storage_engine().to_local().add_unused_rowset(rowset);
+ _local_storage_engine.add_unused_rowset(rowset);
}
// init one delete handler
@@ -843,7 +854,7 @@ Status SchemaChangeHandler::_do_process_alter_tablet_v2(const TAlterTabletReqV2&
// acquire data sources correspond to history versions
RETURN_IF_ERROR(
- base_tablet->capture_rs_readers_unlocked(versions_to_be_changed, &rs_splits));
+ _base_tablet->capture_rs_readers_unlocked(versions_to_be_changed, &rs_splits));
if (rs_splits.empty()) {
res = Status::Error<ALTER_DELTA_DOES_NOT_EXISTS>(
"fail to acquire all data sources. version_num={}, data_source_num={}",
@@ -856,30 +867,30 @@ Status SchemaChangeHandler::_do_process_alter_tablet_v2(const TAlterTabletReqV2&
if (!rs_meta->has_delete_predicate() || rs_meta->start_version() > end_version) {
continue;
}
- base_tablet_schema->merge_dropped_columns(*rs_meta->tablet_schema());
+ _base_tablet_schema->merge_dropped_columns(*rs_meta->tablet_schema());
del_preds.push_back(rs_meta);
}
- res = delete_handler.init(base_tablet_schema, del_preds, end_version);
+ res = delete_handler.init(_base_tablet_schema, del_preds, end_version);
if (!res) {
LOG(WARNING) << "init delete handler failed. base_tablet="
- << base_tablet->tablet_id() << ", end_version=" << end_version;
+ << _base_tablet->tablet_id() << ", end_version=" << end_version;
break;
}
reader_context.reader_type = ReaderType::READER_ALTER_TABLE;
- reader_context.tablet_schema = base_tablet_schema;
+ reader_context.tablet_schema = _base_tablet_schema;
reader_context.need_ordered_result = true;
reader_context.delete_handler = &delete_handler;
reader_context.return_columns = &return_columns;
reader_context.sequence_id_idx = reader_context.tablet_schema->sequence_col_idx();
- reader_context.is_unique = base_tablet->keys_type() == UNIQUE_KEYS;
+ reader_context.is_unique = _base_tablet->keys_type() == UNIQUE_KEYS;
reader_context.batch_size = ALTER_TABLE_BATCH_SIZE;
- reader_context.delete_bitmap = &base_tablet->tablet_meta()->delete_bitmap();
+ reader_context.delete_bitmap = &_base_tablet->tablet_meta()->delete_bitmap();
reader_context.version = Version(0, end_version);
for (auto& rs_split : rs_splits) {
res = rs_split.rs_reader->init(&reader_context);
if (!res) {
- LOG(WARNING) << "failed to init rowset reader: " << base_tablet->tablet_id();
+ LOG(WARNING) << "failed to init rowset reader: " << _base_tablet->tablet_id();
break;
}
}
@@ -894,19 +905,11 @@ Status SchemaChangeHandler::_do_process_alter_tablet_v2(const TAlterTabletReqV2&
RETURN_IF_ERROR(
DescriptorTbl::create(&sc_params.pool, request.desc_tbl, &sc_params.desc_tbl));
- sc_params.base_tablet = base_tablet;
- sc_params.new_tablet = new_tablet;
- // During a schema change, the extracted columns of a variant should not be included in the tablet schema.
- // This is because the schema change for a variant needs to ignore the extracted columns.
- // Otherwise, the schema types in different rowsets might be inconsistent. When performing a schema change,
- // the complete variant is constructed by reading all the sub-columns of the variant.
- sc_params.new_tablet_schema = new_tablet->tablet_schema()->copy_without_extracted_columns();
sc_params.ref_rowset_readers.reserve(rs_splits.size());
for (RowSetSplits& split : rs_splits) {
sc_params.ref_rowset_readers.emplace_back(split.rs_reader);
}
sc_params.delete_handler = &delete_handler;
- sc_params.base_tablet_schema = base_tablet_schema;
sc_params.be_exec_version = request.be_exec_version;
DCHECK(request.__isset.alter_tablet_type);
switch (request.alter_tablet_type) {
@@ -934,68 +937,69 @@ Status SchemaChangeHandler::_do_process_alter_tablet_v2(const TAlterTabletReqV2&
}
{
std::lock_guard<std::shared_mutex> wrlock(_mutex);
- _tablet_ids_in_converting.insert(new_tablet->tablet_id());
+ _tablet_ids_in_converting.insert(_new_tablet->tablet_id());
}
int64_t real_alter_version = 0;
+ sc_params.enable_unique_key_merge_on_write =
+ _new_tablet->enable_unique_key_merge_on_write();
res = _convert_historical_rowsets(sc_params, &real_alter_version);
{
std::lock_guard<std::shared_mutex> wrlock(_mutex);
- _tablet_ids_in_converting.erase(new_tablet->tablet_id());
+ _tablet_ids_in_converting.erase(_new_tablet->tablet_id());
}
if (!res) {
break;
}
- if (new_tablet->keys_type() == UNIQUE_KEYS &&
- new_tablet->enable_unique_key_merge_on_write()) {
- res = _calc_delete_bitmap_for_mow_table(new_tablet, real_alter_version);
+ if (_new_tablet->keys_type() == UNIQUE_KEYS &&
+ _new_tablet->enable_unique_key_merge_on_write()) {
+ res = _calc_delete_bitmap_for_mow_table(real_alter_version);
if (!res) {
break;
}
} else {
// set state to ready
- std::lock_guard<std::shared_mutex> new_wlock(new_tablet->get_header_lock());
+ std::lock_guard<std::shared_mutex> new_wlock(_new_tablet->get_header_lock());
SCOPED_SIMPLE_TRACE_IF_TIMEOUT(TRACE_TABLET_LOCK_THRESHOLD);
- res = new_tablet->set_tablet_state(TabletState::TABLET_RUNNING);
+ res = _new_tablet->set_tablet_state(TabletState::TABLET_RUNNING);
if (!res) {
break;
}
- new_tablet->save_meta();
+ _new_tablet->save_meta();
}
} while (false);
if (res) {
// _validate_alter_result should be outside the above while loop.
// to avoid requiring the header lock twice.
- res = _validate_alter_result(new_tablet, request);
+ res = _validate_alter_result(request);
}
// if failed convert history data, then just remove the new tablet
if (!res) {
- LOG(WARNING) << "failed to alter tablet. base_tablet=" << base_tablet->tablet_id()
- << ", drop new_tablet=" << new_tablet->tablet_id();
+ LOG(WARNING) << "failed to alter tablet. base_tablet=" << _base_tablet->tablet_id()
+ << ", drop new_tablet=" << _new_tablet->tablet_id();
// do not drop the new tablet and its data. GC thread will
}
return res;
}
-bool SchemaChangeHandler::tablet_in_converting(int64_t tablet_id) {
+bool SchemaChangeJob::tablet_in_converting(int64_t tablet_id) {
std::shared_lock rdlock(_mutex);
return _tablet_ids_in_converting.find(tablet_id) != _tablet_ids_in_converting.end();
}
-Status SchemaChangeHandler::_get_versions_to_be_changed(
- TabletSharedPtr base_tablet, std::vector<Version>* versions_to_be_changed,
- RowsetSharedPtr* max_rowset) {
- RowsetSharedPtr rowset = base_tablet->get_rowset_with_max_version();
+Status SchemaChangeJob::_get_versions_to_be_changed(std::vector<Version>* versions_to_be_changed,
+ RowsetSharedPtr* max_rowset) {
+ RowsetSharedPtr rowset = _base_tablet->get_rowset_with_max_version();
if (rowset == nullptr) {
return Status::Error<ALTER_DELTA_DOES_NOT_EXISTS>("Tablet has no version. base_tablet={}",
- base_tablet->tablet_id());
+ _base_tablet->tablet_id());
}
*max_rowset = rowset;
- RETURN_IF_ERROR(base_tablet->capture_consistent_versions_unlocked(
+ RETURN_IF_ERROR(_base_tablet->capture_consistent_versions_unlocked(
Version(0, rowset->version().second), versions_to_be_changed, false, false));
return Status::OK();
@@ -1003,11 +1007,11 @@ Status SchemaChangeHandler::_get_versions_to_be_changed(
// The `real_alter_version` parameter indicates that the version of [0-real_alter_version] is
// converted from a base tablet, only used for the mow table now.
-Status SchemaChangeHandler::_convert_historical_rowsets(const SchemaChangeParams& sc_params,
- int64_t* real_alter_version) {
+Status SchemaChangeJob::_convert_historical_rowsets(const SchemaChangeParams& sc_params,
+ int64_t* real_alter_version) {
LOG(INFO) << "begin to convert historical rowsets for new_tablet from base_tablet."
- << " base_tablet=" << sc_params.base_tablet->tablet_id()
- << ", new_tablet=" << sc_params.new_tablet->tablet_id();
+ << " base_tablet=" << _base_tablet->tablet_id()
+ << ", new_tablet=" << _new_tablet->tablet_id();
// find end version
int32_t end_version = -1;
@@ -1017,35 +1021,35 @@ Status SchemaChangeHandler::_convert_historical_rowsets(const SchemaChangeParams
}
}
- // Add filter information in change, and filter column information will be set in _parse_request
+ // Add filter information in change, and filter column information will be set in parse_request
// And filter some data every time the row block changes
- BlockChanger changer(sc_params.new_tablet_schema, *sc_params.desc_tbl);
+ BlockChanger changer(_new_tablet_schema, *sc_params.desc_tbl);
bool sc_sorting = false;
bool sc_directly = false;
// a.Parse the Alter request and convert it into an internal representation
- Status res = _parse_request(sc_params, &changer, &sc_sorting, &sc_directly);
+ Status res = parse_request(sc_params, _base_tablet_schema.get(), _new_tablet_schema.get(),
+ &changer, &sc_sorting, &sc_directly);
LOG(INFO) << "schema change type, sc_sorting: " << sc_sorting
- << ", sc_directly: " << sc_directly
- << ", base_tablet=" << sc_params.base_tablet->tablet_id()
- << ", new_tablet=" << sc_params.new_tablet->tablet_id();
+ << ", sc_directly: " << sc_directly << ", base_tablet=" << _base_tablet->tablet_id()
+ << ", new_tablet=" << _new_tablet->tablet_id();
auto process_alter_exit = [&]() -> Status {
{
// save tablet meta here because rowset meta is not saved during add rowset
- std::lock_guard<std::shared_mutex> new_wlock(sc_params.new_tablet->get_header_lock());
+ std::lock_guard new_wlock(_new_tablet->get_header_lock());
SCOPED_SIMPLE_TRACE_IF_TIMEOUT(TRACE_TABLET_LOCK_THRESHOLD);
- sc_params.new_tablet->save_meta();
+ _new_tablet->save_meta();
}
if (res) {
Version test_version(0, end_version);
- res = sc_params.new_tablet->check_version_integrity(test_version);
+ res = _new_tablet->check_version_integrity(test_version);
}
LOG(INFO) << "finish converting rowsets for new_tablet from base_tablet. "
- << "base_tablet=" << sc_params.base_tablet->tablet_id()
- << ", new_tablet=" << sc_params.new_tablet->tablet_id();
+ << "base_tablet=" << _base_tablet->tablet_id()
+ << ", new_tablet=" << _new_tablet->tablet_id();
return res;
};
@@ -1061,41 +1065,35 @@ Status SchemaChangeHandler::_convert_historical_rowsets(const SchemaChangeParams
}
// b. Generate historical data converter
- auto sc_procedure = get_sc_procedure(changer, sc_sorting, sc_directly);
+ auto sc_procedure = _get_sc_procedure(changer, sc_sorting, sc_directly);
// c.Convert historical data
for (const auto& rs_reader : sc_params.ref_rowset_readers) {
- VLOG_TRACE << "begin to convert a history rowset. version=" << rs_reader->version().first
- << "-" << rs_reader->version().second;
-
// set status for monitor
// As long as there is a new_table as running, ref table is set as running
// NOTE If the first sub_table fails first, it will continue to go as normal here
- TabletSharedPtr new_tablet = sc_params.new_tablet;
// When tablet create new rowset writer, it may change rowset type, in this case
// linked schema change will not be used.
RowsetWriterContext context;
context.version = rs_reader->version();
context.rowset_state = VISIBLE;
context.segments_overlap = rs_reader->rowset()->rowset_meta()->segments_overlap();
- context.tablet_schema = sc_params.new_tablet_schema;
- context.original_tablet_schema = sc_params.new_tablet_schema;
+ context.tablet_schema = _new_tablet_schema;
+ context.original_tablet_schema = _new_tablet_schema;
context.newest_write_timestamp = rs_reader->newest_write_timestamp();
context.fs = rs_reader->rowset()->rowset_meta()->fs();
context.write_type = DataWriteType::TYPE_SCHEMA_CHANGE;
- auto result = new_tablet->create_rowset_writer(context, false);
+ auto result = _new_tablet->create_rowset_writer(context, false);
if (!result.has_value()) {
res = Status::Error<ROWSET_BUILDER_INIT>("create_rowset_writer failed, reason={}",
result.error().to_string());
return process_alter_exit();
}
auto rowset_writer = std::move(result).value();
- auto pending_rs_guard =
- ExecEnv::GetInstance()->storage_engine().to_local().add_pending_rowset(context);
+ auto pending_rs_guard = _local_storage_engine.add_pending_rowset(context);
- if (res = sc_procedure->process(rs_reader, rowset_writer.get(), sc_params.new_tablet,
- sc_params.base_tablet, sc_params.base_tablet_schema,
- sc_params.new_tablet_schema);
+ if (res = sc_procedure->process(rs_reader, rowset_writer.get(), _new_tablet, _base_tablet,
+ _base_tablet_schema, _new_tablet_schema);
!res) {
LOG(WARNING) << "failed to process the version."
<< " version=" << rs_reader->version().first << "-"
@@ -1104,28 +1102,28 @@ Status SchemaChangeHandler::_convert_historical_rowsets(const SchemaChangeParams
}
// Add the new version of the data to the header
// In order to prevent the occurrence of deadlock, we must first lock the old table, and then lock the new table
- std::lock_guard<std::mutex> lock(sc_params.new_tablet->get_push_lock());
+ std::lock_guard lock(_new_tablet->get_push_lock());
RowsetSharedPtr new_rowset;
if (!(res = rowset_writer->build(new_rowset)).ok()) {
LOG(WARNING) << "failed to build rowset, exit alter process";
return process_alter_exit();
}
- res = sc_params.new_tablet->add_rowset(new_rowset);
+ res = _new_tablet->add_rowset(new_rowset);
if (res.is<PUSH_VERSION_ALREADY_EXIST>()) {
LOG(WARNING) << "version already exist, version revert occurred. "
- << "tablet=" << sc_params.new_tablet->tablet_id() << ", version='"
+ << "tablet=" << _new_tablet->tablet_id() << ", version='"
<< rs_reader->version().first << "-" << rs_reader->version().second;
- ExecEnv::GetInstance()->storage_engine().to_local().add_unused_rowset(new_rowset);
+ _local_storage_engine.add_unused_rowset(new_rowset);
res = Status::OK();
} else if (!res) {
LOG(WARNING) << "failed to register new version. "
- << " tablet=" << sc_params.new_tablet->tablet_id()
+ << " tablet=" << _new_tablet->tablet_id()
<< ", version=" << rs_reader->version().first << "-"
<< rs_reader->version().second;
- ExecEnv::GetInstance()->storage_engine().to_local().add_unused_rowset(new_rowset);
+ _local_storage_engine.add_unused_rowset(new_rowset);
return process_alter_exit();
} else {
- VLOG_NOTICE << "register new version. tablet=" << sc_params.new_tablet->tablet_id()
+ VLOG_NOTICE << "register new version. tablet=" << _new_tablet->tablet_id()
<< ", version=" << rs_reader->version().first << "-"
<< rs_reader->version().second;
}
@@ -1144,19 +1142,16 @@ static const std::string WHERE_SIGN_LOWER = to_lower("__DORIS_WHERE_SIGN__");
// @static
// Analyze the mapping of the column and the mapping of the filter key
-Status SchemaChangeHandler::_parse_request(const SchemaChangeParams& sc_params,
- BlockChanger* changer, bool* sc_sorting,
- bool* sc_directly) {
+Status SchemaChangeJob::parse_request(const SchemaChangeParams& sc_params,
+ TabletSchema* base_tablet_schema,
+ TabletSchema* new_tablet_schema, BlockChanger* changer,
+ bool* sc_sorting, bool* sc_directly) {
changer->set_type(sc_params.alter_tablet_type);
changer->set_compatible_version(sc_params.be_exec_version);
- TabletSharedPtr base_tablet = sc_params.base_tablet;
- TabletSharedPtr new_tablet = sc_params.new_tablet;
- TabletSchemaSPtr base_tablet_schema = sc_params.base_tablet_schema;
const std::unordered_map<std::string, AlterMaterializedViewParam>& materialized_function_map =
sc_params.materialized_params_map;
DescriptorTbl desc_tbl = *sc_params.desc_tbl;
- TabletSchemaSPtr new_tablet_schema = sc_params.new_tablet_schema;
// set column mapping
for (int i = 0, new_schema_size = new_tablet_schema->num_columns(); i < new_schema_size; ++i) {
@@ -1208,8 +1203,7 @@ Status SchemaChangeHandler::_parse_request(const SchemaChangeParams& sc_params,
LOG(INFO) << "A column with default value will be added after schema changing. "
<< "column=" << new_column.name()
- << ", default_value=" << new_column.default_value() << " to table "
- << new_tablet->get_table_id();
+ << ", default_value=" << new_column.default_value();
}
if (materialized_function_map.contains(WHERE_SIGN_LOWER)) {
@@ -1219,7 +1213,8 @@ Status SchemaChangeHandler::_parse_request(const SchemaChangeParams& sc_params,
// If the reference sequence of the Key column is out of order, it needs to be reordered
int num_default_value = 0;
- for (int i = 0, new_schema_size = new_tablet->num_key_columns(); i < new_schema_size; ++i) {
+ for (int i = 0, new_schema_size = new_tablet_schema->num_key_columns(); i < new_schema_size;
+ ++i) {
ColumnMapping* column_mapping = changer->get_mutable_column_mapping(i);
if (!column_mapping->has_reference()) {
@@ -1248,7 +1243,7 @@ Status SchemaChangeHandler::_parse_request(const SchemaChangeParams& sc_params,
// old keys: A B C D
// new keys: A B
if (new_tablet_schema->keys_type() != KeysType::DUP_KEYS &&
- new_tablet->num_key_columns() < base_tablet_schema->num_key_columns()) {
+ new_tablet_schema->num_key_columns() < base_tablet_schema->num_key_columns()) {
// this is a table with aggregate key type, and num of key columns in new schema
// is less, which means the data in new tablet should be more aggregated.
// so we use sorting schema change to sort and merge the data.
@@ -1261,13 +1256,13 @@ Status SchemaChangeHandler::_parse_request(const SchemaChangeParams& sc_params,
return Status::OK();
}
- if (new_tablet->enable_unique_key_merge_on_write() &&
- new_tablet->num_key_columns() > base_tablet_schema->num_key_columns()) {
+ if (sc_params.enable_unique_key_merge_on_write &&
+ new_tablet_schema->num_key_columns() > base_tablet_schema->num_key_columns()) {
*sc_directly = true;
return Status::OK();
}
- if (base_tablet_schema->num_short_key_columns() != new_tablet->num_short_key_columns()) {
+ if (base_tablet_schema->num_short_key_columns() != new_tablet_schema->num_short_key_columns()) {
// the number of short_keys changed, can't do linked schema change
*sc_directly = true;
return Status::OK();
@@ -1279,7 +1274,7 @@ Status SchemaChangeHandler::_parse_request(const SchemaChangeParams& sc_params,
return Status::OK();
}
- for (size_t i = 0; i < new_tablet->num_columns(); ++i) {
+ for (size_t i = 0; i < new_tablet_schema->num_columns(); ++i) {
ColumnMapping* column_mapping = changer->get_mutable_column_mapping(i);
if (column_mapping->expr != nullptr) {
*sc_directly = true;
@@ -1302,9 +1297,9 @@ Status SchemaChangeHandler::_parse_request(const SchemaChangeParams& sc_params,
return Status::OK();
}
-Status SchemaChangeHandler::_init_column_mapping(ColumnMapping* column_mapping,
- const TabletColumn& column_schema,
- const std::string& value) {
+Status SchemaChangeJob::_init_column_mapping(ColumnMapping* column_mapping,
+ const TabletColumn& column_schema,
+ const std::string& value) {
if (auto field = WrapperField::create(column_schema); field.has_value()) {
column_mapping->default_value = field.value();
} else {
@@ -1321,11 +1316,10 @@ Status SchemaChangeHandler::_init_column_mapping(ColumnMapping* column_mapping,
return Status::OK();
}
-Status SchemaChangeHandler::_validate_alter_result(TabletSharedPtr new_tablet,
- const TAlterTabletReqV2& request) {
+Status SchemaChangeJob::_validate_alter_result(const TAlterTabletReqV2& request) {
Version max_continuous_version = {-1, 0};
- new_tablet->max_continuous_version_from_beginning(&max_continuous_version);
- LOG(INFO) << "find max continuous version of tablet=" << new_tablet->tablet_id()
+ _new_tablet->max_continuous_version_from_beginning(&max_continuous_version);
+ LOG(INFO) << "find max continuous version of tablet=" << _new_tablet->tablet_id()
<< ", start_version=" << max_continuous_version.first
<< ", end_version=" << max_continuous_version.second;
if (max_continuous_version.second < request.alter_version) {
@@ -1335,14 +1329,14 @@ Status SchemaChangeHandler::_validate_alter_result(TabletSharedPtr new_tablet,
std::vector<std::pair<Version, RowsetSharedPtr>> version_rowsets;
{
- std::shared_lock rdlock(new_tablet->get_header_lock());
- new_tablet->acquire_version_and_rowsets(&version_rowsets);
+ std::shared_lock rdlock(_new_tablet->get_header_lock());
+ _new_tablet->acquire_version_and_rowsets(&version_rowsets);
}
for (auto& pair : version_rowsets) {
RowsetSharedPtr rowset = pair.second;
if (!rowset->check_file_exist()) {
return Status::Error<FILE_NOT_EXIST>(
- "SchemaChangeHandler::_validate_alter_result meet invalid rowset");
+ "SchemaChangeJob::_validate_alter_result meet invalid rowset");
}
}
return Status::OK();
@@ -1358,56 +1352,55 @@ Status SchemaChangeHandler::_validate_alter_result(TabletSharedPtr new_tablet,
// incremental rowsets.
// 4. Switch the tablet status to TABLET_RUNNING. The newly imported
// data will calculate delete bitmap.
-Status SchemaChangeHandler::_calc_delete_bitmap_for_mow_table(TabletSharedPtr new_tablet,
- int64_t alter_version) {
- DBUG_EXECUTE_IF("SchemaChangeHandler._calc_delete_bitmap_for_mow_table.random_failed", {
+Status SchemaChangeJob::_calc_delete_bitmap_for_mow_table(int64_t alter_version) {
+ DBUG_EXECUTE_IF("SchemaChangeJob._calc_delete_bitmap_for_mow_table.random_failed", {
if (rand() % 100 < (100 * dp->param("percent", 0.1))) {
- LOG_WARNING("SchemaChangeHandler._calc_delete_bitmap_for_mow_table.random_failed");
+ LOG_WARNING("SchemaChangeJob._calc_delete_bitmap_for_mow_table.random_failed");
return Status::InternalError("debug schema change calc delete bitmap random failed");
}
});
// can't do compaction when calc delete bitmap, if the rowset being calculated does
// a compaction, it may cause the delete bitmap to be missed.
- std::lock_guard base_compaction_lock(new_tablet->get_base_compaction_lock());
- std::lock_guard cumu_compaction_lock(new_tablet->get_cumulative_compaction_lock());
+ std::lock_guard base_compaction_lock(_new_tablet->get_base_compaction_lock());
+ std::lock_guard cumu_compaction_lock(_new_tablet->get_cumulative_compaction_lock());
// step 2
- int64_t max_version = new_tablet->max_version().second;
+ int64_t max_version = _new_tablet->max_version().second;
std::vector<RowsetSharedPtr> rowsets;
if (alter_version < max_version) {
LOG(INFO) << "alter table for unique with merge-on-write, calculate delete bitmap of "
<< "double write rowsets for version: " << alter_version + 1 << "-" << max_version
- << " new_tablet=" << new_tablet->tablet_id();
- std::shared_lock<std::shared_mutex> rlock(new_tablet->get_header_lock());
- RETURN_IF_ERROR(new_tablet->capture_consistent_rowsets_unlocked(
+ << " new_tablet=" << _new_tablet->tablet_id();
+ std::shared_lock rlock(_new_tablet->get_header_lock());
+ RETURN_IF_ERROR(_new_tablet->capture_consistent_rowsets_unlocked(
{alter_version + 1, max_version}, &rowsets));
}
for (auto rowset_ptr : rowsets) {
- std::lock_guard<std::mutex> rwlock(new_tablet->get_rowset_update_lock());
- std::shared_lock<std::shared_mutex> rlock(new_tablet->get_header_lock());
- RETURN_IF_ERROR(Tablet::update_delete_bitmap_without_lock(new_tablet, rowset_ptr));
+ std::lock_guard rwlock(_new_tablet->get_rowset_update_lock());
+ std::shared_lock rlock(_new_tablet->get_header_lock());
+ RETURN_IF_ERROR(Tablet::update_delete_bitmap_without_lock(_new_tablet, rowset_ptr));
}
// step 3
- std::lock_guard<std::mutex> rwlock(new_tablet->get_rowset_update_lock());
- std::lock_guard<std::shared_mutex> new_wlock(new_tablet->get_header_lock());
+ std::lock_guard rwlock(_new_tablet->get_rowset_update_lock());
+ std::lock_guard new_wlock(_new_tablet->get_header_lock());
SCOPED_SIMPLE_TRACE_IF_TIMEOUT(TRACE_TABLET_LOCK_THRESHOLD);
- int64_t new_max_version = new_tablet->max_version_unlocked();
+ int64_t new_max_version = _new_tablet->max_version_unlocked();
rowsets.clear();
if (max_version < new_max_version) {
LOG(INFO) << "alter table for unique with merge-on-write, calculate delete bitmap of "
<< "incremental rowsets for version: " << max_version + 1 << "-"
- << new_max_version << " new_tablet=" << new_tablet->tablet_id();
- RETURN_IF_ERROR(new_tablet->capture_consistent_rowsets_unlocked(
+ << new_max_version << " new_tablet=" << _new_tablet->tablet_id();
+ RETURN_IF_ERROR(_new_tablet->capture_consistent_rowsets_unlocked(
{max_version + 1, new_max_version}, &rowsets));
}
for (auto&& rowset_ptr : rowsets) {
- RETURN_IF_ERROR(Tablet::update_delete_bitmap_without_lock(new_tablet, rowset_ptr));
+ RETURN_IF_ERROR(Tablet::update_delete_bitmap_without_lock(_new_tablet, rowset_ptr));
}
// step 4
- RETURN_IF_ERROR(new_tablet->set_tablet_state(TabletState::TABLET_RUNNING));
- new_tablet->save_meta();
+ RETURN_IF_ERROR(_new_tablet->set_tablet_state(TabletState::TABLET_RUNNING));
+ _new_tablet->save_meta();
return Status::OK();
}
diff --git a/be/src/olap/schema_change.h b/be/src/olap/schema_change.h
index 949c1d5514c..2e1198e181a 100644
--- a/be/src/olap/schema_change.h
+++ b/be/src/olap/schema_change.h
@@ -44,7 +44,9 @@
#include "olap/rowset/rowset_reader.h"
#include "olap/rowset/rowset_writer.h"
#include "olap/rowset/segment_v2/inverted_index_writer.h"
+#include "olap/storage_engine.h"
#include "olap/tablet.h"
+#include "olap/tablet_fwd.h"
#include "olap/tablet_schema.h"
#include "runtime/descriptors.h"
#include "runtime/memory/mem_tracker.h"
@@ -84,8 +86,8 @@ public:
bool has_where() const { return _where_expr != nullptr; }
private:
- Status _check_cast_valid(vectorized::ColumnPtr ref_column, vectorized::ColumnPtr new_column,
- AlterTabletType type) const;
+ static Status _check_cast_valid(vectorized::ColumnPtr ref_column,
+ vectorized::ColumnPtr new_column, AlterTabletType type);
// @brief column-mapping specification of new schema
SchemaMapping _schema_mapping;
@@ -105,7 +107,7 @@ public:
virtual ~SchemaChange() = default;
virtual Status process(RowsetReaderSharedPtr rowset_reader, RowsetWriter* rowset_writer,
- TabletSharedPtr new_tablet, TabletSharedPtr base_tablet,
+ BaseTabletSPtr new_tablet, BaseTabletSPtr base_tablet,
TabletSchemaSPtr base_tablet_schema,
TabletSchemaSPtr new_tablet_schema) {
if (rowset_reader->rowset()->empty() || rowset_reader->rowset()->num_rows() == 0) {
@@ -115,7 +117,6 @@ public:
_filtered_rows = 0;
_merged_rows = 0;
-
RETURN_IF_ERROR(_inner_process(rowset_reader, rowset_writer, new_tablet, base_tablet_schema,
new_tablet_schema));
@@ -140,7 +141,7 @@ protected:
void _add_merged_rows(uint64_t merged_rows) { _merged_rows += merged_rows; }
virtual Status _inner_process(RowsetReaderSharedPtr rowset_reader, RowsetWriter* rowset_writer,
- TabletSharedPtr new_tablet, TabletSchemaSPtr base_tablet_schema,
+ BaseTabletSPtr new_tablet, TabletSchemaSPtr base_tablet_schema,
TabletSchemaSPtr new_tablet_schema) {
return Status::NotSupported("inner process unsupported.");
}
@@ -167,16 +168,15 @@ private:
class LinkedSchemaChange : public SchemaChange {
public:
- explicit LinkedSchemaChange(const BlockChanger& changer) : _changer(changer) {}
+ LinkedSchemaChange() = default;
~LinkedSchemaChange() override = default;
Status process(RowsetReaderSharedPtr rowset_reader, RowsetWriter* rowset_writer,
- TabletSharedPtr new_tablet, TabletSharedPtr base_tablet,
+ BaseTabletSPtr new_tablet, BaseTabletSPtr base_tablet,
TabletSchemaSPtr base_tablet_schema,
TabletSchemaSPtr new_tablet_schema) override;
private:
- const BlockChanger& _changer;
DISALLOW_COPY_AND_ASSIGN(LinkedSchemaChange);
};
@@ -186,7 +186,7 @@ public:
private:
Status _inner_process(RowsetReaderSharedPtr rowset_reader, RowsetWriter* rowset_writer,
- TabletSharedPtr new_tablet, TabletSchemaSPtr base_tablet_schema,
+ BaseTabletSPtr new_tablet, TabletSchemaSPtr base_tablet_schema,
TabletSchemaSPtr new_tablet_schema) override;
bool _check_row_nums(RowsetReaderSharedPtr reader, const RowsetWriter& writer) const override {
@@ -196,26 +196,30 @@ private:
const BlockChanger& _changer;
};
-// @breif schema change with sorting
-class VSchemaChangeWithSorting : public SchemaChange {
+class VBaseSchemaChangeWithSorting : public SchemaChange {
public:
- VSchemaChangeWithSorting(const BlockChanger& changer, size_t memory_limitation);
- ~VSchemaChangeWithSorting() override = default;
+ VBaseSchemaChangeWithSorting(const BlockChanger& changer, size_t memory_limitation);
+ ~VBaseSchemaChangeWithSorting() override = default;
-private:
Status _inner_process(RowsetReaderSharedPtr rowset_reader, RowsetWriter* rowset_writer,
- TabletSharedPtr new_tablet, TabletSchemaSPtr base_tablet_schema,
+ BaseTabletSPtr new_tablet, TabletSchemaSPtr base_tablet_schema,
TabletSchemaSPtr new_tablet_schema) override;
- Result<std::pair<RowsetSharedPtr, PendingRowsetGuard>> _internal_sorting(
+ virtual Result<RowsetSharedPtr> _internal_sorting(
const std::vector<std::unique_ptr<vectorized::Block>>& blocks,
const Version& temp_delta_versions, int64_t newest_write_timestamp,
- TabletSharedPtr new_tablet, RowsetTypePB new_rowset_type,
+ BaseTabletSPtr new_tablet, RowsetTypePB new_rowset_type,
SegmentsOverlapPB segments_overlap, TabletSchemaSPtr new_tablet_schema);
Status _external_sorting(std::vector<RowsetSharedPtr>& src_rowsets, RowsetWriter* rowset_writer,
- TabletSharedPtr new_tablet, TabletSchemaSPtr new_tablet_schema);
+ BaseTabletSPtr new_tablet, TabletSchemaSPtr new_tablet_schema);
+protected:
+ // for external sorting
+ // src_rowsets to store the rowset generated by internal sorting
+ std::vector<RowsetSharedPtr> _src_rowsets;
+
+private:
bool _check_row_nums(RowsetReaderSharedPtr reader, const RowsetWriter& writer) const override {
return _changer.has_where() || SchemaChange::_check_row_nums(reader, writer);
}
@@ -226,72 +230,98 @@ private:
std::unique_ptr<MemTracker> _mem_tracker;
};
-class SchemaChangeHandler {
+// @breif schema change with sorting
+// Mixin for local StorageEngine
+class VLocalSchemaChangeWithSorting final : public VBaseSchemaChangeWithSorting {
public:
- // schema change v2, it will not set alter task in base tablet
- static Status process_alter_tablet_v2(const TAlterTabletReqV2& request);
+ VLocalSchemaChangeWithSorting(const BlockChanger& changer, size_t memory_limitation,
+ StorageEngine& local_storage_engine)
+ : VBaseSchemaChangeWithSorting(changer, memory_limitation),
+ _local_storage_engine(local_storage_engine) {}
+ ~VLocalSchemaChangeWithSorting() override = default;
- static std::unique_ptr<SchemaChange> get_sc_procedure(const BlockChanger& changer,
- bool sc_sorting, bool sc_directly) {
+ Status _inner_process(RowsetReaderSharedPtr rowset_reader, RowsetWriter* rowset_writer,
+ BaseTabletSPtr new_tablet, TabletSchemaSPtr base_tablet_schema,
+ TabletSchemaSPtr new_tablet_schema) override;
+
+ Result<RowsetSharedPtr> _internal_sorting(
+ const std::vector<std::unique_ptr<vectorized::Block>>& blocks,
+ const Version& temp_delta_versions, int64_t newest_write_timestamp,
+ BaseTabletSPtr new_tablet, RowsetTypePB new_rowset_type,
+ SegmentsOverlapPB segments_overlap, TabletSchemaSPtr new_tablet_schema) override;
+
+private:
+ StorageEngine& _local_storage_engine;
+ std::vector<PendingRowsetGuard> _pending_rs_guards;
+};
+
+struct AlterMaterializedViewParam {
+ std::string column_name;
+ std::string origin_column_name;
+ std::shared_ptr<TExpr> expr;
+};
+
+struct SchemaChangeParams {
+ AlterTabletType alter_tablet_type;
+ bool enable_unique_key_merge_on_write;
+ std::vector<RowsetReaderSharedPtr> ref_rowset_readers;
+ DeleteHandler* delete_handler = nullptr;
+ std::unordered_map<std::string, AlterMaterializedViewParam> materialized_params_map;
+ DescriptorTbl* desc_tbl = nullptr;
+ ObjectPool pool;
+ int32_t be_exec_version;
+};
+
+class SchemaChangeJob {
+public:
+ SchemaChangeJob(StorageEngine& local_storage_engine, const TAlterTabletReqV2& request);
+ Status process_alter_tablet(const TAlterTabletReqV2& request);
+
+ bool tablet_in_converting(int64_t tablet_id);
+
+ static Status parse_request(const SchemaChangeParams& sc_params,
+ TabletSchema* base_tablet_schema, TabletSchema* new_tablet_schema,
+ BlockChanger* changer, bool* sc_sorting, bool* sc_directly);
+
+private:
+ std::unique_ptr<SchemaChange> _get_sc_procedure(const BlockChanger& changer, bool sc_sorting,
+ bool sc_directly) {
if (sc_sorting) {
- return std::make_unique<VSchemaChangeWithSorting>(
- changer, config::memory_limitation_per_thread_for_schema_change_bytes);
+ return std::make_unique<VLocalSchemaChangeWithSorting>(
+ changer, config::memory_limitation_per_thread_for_schema_change_bytes,
+ _local_storage_engine);
}
if (sc_directly) {
return std::make_unique<VSchemaChangeDirectly>(changer);
}
- return std::make_unique<LinkedSchemaChange>(changer);
+ return std::make_unique<LinkedSchemaChange>();
}
- static bool tablet_in_converting(int64_t tablet_id);
+ Status _get_versions_to_be_changed(std::vector<Version>* versions_to_be_changed,
+ RowsetSharedPtr* max_rowset);
-private:
- static Status _get_versions_to_be_changed(TabletSharedPtr base_tablet,
- std::vector<Version>* versions_to_be_changed,
- RowsetSharedPtr* max_rowset);
-
- struct AlterMaterializedViewParam {
- std::string column_name;
- std::string origin_column_name;
- std::shared_ptr<TExpr> expr;
- };
-
- struct SchemaChangeParams {
- AlterTabletType alter_tablet_type;
- TabletSharedPtr base_tablet;
- TabletSharedPtr new_tablet;
- TabletSchemaSPtr base_tablet_schema = nullptr;
- TabletSchemaSPtr new_tablet_schema = nullptr;
- std::vector<RowsetReaderSharedPtr> ref_rowset_readers;
- DeleteHandler* delete_handler = nullptr;
- std::unordered_map<std::string, AlterMaterializedViewParam> materialized_params_map;
- DescriptorTbl* desc_tbl = nullptr;
- ObjectPool pool;
- int32_t be_exec_version;
- };
-
- static Status _do_process_alter_tablet_v2(const TAlterTabletReqV2& request);
-
- static Status _validate_alter_result(TabletSharedPtr new_tablet,
- const TAlterTabletReqV2& request);
-
- static Status _convert_historical_rowsets(const SchemaChangeParams& sc_params,
- int64_t* real_alter_version);
-
- static Status _parse_request(const SchemaChangeParams& sc_params, BlockChanger* changer,
- bool* sc_sorting, bool* sc_directly);
+ Status _do_process_alter_tablet(const TAlterTabletReqV2& request);
+
+ Status _validate_alter_result(const TAlterTabletReqV2& request);
+
+ Status _convert_historical_rowsets(const SchemaChangeParams& sc_params,
+ int64_t* real_alter_version);
// Initialization Settings for creating a default value
static Status _init_column_mapping(ColumnMapping* column_mapping,
const TabletColumn& column_schema, const std::string& value);
- static Status _calc_delete_bitmap_for_mow_table(TabletSharedPtr new_tablet,
- int64_t alter_version);
+ Status _calc_delete_bitmap_for_mow_table(int64_t alter_version);
- static std::shared_mutex _mutex;
- static std::unordered_set<int64_t> _tablet_ids_in_converting;
- static std::set<std::string> _supported_functions;
+ StorageEngine& _local_storage_engine;
+ TabletSharedPtr _base_tablet;
+ TabletSharedPtr _new_tablet;
+ TabletSchemaSPtr _base_tablet_schema;
+ TabletSchemaSPtr _new_tablet_schema;
+ std::shared_mutex _mutex;
+ std::unordered_set<int64_t> _tablet_ids_in_converting;
+ std::set<std::string> _supported_functions;
};
} // namespace doris
diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h
index 551b2422381..aea9dade310 100644
--- a/be/src/olap/tablet.h
+++ b/be/src/olap/tablet.h
@@ -184,8 +184,6 @@ public:
std::shared_timed_mutex& get_migration_lock() { return _migration_lock; }
- std::mutex& get_schema_change_lock() { return _schema_change_lock; }
-
std::mutex& get_build_inverted_index_lock() { return _build_inverted_index_lock; }
// operation for compaction
@@ -506,7 +504,6 @@ private:
std::mutex _ingest_lock;
std::mutex _base_compaction_lock;
std::mutex _cumulative_compaction_lock;
- std::mutex _schema_change_lock;
std::shared_timed_mutex _migration_lock;
std::mutex _build_inverted_index_lock;
diff --git a/be/src/olap/task/engine_alter_tablet_task.cpp b/be/src/olap/task/engine_alter_tablet_task.cpp
deleted file mode 100644
index 30c3e95809d..00000000000
--- a/be/src/olap/task/engine_alter_tablet_task.cpp
+++ /dev/null
@@ -1,62 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "olap/task/engine_alter_tablet_task.h"
-
-#include <fmt/format.h>
-#include <gen_cpp/AgentService_types.h>
-#include <glog/logging.h>
-
-#include <ostream>
-#include <string>
-
-#include "common/config.h"
-#include "common/exception.h"
-#include "olap/schema_change.h"
-#include "runtime/memory/mem_tracker_limiter.h"
-#include "runtime/thread_context.h"
-#include "util/doris_metrics.h"
-
-namespace doris {
-
-EngineAlterTabletTask::EngineAlterTabletTask(const TAlterTabletReqV2& request)
- : _alter_tablet_req(request) {
- _mem_tracker = std::make_shared<MemTrackerLimiter>(
- MemTrackerLimiter::Type::SCHEMA_CHANGE,
- fmt::format("EngineAlterTabletTask#baseTabletId={}:newTabletId={}",
- std::to_string(_alter_tablet_req.base_tablet_id),
- std::to_string(_alter_tablet_req.new_tablet_id)),
- config::memory_limitation_per_thread_for_schema_change_bytes);
-}
-
-Status EngineAlterTabletTask::execute() {
- SCOPED_ATTACH_TASK(_mem_tracker);
- DorisMetrics::instance()->create_rollup_requests_total->increment(1);
- Status res = Status::OK();
- try {
- res = SchemaChangeHandler::process_alter_tablet_v2(_alter_tablet_req);
- } catch (const Exception& e) {
- res = e.to_status();
- }
- if (!res.ok()) {
- DorisMetrics::instance()->create_rollup_requests_failed->increment(1);
- return res;
- }
- return res;
-} // execute
-
-} // namespace doris
diff --git a/be/src/service/http_service.cpp b/be/src/service/http_service.cpp
index a5eafd087fd..84d930d267e 100644
--- a/be/src/service/http_service.cpp
+++ b/be/src/service/http_service.cpp
@@ -31,6 +31,7 @@
#include "http/action/check_rpc_channel_action.h"
#include "http/action/check_tablet_segment_action.h"
#include "http/action/checksum_action.h"
+#include "http/action/cloud_compaction_action.h"
#include "http/action/compaction_action.h"
#include "http/action/config_action.h"
#include "http/action/debug_point_action.h"
@@ -337,6 +338,24 @@ void HttpService::register_local_handler(StorageEngine& engine) {
void HttpService::register_cloud_handler(CloudStorageEngine& engine) {
// TODO(plat1ko)
+
+ // 2 compaction actions
+ CloudCompactionAction* show_compaction_action =
+ _pool.add(new CloudCompactionAction(CompactionActionType::SHOW_INFO, _env, engine,
+ TPrivilegeHier::GLOBAL, TPrivilegeType::ADMIN));
+ _ev_http_server->register_handler(HttpMethod::GET, "/api/compaction/show",
+ show_compaction_action);
+ CloudCompactionAction* run_compaction_action =
+ _pool.add(new CloudCompactionAction(CompactionActionType::RUN_COMPACTION, _env, engine,
+ TPrivilegeHier::GLOBAL, TPrivilegeType::ADMIN));
+ _ev_http_server->register_handler(HttpMethod::POST, "/api/compaction/run",
+ run_compaction_action);
+ CloudCompactionAction* run_status_compaction_action = _pool.add(
+ new CloudCompactionAction(CompactionActionType::RUN_COMPACTION_STATUS, _env, engine,
+ TPrivilegeHier::GLOBAL, TPrivilegeType::ADMIN));
+
+ _ev_http_server->register_handler(HttpMethod::GET, "/api/compaction/run_status",
+ run_status_compaction_action);
}
// NOLINTEND(readability-function-size)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/AlterHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/AlterHandler.java
index 43ad9de5ffd..362d84caed8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/AlterHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/AlterHandler.java
@@ -27,6 +27,7 @@ import org.apache.doris.catalog.Partition;
import org.apache.doris.catalog.Replica;
import org.apache.doris.catalog.Table;
import org.apache.doris.catalog.Tablet;
+import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.FeConstants;
@@ -81,7 +82,7 @@ public abstract class AlterHandler extends MasterDaemon {
lock.unlock();
}
- protected void addAlterJobV2(AlterJobV2 alterJob) {
+ protected void addAlterJobV2(AlterJobV2 alterJob) throws AnalysisException {
this.alterJobsV2.put(alterJob.getJobId(), alterJob);
LOG.info("add {} job {}", alterJob.getType(), alterJob.getJobId());
}
@@ -262,7 +263,7 @@ public abstract class AlterHandler extends MasterDaemon {
}
// replay the alter job v2
- public void replayAlterJobV2(AlterJobV2 alterJob) {
+ public void replayAlterJobV2(AlterJobV2 alterJob) throws AnalysisException {
AlterJobV2 existingJob = alterJobsV2.get(alterJob.getJobId());
if (existingJob == null) {
// This is the first time to replay the alter job, so just using the replayed alterJob to call replay();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/AlterJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/AlterJobV2.java
index 26c5fc96621..de96d297cb2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/AlterJobV2.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/AlterJobV2.java
@@ -28,6 +28,8 @@ import org.apache.doris.common.io.Text;
import org.apache.doris.common.io.Writable;
import org.apache.doris.common.util.DebugPointUtil;
import org.apache.doris.persist.gson.GsonUtils;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.task.AgentTask;
import com.google.gson.annotations.SerializedName;
import org.apache.logging.log4j.LogManager;
@@ -89,6 +91,8 @@ public abstract class AlterJobV2 implements Writable {
protected long timeoutMs = -1;
@SerializedName(value = "rawSql")
protected String rawSql;
+ @SerializedName(value = "cloudClusterName")
+ protected String cloudClusterName = "";
// The job will wait all transactions before this txn id finished, then send the schema_change/rollup tasks.
@SerializedName(value = "watershedTxnId")
@@ -113,6 +117,22 @@ public abstract class AlterJobV2 implements Writable {
this.type = type;
}
+ public String getCloudClusterName() {
+ return cloudClusterName;
+ }
+
+ public void setCloudClusterName(final String clusterName) {
+ cloudClusterName = clusterName;
+ }
+
+ protected void sleepSeveralSeconds() {
+ try {
+ Thread.sleep(10000);
+ } catch (InterruptedException ie) {
+ LOG.warn("ignore InterruptedException");
+ }
+ }
+
public long getJobId() {
return jobId;
}
@@ -198,6 +218,12 @@ public abstract class AlterJobV2 implements Writable {
return;
}
+ if (!cloudClusterName.isEmpty()) {
+ ConnectContext ctx = new ConnectContext();
+ ctx.setThreadLocalInfo();
+ ctx.setCloudCluster(cloudClusterName);
+ }
+
// /api/debug_point/add/FE.STOP_ALTER_JOB_RUN
if (DebugPointUtil.isEnable("FE.STOP_ALTER_JOB_RUN")) {
LOG.info("debug point FE.STOP_ALTER_JOB_RUN, schema change schedule stopped");
@@ -275,6 +301,8 @@ public abstract class AlterJobV2 implements Writable {
protected abstract void getInfo(List<List<Comparable>> infos);
+ protected void ensureCloudClusterExist(List<AgentTask> tasks) throws AlterCancelException {}
+
public abstract void replay(AlterJobV2 replayedJob);
public static AlterJobV2 read(DataInput in) throws IOException {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/AlterJobV2Factory.java b/fe/fe-core/src/main/java/org/apache/doris/alter/AlterJobV2Factory.java
new file mode 100644
index 00000000000..d5c937fd181
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/AlterJobV2Factory.java
@@ -0,0 +1,76 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.alter;
+
+import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.KeysType;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.Config;
+import org.apache.doris.qe.OriginStatement;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.io.IOException;
+import java.util.List;
+
+public class AlterJobV2Factory {
+ private static final Logger LOG = LogManager.getLogger(AlterJobV2Factory.class);
+
+ public static SchemaChangeJobV2 createSchemaChangeJobV2(String rawSql, long jobId, long dbId,
+ long tableId, String tableName, long timeoutMs) {
+ if (Config.isCloudMode()) {
+ return new CloudSchemaChangeJobV2(rawSql, jobId, dbId, tableId, tableName, timeoutMs);
+ } else {
+ return new SchemaChangeJobV2(rawSql, jobId, dbId, tableId, tableName, timeoutMs);
+ }
+ }
+
+ public static RollupJobV2 createRollupJobV2(String rawSql, long jobId, long dbId, long tableId,
+ String tableName, long timeoutMs, long baseIndexId,
+ long rollupIndexId, String baseIndexName, String rollupIndexName, List<Column> rollupSchema,
+ Column whereColumn,
+ int baseSchemaHash, int rollupSchemaHash, KeysType rollupKeysType,
+ short rollupShortKeyColumnCount,
+ OriginStatement origStmt) throws AnalysisException {
+ if (Config.isCloudMode()) {
+ return new CloudRollupJobV2(rawSql, jobId, dbId, tableId, tableName, timeoutMs, baseIndexId,
+ rollupIndexId, baseIndexName, rollupIndexName, rollupSchema, whereColumn,
+ baseSchemaHash, rollupSchemaHash, rollupKeysType, rollupShortKeyColumnCount, origStmt);
+ } else {
+ return new RollupJobV2(rawSql, jobId, dbId, tableId, tableName, timeoutMs, baseIndexId,
+ rollupIndexId, baseIndexName, rollupIndexName, rollupSchema, whereColumn,
+ baseSchemaHash, rollupSchemaHash, rollupKeysType, rollupShortKeyColumnCount, origStmt);
+ }
+ }
+
+ public static AlterJobV2 rebuildAlterJobV2(AlterJobV2 job) throws AnalysisException {
+ try {
+ if (Config.isCloudMode()) {
+ if (job.getType() == AlterJobV2.JobType.SCHEMA_CHANGE) {
+ job = CloudSchemaChangeJobV2.buildCloudSchemaChangeJobV2((SchemaChangeJobV2) job);
+ } else if (job.getType() == AlterJobV2.JobType.ROLLUP) {
+ job = CloudRollupJobV2.buildCloudRollupJobV2((RollupJobV2) job);
+ }
+ }
+ return job;
+ } catch (IOException e) {
+ throw new AnalysisException(e.getMessage());
+ }
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java
new file mode 100644
index 00000000000..60980d4699f
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java
@@ -0,0 +1,220 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.alter;
+
+import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.Database;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.KeysType;
+import org.apache.doris.catalog.MaterializedIndex;
+import org.apache.doris.catalog.OlapTable;
+import org.apache.doris.catalog.OlapTable.OlapTableState;
+import org.apache.doris.catalog.Partition;
+import org.apache.doris.catalog.Table;
+import org.apache.doris.catalog.Tablet;
+import org.apache.doris.cloud.datasource.CloudInternalCatalog;
+import org.apache.doris.cloud.proto.Cloud;
+import org.apache.doris.cloud.system.CloudSystemInfoService;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.MetaNotFoundException;
+import org.apache.doris.proto.OlapFile;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.OriginStatement;
+import org.apache.doris.task.AgentTask;
+import org.apache.doris.task.AgentTaskQueue;
+import org.apache.doris.thrift.TTabletType;
+import org.apache.doris.thrift.TTaskType;
+
+import com.google.common.base.Preconditions;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class CloudRollupJobV2 extends RollupJobV2 {
+ private static final Logger LOG = LogManager.getLogger(CloudRollupJobV2.class);
+
+ public static AlterJobV2 buildCloudRollupJobV2(RollupJobV2 job) throws IOException {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ DataOutputStream dos = new DataOutputStream(baos);
+ job.write(dos);
+ ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
+ DataInputStream dis = new DataInputStream(bais);
+ return CloudRollupJobV2.read(dis);
+ }
+
+ private CloudRollupJobV2() {}
+
+ // Don't call it directly, use AlterJobV2Factory to replace
+ public CloudRollupJobV2(String rawSql, long jobId, long dbId, long tableId, String tableName, long timeoutMs,
+ long baseIndexId,
+ long rollupIndexId, String baseIndexName, String rollupIndexName, List<Column> rollupSchema,
+ Column whereColumn,
+ int baseSchemaHash, int rollupSchemaHash, KeysType rollupKeysType,
+ short rollupShortKeyColumnCount,
+ OriginStatement origStmt) throws AnalysisException {
+ super(rawSql, jobId, dbId, tableId, tableName, timeoutMs, baseIndexId,
+ rollupIndexId, baseIndexName, rollupIndexName, rollupSchema, whereColumn,
+ baseSchemaHash, rollupSchemaHash, rollupKeysType, rollupShortKeyColumnCount, origStmt);
+ ConnectContext context = ConnectContext.get();
+ if (context != null) {
+ LOG.debug("rollup job add cloud cluster, context not null, cluster: {}", context.getCloudCluster());
+ setCloudClusterName(context.getCloudCluster());
+ }
+ LOG.debug("rollup job add cloud cluster, context {}", context);
+ }
+
+ @Override
+ protected void onCreateRollupReplicaDone() throws AlterCancelException {
+ List<Long> rollupIndexList = new ArrayList<Long>();
+ rollupIndexList.add(rollupIndexId);
+ try {
+ ((CloudInternalCatalog) Env.getCurrentInternalCatalog())
+ .commitMaterializedIndex(tableId, rollupIndexList);
+ } catch (Exception e) {
+ LOG.warn("commitMaterializedIndex Exception:{}", e);
+ throw new AlterCancelException(e.getMessage());
+ }
+
+ LOG.info("onCreateRollupReplicaDone finished, dbId:{}, tableId:{}, jobId:{}, rollupIndexList:{}",
+ dbId, tableId, jobId, rollupIndexList);
+ }
+
+ @Override
+ protected void onCancel() {
+ List<Long> rollupIndexList = new ArrayList<Long>();
+ rollupIndexList.add(rollupIndexId);
+ long tryTimes = 1;
+ while (true) {
+ try {
+ ((CloudInternalCatalog) Env.getCurrentInternalCatalog())
+ .dropMaterializedIndex(tableId, rollupIndexList);
+ break;
+ } catch (Exception e) {
+ LOG.warn("tryTimes:{}, onCancel exception:", tryTimes, e);
+ }
+ sleepSeveralSeconds();
+ tryTimes++;
+ }
+
+ LOG.info("onCancel finished, dbId:{}, tableId:{}, jobId:{}, rollupIndexList:{}",
+ dbId, tableId, jobId, rollupIndexList);
+ }
+
+ @Override
+ protected void createRollupReplica() throws AlterCancelException {
+ Database db = Env.getCurrentInternalCatalog()
+ .getDbOrException(dbId, s -> new AlterCancelException("Database " + s + " does not exist"));
+
+ // 1. create rollup replicas
+ OlapTable tbl;
+ try {
+ tbl = (OlapTable) db.getTableOrMetaException(tableId, Table.TableType.OLAP);
+ } catch (MetaNotFoundException e) {
+ throw new AlterCancelException(e.getMessage());
+ }
+
+ long expiration = (createTimeMs + timeoutMs) / 1000;
+ tbl.readLock();
+ try {
+ Preconditions.checkState(tbl.getState() == OlapTableState.ROLLUP);
+ try {
+ List<Long> rollupIndexList = new ArrayList<Long>();
+ rollupIndexList.add(rollupIndexId);
+ ((CloudInternalCatalog) Env.getCurrentInternalCatalog())
+ .prepareMaterializedIndex(tbl.getId(), rollupIndexList, expiration);
+ createRollupReplicaForPartition(tbl);
+ } catch (Exception e) {
+ LOG.warn("createCloudShadowIndexReplica Exception:{}", e);
+ throw new AlterCancelException(e.getMessage());
+ }
+ } finally {
+ tbl.readUnlock();
+ }
+
+ // create all rollup replicas success.
+ // add rollup index to catalog
+ tbl.writeLockOrAlterCancelException();
+ try {
+ Preconditions.checkState(tbl.getState() == OlapTableState.ROLLUP);
+ addRollupIndexToCatalog(tbl);
+ } finally {
+ tbl.writeUnlock();
+ }
+ }
+
+ private void createRollupReplicaForPartition(OlapTable tbl) throws Exception {
+ for (Map.Entry<Long, MaterializedIndex> entry : this.partitionIdToRollupIndex.entrySet()) {
+ long partitionId = entry.getKey();
+ Partition partition = tbl.getPartition(partitionId);
+ if (partition == null) {
+ continue;
+ }
+ TTabletType tabletType = tbl.getPartitionInfo().getTabletType(partitionId);
+ MaterializedIndex rollupIndex = entry.getValue();
+ Cloud.CreateTabletsRequest.Builder requestBuilder =
+ Cloud.CreateTabletsRequest.newBuilder();
+ for (Tablet rollupTablet : rollupIndex.getTablets()) {
+ OlapFile.TabletMetaCloudPB.Builder builder =
+ ((CloudInternalCatalog) Env.getCurrentInternalCatalog())
+ .createTabletMetaBuilder(tableId, rollupIndexId,
+ partitionId, rollupTablet, tabletType, rollupSchemaHash,
+ rollupKeysType, rollupShortKeyColumnCount, tbl.getCopiedBfColumns(),
+ tbl.getBfFpp(), null, rollupSchema,
+ tbl.getDataSortInfo(), tbl.getCompressionType(), tbl.getStoragePolicy(),
+ tbl.isInMemory(), true,
+ tbl.getName(), tbl.getTTLSeconds(),
+ tbl.getEnableUniqueKeyMergeOnWrite(), tbl.storeRowColumn(),
+ tbl.getBaseSchemaVersion());
+ requestBuilder.addTabletMetas(builder);
+ } // end for rollupTablets
+ ((CloudInternalCatalog) Env.getCurrentInternalCatalog())
+ .sendCreateTabletsRpc(requestBuilder);
+ }
+ }
+
+ @Override
+ protected void ensureCloudClusterExist(List<AgentTask> tasks) throws AlterCancelException {
+ if (((CloudSystemInfoService) Env.getCurrentSystemInfo())
+ .getCloudClusterIdByName(cloudClusterName) == null) {
+ for (AgentTask task : tasks) {
+ task.setFinished(true);
+ AgentTaskQueue.removeTask(task.getBackendId(), TTaskType.ALTER, task.getSignature());
+ }
+ StringBuilder sb = new StringBuilder("cloud cluster(");
+ sb.append(cloudClusterName);
+ sb.append(") has been removed, jobId=");
+ sb.append(jobId);
+ String msg = sb.toString();
+ LOG.warn(msg);
+ throw new AlterCancelException(msg);
+ }
+ }
+
+ @Override
+ protected boolean checkTableStable(Database db) throws AlterCancelException {
+ return true;
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java
new file mode 100644
index 00000000000..848653ca6ac
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java
@@ -0,0 +1,229 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.alter;
+
+import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.Database;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.KeysType;
+import org.apache.doris.catalog.MaterializedIndex;
+import org.apache.doris.catalog.OlapTable;
+import org.apache.doris.catalog.OlapTable.OlapTableState;
+import org.apache.doris.catalog.Partition;
+import org.apache.doris.catalog.TableIf.TableType;
+import org.apache.doris.catalog.Tablet;
+import org.apache.doris.cloud.datasource.CloudInternalCatalog;
+import org.apache.doris.cloud.proto.Cloud;
+import org.apache.doris.cloud.system.CloudSystemInfoService;
+import org.apache.doris.common.MetaNotFoundException;
+import org.apache.doris.proto.OlapFile;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.task.AgentTask;
+import org.apache.doris.task.AgentTaskQueue;
+import org.apache.doris.thrift.TTaskType;
+
+import com.google.common.base.Preconditions;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+public class CloudSchemaChangeJobV2 extends SchemaChangeJobV2 {
+ private static final Logger LOG = LogManager.getLogger(SchemaChangeJobV2.class);
+
+ public static AlterJobV2 buildCloudSchemaChangeJobV2(SchemaChangeJobV2 job) throws IOException {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ DataOutputStream dos = new DataOutputStream(baos);
+ job.write(dos);
+ ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
+ DataInputStream dis = new DataInputStream(bais);
+ return CloudSchemaChangeJobV2.read(dis);
+ }
+
+ public CloudSchemaChangeJobV2(String rawSql, long jobId, long dbId, long tableId,
+ String tableName, long timeoutMs) {
+ super(rawSql, jobId, dbId, tableId, tableName, timeoutMs);
+ ConnectContext context = ConnectContext.get();
+ if (context != null) {
+ LOG.debug("schema change job add cloud cluster, context not null, cluster: {}",
+ context.getCloudCluster());
+ setCloudClusterName(context.getCloudCluster());
+ }
+ LOG.debug("schema change job add cloud cluster, context {}", context);
+ }
+
+ private CloudSchemaChangeJobV2() {}
+
+ @Override
+ protected void commitShadowIndex() throws AlterCancelException {
+ List<Long> shadowIdxList =
+ indexIdMap.keySet().stream().collect(Collectors.toList());
+ try {
+ ((CloudInternalCatalog) Env.getCurrentInternalCatalog())
+ .commitMaterializedIndex(tableId,
+ shadowIdxList);
+ } catch (Exception e) {
+ LOG.warn("commitMaterializedIndex exception:", e);
+ throw new AlterCancelException(e.getMessage());
+ }
+ LOG.info("commitShadowIndex finished, dbId:{}, tableId:{}, jobId:{}, shadowIdxList:{}",
+ dbId, tableId, jobId, shadowIdxList);
+ }
+
+ @Override
+ protected void postProcessShadowIndex() {
+ List<Long> shadowIdxList = indexIdMap.keySet().stream().collect(Collectors.toList());
+ dropIndex(shadowIdxList);
+ }
+
+ @Override
+ protected void postProcessOriginIndex() {
+ List<Long> originIdxList = indexIdMap.values().stream().collect(Collectors.toList());
+ dropIndex(originIdxList);
+ }
+
+ private void dropIndex(List<Long> idxList) {
+ int tryTimes = 1;
+ while (true) {
+ try {
+ ((CloudInternalCatalog) Env.getCurrentInternalCatalog())
+ .dropMaterializedIndex(tableId, idxList);
+ break;
+ } catch (Exception e) {
+ LOG.warn("tryTimes:{}, dropIndex exception:", tryTimes, e);
+ }
+ sleepSeveralSeconds();
+ tryTimes++;
+ }
+
+ LOG.info("dropIndex finished, dbId:{}, tableId:{}, jobId:{}, IdxList:{}",
+ dbId, tableId, jobId, idxList);
+ }
+
+ @Override
+ protected void createShadowIndexReplica() throws AlterCancelException {
+ Database db = Env.getCurrentInternalCatalog()
+ .getDbOrException(dbId, s -> new AlterCancelException("Database " + s + " does not exist"));
+
+ // 1. create replicas
+ OlapTable tbl;
+ try {
+ tbl = (OlapTable) db.getTableOrMetaException(tableId, TableType.OLAP);
+ } catch (MetaNotFoundException e) {
+ throw new AlterCancelException(e.getMessage());
+ }
+
+ Long expiration = (createTimeMs + timeoutMs) / 1000;
+ tbl.readLock();
+ try {
+ Preconditions.checkState(tbl.getState() == OlapTableState.SCHEMA_CHANGE);
+ try {
+ List<Long> shadowIdxList = indexIdMap.keySet().stream().collect(Collectors.toList());
+ ((CloudInternalCatalog) Env.getCurrentInternalCatalog())
+ .prepareMaterializedIndex(tableId, shadowIdxList,
+ expiration);
+ createShadowIndexReplicaForPartition(tbl);
+ } catch (Exception e) {
+ LOG.warn("createCloudShadowIndexReplica Exception:", e);
+ throw new AlterCancelException(e.getMessage());
+ }
+
+ } finally {
+ tbl.readUnlock();
+ }
+
+ // create all replicas success.
+ // add all shadow indexes to catalog
+ tbl.writeLockOrAlterCancelException();
+ try {
+ Preconditions.checkState(tbl.getState() == OlapTableState.SCHEMA_CHANGE);
+ addShadowIndexToCatalog(tbl);
+ } finally {
+ tbl.writeUnlock();
+ }
+ }
+
+ private void createShadowIndexReplicaForPartition(OlapTable tbl) throws Exception {
+ for (long partitionId : partitionIndexMap.rowKeySet()) {
+ Partition partition = tbl.getPartition(partitionId);
+ if (partition == null) {
+ continue;
+ }
+ Map<Long, MaterializedIndex> shadowIndexMap = partitionIndexMap.row(partitionId);
+ for (Map.Entry<Long, MaterializedIndex> entry : shadowIndexMap.entrySet()) {
+ long shadowIdxId = entry.getKey();
+ MaterializedIndex shadowIdx = entry.getValue();
+
+ short shadowShortKeyColumnCount = indexShortKeyMap.get(shadowIdxId);
+ List<Column> shadowSchema = indexSchemaMap.get(shadowIdxId);
+ int shadowSchemaHash = indexSchemaVersionAndHashMap.get(shadowIdxId).schemaHash;
+ int shadowSchemaVersion = indexSchemaVersionAndHashMap.get(shadowIdxId).schemaVersion;
+ long originIndexId = indexIdMap.get(shadowIdxId);
+ KeysType originKeysType = tbl.getKeysTypeByIndexId(originIndexId);
+
+ Cloud.CreateTabletsRequest.Builder requestBuilder =
+ Cloud.CreateTabletsRequest.newBuilder();
+ for (Tablet shadowTablet : shadowIdx.getTablets()) {
+ OlapFile.TabletMetaCloudPB.Builder builder =
+ ((CloudInternalCatalog) Env.getCurrentInternalCatalog())
+ .createTabletMetaBuilder(tableId, shadowIdxId,
+ partitionId, shadowTablet, tbl.getPartitionInfo().getTabletType(partitionId),
+ shadowSchemaHash, originKeysType, shadowShortKeyColumnCount, bfColumns,
+ bfFpp, indexes, shadowSchema, tbl.getDataSortInfo(), tbl.getCompressionType(),
+ tbl.getStoragePolicy(), tbl.isInMemory(), true,
+ tbl.getName(), tbl.getTTLSeconds(),
+ tbl.getEnableUniqueKeyMergeOnWrite(), tbl.storeRowColumn(),
+ shadowSchemaVersion);
+ requestBuilder.addTabletMetas(builder);
+ } // end for rollupTablets
+ ((CloudInternalCatalog) Env.getCurrentInternalCatalog())
+ .sendCreateTabletsRpc(requestBuilder);
+ }
+ }
+ }
+
+ @Override
+ protected void ensureCloudClusterExist(List<AgentTask> tasks) throws AlterCancelException {
+ if (((CloudSystemInfoService) Env.getCurrentSystemInfo())
+ .getCloudClusterIdByName(cloudClusterName) == null) {
+ for (AgentTask task : tasks) {
+ task.setFinished(true);
+ AgentTaskQueue.removeTask(task.getBackendId(), TTaskType.ALTER, task.getSignature());
+ }
+ StringBuilder sb = new StringBuilder("cloud cluster(");
+ sb.append(cloudClusterName);
+ sb.append(") has been removed, jobId=");
+ sb.append(jobId);
+ String msg = sb.toString();
+ LOG.warn(msg);
+ throw new AlterCancelException(msg);
+ }
+ }
+
+ @Override
+ protected boolean checkTableStable(Database db) throws AlterCancelException {
+ return true;
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java
index 1dfe7759ec8..82388d62348 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java
@@ -42,6 +42,7 @@ import org.apache.doris.catalog.OlapTable.OlapTableState;
import org.apache.doris.catalog.Partition;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.Replica;
+import org.apache.doris.catalog.Replica.ReplicaContext;
import org.apache.doris.catalog.Replica.ReplicaState;
import org.apache.doris.catalog.Table;
import org.apache.doris.catalog.Tablet;
@@ -111,12 +112,13 @@ public class MaterializedViewHandler extends AlterHandler {
private Map<Long, Set<Long>> tableRunningJobMap = new ConcurrentHashMap<>();
@Override
- public void addAlterJobV2(AlterJobV2 alterJob) {
+ public void addAlterJobV2(AlterJobV2 alterJob) throws AnalysisException {
+ alterJob = AlterJobV2Factory.rebuildAlterJobV2(alterJob);
super.addAlterJobV2(alterJob);
addAlterJobV2ToTableNotFinalStateJobMap(alterJob);
}
- protected void batchAddAlterJobV2(List<AlterJobV2> alterJobV2List) {
+ protected void batchAddAlterJobV2(List<AlterJobV2> alterJobV2List) throws AnalysisException {
for (AlterJobV2 alterJobV2 : alterJobV2List) {
addAlterJobV2(alterJobV2);
}
@@ -377,7 +379,8 @@ public class MaterializedViewHandler extends AlterHandler {
IdGeneratorBuffer idGeneratorBuffer = env.getIdGeneratorBuffer(bufferSize);
long jobId = idGeneratorBuffer.getNextId();
long mvIndexId = idGeneratorBuffer.getNextId();
- RollupJobV2 mvJob = new RollupJobV2(rawSql, jobId, dbId, tableId, olapTable.getName(), timeoutMs,
+ RollupJobV2 mvJob = AlterJobV2Factory.createRollupJobV2(
+ rawSql, jobId, dbId, tableId, olapTable.getName(), timeoutMs,
baseIndexId, mvIndexId, baseIndexName, mvName,
mvColumns, whereColumn, baseSchemaHash, mvSchemaHash,
mvKeysType, mvShortKeyColumnCount, origStmt);
@@ -429,9 +432,19 @@ public class MaterializedViewHandler extends AlterHandler {
}
Preconditions.checkState(baseReplica.getState() == ReplicaState.NORMAL,
baseReplica.getState());
+ ReplicaContext context = new ReplicaContext();
+ context.replicaId = mvReplicaId;
+ context.backendId = backendId;
+ context.state = ReplicaState.ALTER;
+ context.version = Partition.PARTITION_INIT_VERSION;
+ context.schemaHash = mvSchemaHash;
+ context.dbId = dbId;
+ context.tableId = tableId;
+ context.partitionId = partitionId;
+ context.indexId = mvIndexId;
+ context.originReplica = baseReplica;
// replica's init state is ALTER, so that tablet report process will ignore its report
- Replica mvReplica = new Replica(mvReplicaId, backendId, ReplicaState.ALTER,
- Partition.PARTITION_INIT_VERSION, mvSchemaHash);
+ Replica mvReplica = EnvFactory.getInstance().createReplica(context);
newTablet.addReplica(mvReplica);
healthyReplicaNum++;
} // end for baseReplica
@@ -1092,7 +1105,8 @@ public class MaterializedViewHandler extends AlterHandler {
// replay the alter job v2
@Override
- public void replayAlterJobV2(AlterJobV2 alterJob) {
+ public void replayAlterJobV2(AlterJobV2 alterJob) throws AnalysisException {
+ alterJob = AlterJobV2Factory.rebuildAlterJobV2(alterJob);
super.replayAlterJobV2(alterJob);
if (!alterJob.isDone()) {
addAlterJobV2ToTableNotFinalStateJobMap(alterJob);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
index b68f8bf23b0..e8abaab5777 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
@@ -101,33 +101,33 @@ public class RollupJobV2 extends AlterJobV2 implements GsonPostProcessable {
@SerializedName(value = "partitionIdToBaseRollupTabletIdMap")
private Map<Long, Map<Long, Long>> partitionIdToBaseRollupTabletIdMap = Maps.newHashMap();
@SerializedName(value = "partitionIdToRollupIndex")
- private Map<Long, MaterializedIndex> partitionIdToRollupIndex = Maps.newHashMap();
+ protected Map<Long, MaterializedIndex> partitionIdToRollupIndex = Maps.newHashMap();
// rollup and base schema info
@SerializedName(value = "baseIndexId")
- private long baseIndexId;
+ protected long baseIndexId;
@SerializedName(value = "rollupIndexId")
- private long rollupIndexId;
+ protected long rollupIndexId;
@SerializedName(value = "baseIndexName")
- private String baseIndexName;
+ protected String baseIndexName;
@SerializedName(value = "rollupIndexName")
- private String rollupIndexName;
+ protected String rollupIndexName;
@SerializedName(value = "rollupSchema")
- private List<Column> rollupSchema = Lists.newArrayList();
+ protected List<Column> rollupSchema = Lists.newArrayList();
@SerializedName(value = "whereColumn")
- private Column whereColumn;
+ protected Column whereColumn;
@SerializedName(value = "baseSchemaHash")
- private int baseSchemaHash;
+ protected int baseSchemaHash;
@SerializedName(value = "rollupSchemaHash")
- private int rollupSchemaHash;
+ protected int rollupSchemaHash;
@SerializedName(value = "rollupKeysType")
- private KeysType rollupKeysType;
+ protected KeysType rollupKeysType;
@SerializedName(value = "rollupShortKeyColumnCount")
- private short rollupShortKeyColumnCount;
+ protected short rollupShortKeyColumnCount;
@SerializedName(value = "origStmt")
- private OriginStatement origStmt;
+ protected OriginStatement origStmt;
// optional
@SerializedName(value = "storageFormat")
@@ -140,10 +140,11 @@ public class RollupJobV2 extends AlterJobV2 implements GsonPostProcessable {
private Analyzer analyzer;
- private RollupJobV2() {
+ protected RollupJobV2() {
super(JobType.ROLLUP);
}
+ // Don't call it directly, use AlterJobV2Factory to replace
public RollupJobV2(String rawSql, long jobId, long dbId, long tableId, String tableName, long timeoutMs,
long baseIndexId,
long rollupIndexId, String baseIndexName, String rollupIndexName, List<Column> rollupSchema,
@@ -177,6 +178,9 @@ public class RollupJobV2 extends AlterJobV2 implements GsonPostProcessable {
}
public void addMVIndex(long partitionId, MaterializedIndex mvIndex) {
+ if (mvIndex == null) {
+ LOG.info("lightman ?????????");
+ }
this.partitionIdToRollupIndex.put(partitionId, mvIndex);
}
@@ -196,24 +200,10 @@ public class RollupJobV2 extends AlterJobV2 implements GsonPostProcessable {
analyzer = new Analyzer(Env.getCurrentEnv(), connectContext);
}
- /**
- * runPendingJob():
- * 1. Create all rollup replicas and wait them finished.
- * 2. After creating done, add this shadow rollup index to catalog, user can not see this
- * rollup, but internal load process will generate data for this rollup index.
- * 3. Get a new transaction id, then set job's state to WAITING_TXN
- */
- @Override
- protected void runPendingJob() throws Exception {
- Preconditions.checkState(jobState == JobState.PENDING, jobState);
-
- LOG.info("begin to send create rollup replica tasks. job: {}", jobId);
+ protected void createRollupReplica() throws AlterCancelException {
Database db = Env.getCurrentInternalCatalog()
.getDbOrException(dbId, s -> new AlterCancelException("Database " + s + " does not exist"));
- if (!checkTableStable(db)) {
- return;
- }
-
+ LOG.info("lightman1 add size {}", this.partitionIdToRollupIndex.size());
// 1. create rollup replicas
AgentBatchTask batchTask = new AgentBatchTask();
// count total replica num
@@ -230,7 +220,7 @@ public class RollupJobV2 extends AlterJobV2 implements GsonPostProcessable {
} catch (MetaNotFoundException e) {
throw new AlterCancelException(e.getMessage());
}
-
+ LOG.info("lightman2 add size {}", this.partitionIdToRollupIndex.size());
tbl.readLock();
try {
BinlogConfig binlogConfig = new BinlogConfig(tbl.getBinlogConfig());
@@ -289,7 +279,7 @@ public class RollupJobV2 extends AlterJobV2 implements GsonPostProcessable {
} finally {
tbl.readUnlock();
}
-
+ LOG.info("lightman3 add size {}", this.partitionIdToRollupIndex.size());
if (!FeConstants.runningUnitTest) {
// send all tasks and wait them finished
AgentTaskQueue.addBatchTask(batchTask);
@@ -321,7 +311,7 @@ public class RollupJobV2 extends AlterJobV2 implements GsonPostProcessable {
throw new AlterCancelException("Create rollup replicas failed. Error: " + errMsg);
}
}
-
+ LOG.info("lightman4 add size {}", this.partitionIdToRollupIndex.size());
// create all rollup replicas success.
// add rollup index to catalog
tbl.writeLockOrAlterCancelException();
@@ -331,6 +321,26 @@ public class RollupJobV2 extends AlterJobV2 implements GsonPostProcessable {
} finally {
tbl.writeUnlock();
}
+ }
+
+ /**
+ * runPendingJob():
+ * 1. Create all rollup replicas and wait them finished.
+ * 2. After creating done, add this shadow rollup index to catalog, user can not see this
+ * rollup, but internal load process will generate data for this rollup index.
+ * 3. Get a new transaction id, then set job's state to WAITING_TXN
+ */
+ @Override
+ protected void runPendingJob() throws Exception {
+ Preconditions.checkState(jobState == JobState.PENDING, jobState);
+
+ LOG.info("begin to send create rollup replica tasks. job: {}", jobId);
+ Database db = Env.getCurrentInternalCatalog()
+ .getDbOrException(dbId, s -> new AlterCancelException("Database " + s + " does not exist"));
+ if (!checkTableStable(db)) {
+ return;
+ }
+ createRollupReplica();
this.watershedTxnId = Env.getCurrentGlobalTransactionMgr().getNextTransactionId();
this.jobState = JobState.WAITING_TXN;
@@ -340,10 +350,14 @@ public class RollupJobV2 extends AlterJobV2 implements GsonPostProcessable {
LOG.info("transfer rollup job {} state to {}, watershed txn id: {}", jobId, this.jobState, watershedTxnId);
}
- private void addRollupIndexToCatalog(OlapTable tbl) {
+ protected void addRollupIndexToCatalog(OlapTable tbl) {
for (Partition partition : tbl.getPartitions()) {
long partitionId = partition.getId();
MaterializedIndex rollupIndex = this.partitionIdToRollupIndex.get(partitionId);
+ if (rollupIndex == null) {
+ LOG.warn("lightman partitionId {} size {}", partitionId,
+ this.partitionIdToRollupIndex.size());
+ }
Preconditions.checkNotNull(rollupIndex);
Preconditions.checkState(rollupIndex.getState() == IndexState.SHADOW, rollupIndex.getState());
partition.createRollupIndex(rollupIndex);
@@ -388,6 +402,7 @@ public class RollupJobV2 extends AlterJobV2 implements GsonPostProcessable {
tbl.readLock();
Map<Object, List<TColumn>> tcloumnsPool = Maps.newHashMap();
try {
+ long expiration = (createTimeMs + timeoutMs) / 1000;
Preconditions.checkState(tbl.getState() == OlapTableState.ROLLUP);
for (Map.Entry<Long, MaterializedIndex> entry : this.partitionIdToRollupIndex.entrySet()) {
long partitionId = entry.getKey();
@@ -466,8 +481,7 @@ public class RollupJobV2 extends AlterJobV2 implements GsonPostProcessable {
partitionId, rollupIndexId, baseIndexId, rollupTabletId, baseTabletId,
rollupReplica.getId(), rollupSchemaHash, baseSchemaHash, visibleVersion, jobId,
JobType.ROLLUP, defineExprs, descTable, tbl.getSchemaByIndexId(baseIndexId, true),
- tcloumnsPool,
- whereClause);
+ tcloumnsPool, whereClause, expiration);
rollupBatchTask.addTask(rollupTask);
}
}
@@ -507,9 +521,11 @@ public class RollupJobV2 extends AlterJobV2 implements GsonPostProcessable {
} catch (MetaNotFoundException e) {
throw new AlterCancelException(e.getMessage());
}
+ LOG.debug("jobId:{}, cloudClusterName:{}", jobId, cloudClusterName);
if (!rollupBatchTask.isFinished()) {
LOG.info("rollup tasks not finished. job: {}", jobId);
List<AgentTask> tasks = rollupBatchTask.getUnfinishedTasks(2000);
+ ensureCloudClusterExist(tasks);
for (AgentTask task : tasks) {
if (task.getFailedTimes() > 0) {
task.setFinished(true);
@@ -574,7 +590,7 @@ public class RollupJobV2 extends AlterJobV2 implements GsonPostProcessable {
}
} // end for tablets
} // end for partitions
-
+ onCreateRollupReplicaDone();
onFinished(tbl);
} finally {
tbl.writeUnlock();
@@ -631,6 +647,9 @@ public class RollupJobV2 extends AlterJobV2 implements GsonPostProcessable {
this.errMsg = errMsg;
this.finishedTimeMs = System.currentTimeMillis();
Env.getCurrentEnv().getEditLog().logAlterJob(this);
+ // try best to drop roll index, when job is cancelled
+ onCancel();
+
LOG.info("cancel {} job {}, err: {}", this.type, jobId, errMsg);
return true;
}
@@ -760,6 +779,8 @@ public class RollupJobV2 extends AlterJobV2 implements GsonPostProcessable {
*/
private void replayCancelled(RollupJobV2 replayedJob) {
cancelInternal();
+ // try best to drop roll index, when job is cancelled
+ onCancel();
this.jobState = JobState.CANCELLED;
this.finishedTimeMs = replayedJob.finishedTimeMs;
this.errMsg = replayedJob.errMsg;
@@ -881,6 +902,11 @@ public class RollupJobV2 extends AlterJobV2 implements GsonPostProcessable {
setColumnsDefineExpr(stmt.getMVColumnItemList());
}
+ protected void onCreateRollupReplicaDone() throws AlterCancelException {}
+
+ // try best to drop roll index, when job is cancelled
+ protected void onCancel() {}
+
@Override
public String toJson() {
return GsonUtils.GSON.toJson(this);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
index a348aa26de1..bab1a0c32b8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
@@ -58,6 +58,7 @@ import org.apache.doris.catalog.PartitionType;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.RandomDistributionInfo;
import org.apache.doris.catalog.Replica;
+import org.apache.doris.catalog.Replica.ReplicaContext;
import org.apache.doris.catalog.Replica.ReplicaState;
import org.apache.doris.catalog.ReplicaAllocation;
import org.apache.doris.catalog.Table;
@@ -1504,7 +1505,7 @@ public class SchemaChangeHandler extends AlterHandler {
IdGeneratorBuffer idGeneratorBuffer = Env.getCurrentEnv().getIdGeneratorBuffer(bufferSize);
long jobId = idGeneratorBuffer.getNextId();
SchemaChangeJobV2 schemaChangeJob =
- new SchemaChangeJobV2(rawSql, jobId, dbId, olapTable.getId(), olapTable.getName(),
+ AlterJobV2Factory.createSchemaChangeJobV2(rawSql, jobId, dbId, olapTable.getId(), olapTable.getName(),
timeoutSecond * 1000);
schemaChangeJob.setBloomFilterInfo(hasBfChange, bfColumns, bfFpp);
schemaChangeJob.setAlterIndexInfo(hasIndexChange, indexes);
@@ -1584,9 +1585,19 @@ public class SchemaChangeHandler extends AlterHandler {
}
Preconditions.checkState(originReplica.getState() == ReplicaState.NORMAL,
originReplica.getState());
+ ReplicaContext context = new ReplicaContext();
+ context.replicaId = shadowReplicaId;
+ context.backendId = backendId;
+ context.state = ReplicaState.ALTER;
+ context.version = Partition.PARTITION_INIT_VERSION;
+ context.schemaHash = newSchemaHash;
+ context.dbId = dbId;
+ context.tableId = tableId;
+ context.partitionId = partitionId;
+ context.indexId = shadowIndexId;
+ context.originReplica = originReplica;
// replica's init state is ALTER, so that tablet report process will ignore its report
- Replica shadowReplica = new Replica(shadowReplicaId, backendId, ReplicaState.ALTER,
- Partition.PARTITION_INIT_VERSION, newSchemaHash);
+ Replica shadowReplica = EnvFactory.getInstance().createReplica(context);
shadowTablet.addReplica(shadowReplica);
healthyReplicaNum++;
}
@@ -1894,12 +1905,10 @@ public class SchemaChangeHandler extends AlterHandler {
lightSchemaChange = Boolean.parseBoolean(
properties.get(PropertyAnalyzer.PROPERTIES_ENABLE_LIGHT_SCHEMA_CHANGE));
if (Objects.equals(olapTable.getEnableLightSchemaChange(), lightSchemaChange)) {
- throw new DdlException(
- String.format("Table %s.%s has already support light_schema_change=%s",
- db.getFullName(), olapTable.getName(), lightSchemaChange));
+ return;
}
if (!lightSchemaChange) {
- throw new DdlException("Can not alter light_schema_change to false currently");
+ throw new DdlException("Can not alter light_schema_change from true to false currently");
}
enableLightSchemaChange(db, olapTable);
return;
@@ -2591,7 +2600,8 @@ public class SchemaChangeHandler extends AlterHandler {
}
@Override
- public void addAlterJobV2(AlterJobV2 alterJob) {
+ public void addAlterJobV2(AlterJobV2 alterJob) throws AnalysisException {
+ alterJob = AlterJobV2Factory.rebuildAlterJobV2(alterJob);
super.addAlterJobV2(alterJob);
runnableSchemaChangeJobV2.put(alterJob.getJobId(), alterJob);
}
@@ -2641,7 +2651,8 @@ public class SchemaChangeHandler extends AlterHandler {
}
@Override
- public void replayAlterJobV2(AlterJobV2 alterJob) {
+ public void replayAlterJobV2(AlterJobV2 alterJob) throws AnalysisException {
+ alterJob = AlterJobV2Factory.rebuildAlterJobV2(alterJob);
if (!alterJob.isDone() && !runnableSchemaChangeJobV2.containsKey(alterJob.getJobId())) {
runnableSchemaChangeJobV2.put(alterJob.getJobId(), alterJob);
}
@@ -2653,7 +2664,7 @@ public class SchemaChangeHandler extends AlterHandler {
Map<Long, LinkedList<Column>> indexSchemaMap, List<Index> indexes,
List<Index> alterIndexes, boolean isDropIndex,
long jobId, boolean isReplay)
- throws DdlException {
+ throws DdlException, AnalysisException {
if (LOG.isDebugEnabled()) {
LOG.debug("indexSchemaMap:{}, indexes:{}", indexSchemaMap, indexes);
@@ -2680,7 +2691,8 @@ public class SchemaChangeHandler extends AlterHandler {
//for compatibility, we need create a finished state schema change job v2
- SchemaChangeJobV2 schemaChangeJob = new SchemaChangeJobV2(rawSql, jobId, db.getId(), olapTable.getId(),
+ SchemaChangeJobV2 schemaChangeJob = AlterJobV2Factory.createSchemaChangeJobV2(
+ rawSql, jobId, db.getId(), olapTable.getId(),
olapTable.getName(), 1000);
for (Map.Entry<Long, List<Column>> entry : changedIndexIdToSchema.entrySet()) {
@@ -2747,7 +2759,8 @@ public class SchemaChangeHandler extends AlterHandler {
}
}
- public void replayModifyTableLightSchemaChange(TableAddOrDropColumnsInfo info) throws MetaNotFoundException {
+ public void replayModifyTableLightSchemaChange(TableAddOrDropColumnsInfo info)
+ throws MetaNotFoundException, AnalysisException {
if (LOG.isDebugEnabled()) {
LOG.debug("info:{}", info);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
index 70787936105..8211dd3ccef 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
@@ -92,10 +92,10 @@ public class SchemaChangeJobV2 extends AlterJobV2 {
private Table<Long, Long, Map<Long, Long>> partitionIndexTabletMap = HashBasedTable.create();
// partition id -> (shadow index id -> shadow index))
@SerializedName(value = "partitionIndexMap")
- private Table<Long, Long, MaterializedIndex> partitionIndexMap = HashBasedTable.create();
+ protected Table<Long, Long, MaterializedIndex> partitionIndexMap = HashBasedTable.create();
// shadow index id -> origin index id
@SerializedName(value = "indexIdMap")
- private Map<Long, Long> indexIdMap = Maps.newHashMap();
+ protected Map<Long, Long> indexIdMap = Maps.newHashMap();
// partition id -> origin index id
@SerializedName(value = "partitionOriginIndexIdMap")
private Map<Long, Long> partitionOriginIndexIdMap = Maps.newHashMap();
@@ -104,27 +104,27 @@ public class SchemaChangeJobV2 extends AlterJobV2 {
private Map<Long, String> indexIdToName = Maps.newHashMap();
// shadow index id -> index schema
@SerializedName(value = "indexSchemaMap")
- private Map<Long, List<Column>> indexSchemaMap = Maps.newHashMap();
+ protected Map<Long, List<Column>> indexSchemaMap = Maps.newHashMap();
// shadow index id -> (shadow index schema version : schema hash)
@SerializedName(value = "indexSchemaVersionAndHashMap")
- private Map<Long, SchemaVersionAndHash> indexSchemaVersionAndHashMap = Maps.newHashMap();
+ protected Map<Long, SchemaVersionAndHash> indexSchemaVersionAndHashMap = Maps.newHashMap();
// shadow index id -> shadow index short key count
@SerializedName(value = "indexShortKeyMap")
- private Map<Long, Short> indexShortKeyMap = Maps.newHashMap();
+ protected Map<Long, Short> indexShortKeyMap = Maps.newHashMap();
// bloom filter info
@SerializedName(value = "hasBfChange")
private boolean hasBfChange;
@SerializedName(value = "bfColumns")
- private Set<String> bfColumns = null;
+ protected Set<String> bfColumns = null;
@SerializedName(value = "bfFpp")
- private double bfFpp = 0;
+ protected double bfFpp = 0;
// alter index info
@SerializedName(value = "indexChange")
private boolean indexChange = false;
@SerializedName(value = "indexes")
- private List<Index> indexes = null;
+ protected List<Index> indexes = null;
@SerializedName(value = "storageFormat")
private TStorageFormat storageFormat = TStorageFormat.DEFAULT;
@@ -134,11 +134,13 @@ public class SchemaChangeJobV2 extends AlterJobV2 {
// save failed task after retry three times, tabletId -> agentTask
private Map<Long, List<AgentTask>> failedAgentTasks = Maps.newHashMap();
- private SchemaChangeJobV2() {
+ protected SchemaChangeJobV2() {
super(JobType.SCHEMA_CHANGE);
}
- public SchemaChangeJobV2(String rawSql, long jobId, long dbId, long tableId, String tableName, long timeoutMs) {
+ // Don't call it directly, use AlterJobV2Factory to replace
+ public SchemaChangeJobV2(String rawSql, long jobId, long dbId, long tableId, String tableName,
+ long timeoutMs) {
super(rawSql, jobId, JobType.SCHEMA_CHANGE, dbId, tableId, tableName, timeoutMs);
}
@@ -196,17 +198,7 @@ public class SchemaChangeJobV2 extends AlterJobV2 {
partitionOriginIndexIdMap.clear();
}
- /**
- * runPendingJob():
- * 1. Create all replicas of all shadow indexes and wait them finished.
- * 2. After creating done, add the shadow indexes to catalog, user can not see this
- * shadow index, but internal load process will generate data for these indexes.
- * 3. Get a new transaction id, then set job's state to WAITING_TXN
- */
- @Override
- protected void runPendingJob() throws Exception {
- Preconditions.checkState(jobState == JobState.PENDING, jobState);
- LOG.info("begin to send create replica tasks. job: {}", jobId);
+ protected void createShadowIndexReplica() throws AlterCancelException {
Database db = Env.getCurrentInternalCatalog()
.getDbOrException(dbId, s -> new AlterCancelException("Database " + s + " does not exist"));
@@ -298,7 +290,6 @@ public class SchemaChangeJobV2 extends AlterJobV2 {
} finally {
tbl.readUnlock();
}
-
if (!FeConstants.runningUnitTest) {
// send all tasks and wait them finished
AgentTaskQueue.addBatchTask(batchTask);
@@ -322,7 +313,7 @@ public class SchemaChangeJobV2 extends AlterJobV2 {
} else {
// only show at most 3 results
List<String> subList = countDownLatch.getLeftMarks().stream().limit(3)
- .map(item -> "(backendId = " + item.getKey() + ", tabletId = " + item.getValue() + ")")
+ .map(item -> "(backendId = " + item.getKey() + ", tabletId = " + item.getValue() + ")")
.collect(Collectors.toList());
errMsg = "Error replicas:" + Joiner.on(", ").join(subList);
}
@@ -340,6 +331,27 @@ public class SchemaChangeJobV2 extends AlterJobV2 {
} finally {
tbl.writeUnlock();
}
+ }
+
+ /**
+ * runPendingJob():
+ * 1. Create all replicas of all shadow indexes and wait them finished.
+ * 2. After creating done, add the shadow indexes to catalog, user can not see this
+ * shadow index, but internal load process will generate data for these indexes.
+ * 3. Get a new transaction id, then set job's state to WAITING_TXN
+ */
+ @Override
+ protected void runPendingJob() throws Exception {
+ Preconditions.checkState(jobState == JobState.PENDING, jobState);
+ LOG.info("begin to send create replica tasks. job: {}", jobId);
+ Database db = Env.getCurrentInternalCatalog()
+ .getDbOrException(dbId, s -> new AlterCancelException("Database " + s + " does not exist"));
+
+ if (!checkTableStable(db)) {
+ return;
+ }
+
+ createShadowIndexReplica();
this.watershedTxnId = Env.getCurrentGlobalTransactionMgr().getNextTransactionId();
this.jobState = JobState.WAITING_TXN;
@@ -350,7 +362,7 @@ public class SchemaChangeJobV2 extends AlterJobV2 {
jobId, this.jobState, watershedTxnId);
}
- private void addShadowIndexToCatalog(OlapTable tbl) {
+ protected void addShadowIndexToCatalog(OlapTable tbl) {
for (long partitionId : partitionIndexMap.rowKeySet()) {
Partition partition = tbl.getPartition(partitionId);
if (partition == null) {
@@ -407,6 +419,7 @@ public class SchemaChangeJobV2 extends AlterJobV2 {
tbl.readLock();
Map<Object, List<TColumn>> tcloumnsPool = Maps.newHashMap();
try {
+ long expiration = (createTimeMs + timeoutMs) / 1000;
Map<String, Column> indexColumnMap = Maps.newHashMap();
for (Map.Entry<Long, List<Column>> entry : indexSchemaMap.entrySet()) {
for (Column column : entry.getValue()) {
@@ -469,7 +482,7 @@ public class SchemaChangeJobV2 extends AlterJobV2 {
tableId, partitionId, shadowIdxId, originIdxId, shadowTabletId, originTabletId,
shadowReplica.getId(), shadowSchemaHash, originSchemaHash, visibleVersion, jobId,
JobType.SCHEMA_CHANGE, defineExprs, descTable, originSchemaColumns, tcloumnsPool,
- null);
+ null, expiration);
schemaChangeBatchTask.addTask(rollupTask);
}
}
@@ -514,6 +527,7 @@ public class SchemaChangeJobV2 extends AlterJobV2 {
if (!schemaChangeBatchTask.isFinished()) {
LOG.info("schema change tasks not finished. job: {}", jobId);
List<AgentTask> tasks = schemaChangeBatchTask.getUnfinishedTasks(2000);
+ ensureCloudClusterExist(tasks);
for (AgentTask task : tasks) {
if (task.getFailedTimes() > 0) {
task.setFinished(true);
@@ -575,14 +589,16 @@ public class SchemaChangeJobV2 extends AlterJobV2 {
}
if (healthyReplicaNum < expectReplicationNum / 2 + 1) {
- LOG.warn("shadow tablet {} has few healthy replicas: {}, schema change job: {}",
- shadowTablet.getId(), replicas, jobId);
+ LOG.warn("shadow tablet {} has few healthy replicas: {}, schema change job: {}"
+ + " healthyReplicaNum {} expectReplicationNum {}",
+ shadowTablet.getId(), replicas, jobId, healthyReplicaNum, expectReplicationNum);
throw new AlterCancelException(
"shadow tablet " + shadowTablet.getId() + " has few healthy replicas");
}
} // end for tablets
}
} // end for partitions
+ commitShadowIndex();
// all partitions are good
onFinished(tbl);
} finally {
@@ -598,6 +614,7 @@ public class SchemaChangeJobV2 extends AlterJobV2 {
changeTableState(dbId, tableId, OlapTableState.NORMAL);
LOG.info("set table's state to NORMAL, table id: {}, job id: {}", tableId, jobId);
+ postProcessOriginIndex();
}
private void onFinished(OlapTable tbl) {
@@ -704,6 +721,7 @@ public class SchemaChangeJobV2 extends AlterJobV2 {
changeTableState(dbId, tableId, OlapTableState.NORMAL);
LOG.info("set table's state to NORMAL when cancel, table id: {}, job id: {}", tableId, jobId);
+ postProcessShadowIndex();
return true;
}
@@ -827,6 +845,7 @@ public class SchemaChangeJobV2 extends AlterJobV2 {
}
}
+ postProcessOriginIndex();
jobState = JobState.FINISHED;
this.finishedTimeMs = replayedJob.finishedTimeMs;
LOG.info("replay finished schema change job: {} table id: {}", jobId, tableId);
@@ -839,6 +858,8 @@ public class SchemaChangeJobV2 extends AlterJobV2 {
*/
private void replayCancelled(SchemaChangeJobV2 replayedJob) {
cancelInternal();
+ // try best to drop shadow index
+ postProcessShadowIndex();
this.jobState = JobState.CANCELLED;
this.finishedTimeMs = replayedJob.finishedTimeMs;
this.errMsg = replayedJob.errMsg;
@@ -937,6 +958,15 @@ public class SchemaChangeJobV2 extends AlterJobV2 {
}
}
+ // commit shadowIndex after the job is done in cloud mode
+ protected void commitShadowIndex() throws AlterCancelException {}
+
+ // try best to drop shadow index, when job is cancelled in cloud mode
+ protected void postProcessShadowIndex() {}
+
+ // try best to drop origin index in cloud mode
+ protected void postProcessOriginIndex() {}
+
@Override
public void write(DataOutput out) throws IOException {
String json = GsonUtils.GSON.toJson(this, AlterJobV2.class);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
index 16cb0bf7b0d..d5bcd324d9b 100755
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
@@ -1976,7 +1976,8 @@ public class Env {
return checksum;
}
- public long loadAlterJob(DataInputStream dis, long checksum) throws IOException {
+ public long loadAlterJob(DataInputStream dis, long checksum)
+ throws IOException, AnalysisException {
long newChecksum = checksum;
for (JobType type : JobType.values()) {
newChecksum = loadAlterJob(dis, newChecksum, type);
@@ -1985,7 +1986,8 @@ public class Env {
return newChecksum;
}
- public long loadAlterJob(DataInputStream dis, long checksum, JobType type) throws IOException {
+ public long loadAlterJob(DataInputStream dis, long checksum, JobType type)
+ throws IOException, AnalysisException {
// alter jobs
int size = dis.readInt();
long newChecksum = checksum ^ size;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/EnvFactory.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/EnvFactory.java
index f8a73f963d9..a286cf33cea 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/EnvFactory.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/EnvFactory.java
@@ -92,6 +92,10 @@ public class EnvFactory {
return new Replica();
}
+ public Replica createReplica(Replica.ReplicaContext context) {
+ return new Replica(context);
+ }
+
public ReplicaAllocation createDefReplicaAllocation() {
return new ReplicaAllocation((short) 3);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Replica.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Replica.java
index e20fa06daf5..762ce95839c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Replica.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Replica.java
@@ -74,6 +74,19 @@ public class Replica implements Writable {
DROP, // user force drop replica on this backend
}
+ public static class ReplicaContext {
+ public long replicaId;
+ public long backendId;
+ public ReplicaState state;
+ public long version;
+ public int schemaHash;
+ public long dbId;
+ public long tableId;
+ public long partitionId;
+ public long indexId;
+ public Replica originReplica;
+ }
+
@SerializedName(value = "id")
private long id;
@SerializedName(value = "backendId")
@@ -165,6 +178,10 @@ public class Replica implements Writable {
public Replica() {
}
+ public Replica(ReplicaContext context) {
+ this(context.replicaId, context.backendId, context.state, context.version, context.schemaHash);
+ }
+
// for rollup
// the new replica's version is -1 and last failed version is -1
public Replica(long replicaId, long backendId, int schemaHash, ReplicaState state) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudEnvFactory.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudEnvFactory.java
index 9e9934db9cb..8990f2cd966 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudEnvFactory.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudEnvFactory.java
@@ -103,6 +103,11 @@ public class CloudEnvFactory extends EnvFactory {
return new CloudReplica();
}
+ @Override
+ public Replica createReplica(Replica.ReplicaContext context) {
+ return new CloudReplica(context);
+ }
+
@Override
public ReplicaAllocation createDefReplicaAllocation() {
return new ReplicaAllocation((short) 1);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudReplica.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudReplica.java
index 57ef2b303cc..dbd340ee17a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudReplica.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudReplica.java
@@ -20,6 +20,7 @@ package org.apache.doris.cloud.catalog;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.Partition;
import org.apache.doris.catalog.Replica;
+import org.apache.doris.catalog.Replica.ReplicaContext;
import org.apache.doris.cloud.system.CloudSystemInfoService;
import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
@@ -67,7 +68,14 @@ public class CloudReplica extends Replica {
public CloudReplica() {
}
- public CloudReplica(long replicaId, List<Long> backendIds, ReplicaState state, long version, int schemaHash,
+ public CloudReplica(ReplicaContext context) {
+ this(context.replicaId, context.backendId, context.state, context.version,
+ context.schemaHash, context.dbId, context.tableId, context.partitionId,
+ context.indexId,
+ context.originReplica != null ? ((CloudReplica) context.originReplica).getIdx() : -1);
+ }
+
+ public CloudReplica(long replicaId, Long backendId, ReplicaState state, long version, int schemaHash,
long dbId, long tableId, long partitionId, long indexId, long idx) {
super(replicaId, -1, state, version, schemaHash);
this.dbId = dbId;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
index 29344a561c5..b60ffb5467a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
@@ -161,7 +161,7 @@ public class CloudInternalCatalog extends InternalCatalog {
return partition;
}
- private OlapFile.TabletMetaCloudPB.Builder createTabletMetaBuilder(long tableId, long indexId,
+ public OlapFile.TabletMetaCloudPB.Builder createTabletMetaBuilder(long tableId, long indexId,
long partitionId, Tablet tablet, TTabletType tabletType, int schemaHash, KeysType keysType,
short shortKeyColumnCount, Set<String> bfColumns, double bfFpp, List<Index> indexes,
List<Column> schemaColumns, DataSortInfo dataSortInfo, TCompressionType compressionType,
@@ -333,7 +333,7 @@ public class CloudInternalCatalog extends InternalCatalog {
protected void beforeCreatePartitions(long dbId, long tableId, List<Long> partitionIds, List<Long> indexIds)
throws DdlException {
if (partitionIds == null) {
- prepareMaterializedIndex(tableId, indexIds);
+ prepareMaterializedIndex(tableId, indexIds, 0);
} else {
preparePartition(dbId, tableId, partitionIds, indexIds);
}
@@ -416,12 +416,13 @@ public class CloudInternalCatalog extends InternalCatalog {
}
}
- private void prepareMaterializedIndex(Long tableId, List<Long> indexIds) throws DdlException {
+ // if `expiration` = 0, recycler will delete uncommitted indexes in `retention_seconds`
+ public void prepareMaterializedIndex(Long tableId, List<Long> indexIds, long expiration) throws DdlException {
Cloud.IndexRequest.Builder indexRequestBuilder = Cloud.IndexRequest.newBuilder();
indexRequestBuilder.setCloudUniqueId(Config.cloud_unique_id);
indexRequestBuilder.addAllIndexIds(indexIds);
indexRequestBuilder.setTableId(tableId);
- indexRequestBuilder.setExpiration(0);
+ indexRequestBuilder.setExpiration(expiration);
final Cloud.IndexRequest indexRequest = indexRequestBuilder.build();
Cloud.IndexResponse response = null;
@@ -447,7 +448,7 @@ public class CloudInternalCatalog extends InternalCatalog {
}
}
- private void commitMaterializedIndex(Long tableId, List<Long> indexIds) throws DdlException {
+ public void commitMaterializedIndex(Long tableId, List<Long> indexIds) throws DdlException {
Cloud.IndexRequest.Builder indexRequestBuilder = Cloud.IndexRequest.newBuilder();
indexRequestBuilder.setCloudUniqueId(Config.cloud_unique_id);
indexRequestBuilder.addAllIndexIds(indexIds);
@@ -477,7 +478,7 @@ public class CloudInternalCatalog extends InternalCatalog {
}
}
- private void sendCreateTabletsRpc(Cloud.CreateTabletsRequest.Builder requestBuilder) throws DdlException {
+ public void sendCreateTabletsRpc(Cloud.CreateTabletsRequest.Builder requestBuilder) throws DdlException {
requestBuilder.setCloudUniqueId(Config.cloud_unique_id);
Cloud.CreateTabletsRequest createTabletsReq = requestBuilder.build();
@@ -624,7 +625,7 @@ public class CloudInternalCatalog extends InternalCatalog {
}
}
- private void dropMaterializedIndex(Long tableId, List<Long> indexIds) throws DdlException {
+ public void dropMaterializedIndex(Long tableId, List<Long> indexIds) throws DdlException {
Cloud.IndexRequest.Builder indexRequestBuilder = Cloud.IndexRequest.newBuilder();
indexRequestBuilder.setCloudUniqueId(Config.cloud_unique_id);
indexRequestBuilder.addAllIndexIds(indexIds);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java b/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java
index 43d1b907247..6921cebd579 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java
@@ -18,6 +18,8 @@
package org.apache.doris.persist.gson;
import org.apache.doris.alter.AlterJobV2;
+import org.apache.doris.alter.CloudRollupJobV2;
+import org.apache.doris.alter.CloudSchemaChangeJobV2;
import org.apache.doris.alter.RollupJobV2;
import org.apache.doris.alter.SchemaChangeJobV2;
import org.apache.doris.catalog.AggStateType;
@@ -198,7 +200,9 @@ public class GsonUtils {
private static RuntimeTypeAdapterFactory<AlterJobV2> alterJobV2TypeAdapterFactory = RuntimeTypeAdapterFactory
.of(AlterJobV2.class, "clazz")
.registerSubtype(RollupJobV2.class, RollupJobV2.class.getSimpleName())
- .registerSubtype(SchemaChangeJobV2.class, SchemaChangeJobV2.class.getSimpleName());
+ .registerSubtype(SchemaChangeJobV2.class, SchemaChangeJobV2.class.getSimpleName())
+ .registerSubtype(CloudSchemaChangeJobV2.class, CloudSchemaChangeJobV2.class.getSimpleName())
+ .registerSubtype(CloudRollupJobV2.class, CloudRollupJobV2.class.getSimpleName());
// runtime adapter for class "SyncJob"
private static RuntimeTypeAdapterFactory<SyncJob> syncJobTypeAdapterFactory = RuntimeTypeAdapterFactory
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index a03d6246ad9..6c0fecf0d34 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -1429,7 +1429,7 @@ public class SessionVariable implements Serializable, Writable {
public boolean truncateCharOrVarcharColumns = false;
@VariableMgr.VarAttr(name = ENABLE_MEMTABLE_ON_SINK_NODE, needForward = true)
- public boolean enableMemtableOnSinkNode = true;
+ public boolean enableMemtableOnSinkNode = false;
@VariableMgr.VarAttr(name = LOAD_STREAM_PER_NODE)
public int loadStreamPerNode = 2;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/task/AlterReplicaTask.java b/fe/fe-core/src/main/java/org/apache/doris/task/AlterReplicaTask.java
index 267a99a12b9..1c57fc5d096 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/task/AlterReplicaTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/task/AlterReplicaTask.java
@@ -56,6 +56,8 @@ public class AlterReplicaTask extends AgentTask {
private Map<Object, List<TColumn>> tcloumnsPool;
private List<Column> baseSchemaColumns;
+ private long expiration;
+
/**
* AlterReplicaTask constructor.
*
@@ -64,7 +66,7 @@ public class AlterReplicaTask extends AgentTask {
long baseIndexId, long rollupTabletId, long baseTabletId, long newReplicaId, int newSchemaHash,
int baseSchemaHash, long version, long jobId, AlterJobV2.JobType jobType, Map<String, Expr> defineExprs,
DescriptorTable descTable, List<Column> baseSchemaColumns, Map<Object, List<TColumn>> tcloumnsPool,
- Expr whereClause) {
+ Expr whereClause, long expiration) {
super(null, backendId, TTaskType.ALTER, dbId, tableId, partitionId, rollupIndexId, rollupTabletId);
this.baseTabletId = baseTabletId;
@@ -82,6 +84,7 @@ public class AlterReplicaTask extends AgentTask {
this.descTable = descTable;
this.baseSchemaColumns = baseSchemaColumns;
this.tcloumnsPool = tcloumnsPool;
+ this.expiration = expiration;
}
public long getBaseTabletId() {
@@ -115,6 +118,8 @@ public class AlterReplicaTask extends AgentTask {
public TAlterTabletReqV2 toThrift() {
TAlterTabletReqV2 req = new TAlterTabletReqV2(baseTabletId, signature, baseSchemaHash, newSchemaHash);
req.setAlterVersion(version);
+ req.setJobId(jobId);
+ req.setExpiration(expiration);
req.setBeExecVersion(Config.be_exec_version);
switch (jobType) {
case ROLLUP:
diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/RollupJobV2Test.java b/fe/fe-core/src/test/java/org/apache/doris/alter/RollupJobV2Test.java
index d4cdadf53bd..2a47520f5c5 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/alter/RollupJobV2Test.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/alter/RollupJobV2Test.java
@@ -327,7 +327,8 @@ public class RollupJobV2Test {
Column column = new Column(mvColumnName, Type.BITMAP, false, AggregateType.BITMAP_UNION, false, "1", "");
columns.add(column);
- RollupJobV2 rollupJobV2 = new RollupJobV2("", 1, 1, 1, "test", 1, 1, 1, "test", "rollup", columns, null, 1, 1,
+ RollupJobV2 rollupJobV2 = AlterJobV2Factory.createRollupJobV2(
+ "", 1, 1, 1, "test", 1, 1, 1, "test", "rollup", columns, null, 1, 1,
KeysType.AGG_KEYS, keysCount,
new OriginStatement("create materialized view rollup as select bitmap_union(to_bitmap(c1)) from test",
0));
diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeJobV2Test.java b/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeJobV2Test.java
index 51b3bd3a8aa..5e99599b041 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeJobV2Test.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeJobV2Test.java
@@ -399,7 +399,8 @@ public class SchemaChangeJobV2Test {
file.deleteOnExit();
DataOutputStream out = new DataOutputStream(new FileOutputStream(file));
- SchemaChangeJobV2 schemaChangeJobV2 = new SchemaChangeJobV2("", 1, 1, 1, "test", 600000);
+ SchemaChangeJobV2 schemaChangeJobV2 = AlterJobV2Factory.createSchemaChangeJobV2(
+ "", 1, 1, 1, "test", 600000);
schemaChangeJobV2.setStorageFormat(TStorageFormat.V2);
Deencapsulation.setField(schemaChangeJobV2, "jobState", AlterJobV2.JobState.FINISHED);
Map<Long, SchemaVersionAndHash> indexSchemaVersionAndHashMap = Maps.newHashMap();
diff --git a/regression-test/pipeline/cloud_p0/conf/regression-conf-custom.groovy b/regression-test/pipeline/cloud_p0/conf/regression-conf-custom.groovy
index 8637ef11033..0af6e798f1e 100644
--- a/regression-test/pipeline/cloud_p0/conf/regression-conf-custom.groovy
+++ b/regression-test/pipeline/cloud_p0/conf/regression-conf-custom.groovy
@@ -16,7 +16,7 @@
// under the License.
testGroups = "p0"
-testDirectories = "ddl_p0,database_p0,load,load_p0,query_p0,table_p0,account_p0,autobucket,bitmap_functions,bloom_filter_p0,cast_decimal_to_boolean,cast_double_to_decimal,compression_p0,connector_p0,correctness,correctness_p0,csv_header_p0,data_model_p0,database_p0,datatype_p0,delete_p0,demo_p0,empty_relation,export_p0,external_table_p0,fault_injection_p0,flink_connector_p0,insert_overwrite_p0,insert_p0,internal_schema_p0,javaudf_p0,job_p0,json_p0,jsonb_p0,meta_action_p0,metrics_p0,mtmv_ [...]
+testDirectories = "ddl_p0,database_p0,load,load_p0,query_p0,table_p0,account_p0,autobucket,bitmap_functions,bloom_filter_p0,cast_decimal_to_boolean,cast_double_to_decimal,compression_p0,connector_p0,correctness,correctness_p0,csv_header_p0,data_model_p0,database_p0,datatype_p0,delete_p0,demo_p0,empty_relation,export_p0,external_table_p0,fault_injection_p0,flink_connector_p0,insert_overwrite_p0,insert_p0,internal_schema_p0,javaudf_p0,job_p0,json_p0,jsonb_p0,meta_action_p0,metrics_p0,mtmv_ [...]
//exclude groups and exclude suites is more prior than include groups and include suites.
excludeSuites = "test_index_failure_injection,test_dump_image,test_profile,test_spark_load,test_refresh_mtmv,test_bitmap_filter,test_information_schema_external"
excludeDirectories = "workload_manager_p1,nereids_rules_p0/subquery"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org