You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by kx...@apache.org on 2023/06/27 16:50:35 UTC
[doris] 03/05: [fix](inverted index) update output rowset index meta with input rowset when drop inverted index (#21248)
This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit cff0918deb806a82cc3a1f6a683749f90b5ab410
Author: YueW <45...@users.noreply.github.com>
AuthorDate: Tue Jun 27 23:54:35 2023 +0800
[fix](inverted index) update output rowset index meta with input rowset when drop inverted index (#21248)
---
be/src/olap/olap_server.cpp | 5 ++---
be/src/olap/tablet_schema.cpp | 11 +++++++++++
be/src/olap/tablet_schema.h | 1 +
be/src/olap/task/index_builder.cpp | 19 ++++++++++++++++---
be/src/olap/task/index_builder.h | 2 --
.../java/org/apache/doris/alter/IndexChangeJob.java | 2 +-
.../org/apache/doris/task/AlterInvertedIndexTask.java | 7 ++++++-
7 files changed, 37 insertions(+), 10 deletions(-)
diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp
index 57bf40a147..09875b9e52 100644
--- a/be/src/olap/olap_server.cpp
+++ b/be/src/olap/olap_server.cpp
@@ -1021,9 +1021,8 @@ Status StorageEngine::process_index_change_task(const TAlterInvertedIndexReq& re
return Status::InternalError("tablet not exist, tablet_id={}.", tablet_id);
}
- IndexBuilderSharedPtr index_builder =
- std::make_shared<IndexBuilder>(tablet, request.columns, request.indexes_desc,
- request.alter_inverted_indexes, request.is_drop_op);
+ IndexBuilderSharedPtr index_builder = std::make_shared<IndexBuilder>(
+ tablet, request.columns, request.alter_inverted_indexes, request.is_drop_op);
RETURN_IF_ERROR(_handle_index_change(index_builder));
return Status::OK();
}
diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp
index f5f62e6857..414036ab58 100644
--- a/be/src/olap/tablet_schema.cpp
+++ b/be/src/olap/tablet_schema.cpp
@@ -637,6 +637,17 @@ void TabletSchema::append_index(TabletIndex index) {
_indexes.push_back(std::move(index));
}
+void TabletSchema::remove_index(int64_t index_id) {
+ std::vector<TabletIndex> indexes;
+ for (auto index : _indexes) {
+ if (index.index_id() == index_id) {
+ continue;
+ }
+ indexes.emplace_back(std::move(index));
+ }
+ _indexes = std::move(indexes);
+}
+
void TabletSchema::clear_columns() {
_field_name_to_index.clear();
_field_id_to_index.clear();
diff --git a/be/src/olap/tablet_schema.h b/be/src/olap/tablet_schema.h
index ceadf76a0a..5a521f3610 100644
--- a/be/src/olap/tablet_schema.h
+++ b/be/src/olap/tablet_schema.h
@@ -207,6 +207,7 @@ public:
void to_schema_pb(TabletSchemaPB* tablet_meta_pb) const;
void append_column(TabletColumn column, bool is_dropped_column = false);
void append_index(TabletIndex index);
+ void remove_index(int64_t index_id);
// Must make sure the row column is always the last column
void add_row_column();
void copy_from(const TabletSchema& tablet_schema);
diff --git a/be/src/olap/task/index_builder.cpp b/be/src/olap/task/index_builder.cpp
index 05ba099f90..4a82240474 100644
--- a/be/src/olap/task/index_builder.cpp
+++ b/be/src/olap/task/index_builder.cpp
@@ -30,12 +30,10 @@
namespace doris {
IndexBuilder::IndexBuilder(const TabletSharedPtr& tablet, const std::vector<TColumn>& columns,
- const std::vector<TOlapTableIndex> exist_indexes,
const std::vector<doris::TOlapTableIndex>& alter_inverted_indexes,
bool is_drop_op)
: _tablet(tablet),
_columns(columns),
- _exist_indexes(exist_indexes),
_alter_inverted_indexes(alter_inverted_indexes),
_is_drop_op(is_drop_op) {
_olap_data_convertor = std::make_unique<vectorized::OlapBlockDataConvertor>();
@@ -63,8 +61,16 @@ Status IndexBuilder::update_inverted_index_info() {
auto input_rs_tablet_schema = input_rowset->tablet_schema();
output_rs_tablet_schema->copy_from(*input_rs_tablet_schema);
if (_is_drop_op) {
- output_rs_tablet_schema->update_indexes_from_thrift(_exist_indexes);
+ // base on input rowset's tablet_schema to build
+ // output rowset's tablet_schema which only remove
+ // the indexes specified in this drop index request
+ for (auto t_inverted_index : _alter_inverted_indexes) {
+ output_rs_tablet_schema->remove_index(t_inverted_index.index_id);
+ }
} else {
+ // base on input rowset's tablet_schema to build
+ // output rowset's tablet_schema which only add
+ // the indexes specified in this build index request
for (auto t_inverted_index : _alter_inverted_indexes) {
TabletIndex index;
index.init_from_thrift(t_inverted_index, *input_rs_tablet_schema);
@@ -427,6 +433,13 @@ Status IndexBuilder::do_build_inverted_index() {
}
Status IndexBuilder::modify_rowsets(const Merger::Statistics* stats) {
+ for (auto rowset_ptr : _output_rowsets) {
+ auto rowset_id = rowset_ptr->rowset_id();
+ if (StorageEngine::instance()->check_rowset_id_in_unused_rowsets(rowset_id)) {
+ DCHECK(false) << "output rowset: " << rowset_id.to_string() << " in unused rowsets";
+ }
+ }
+
if (_tablet->keys_type() == KeysType::UNIQUE_KEYS &&
_tablet->enable_unique_key_merge_on_write()) {
std::lock_guard<std::mutex> rwlock(_tablet->get_rowset_update_lock());
diff --git a/be/src/olap/task/index_builder.h b/be/src/olap/task/index_builder.h
index 562cb1148d..9e406c22c1 100644
--- a/be/src/olap/task/index_builder.h
+++ b/be/src/olap/task/index_builder.h
@@ -36,7 +36,6 @@ using RowsetWriterUniquePtr = std::unique_ptr<RowsetWriter>;
class IndexBuilder {
public:
IndexBuilder(const TabletSharedPtr& tablet, const std::vector<TColumn>& columns,
- const std::vector<TOlapTableIndex> exist_indexes,
const std::vector<doris::TOlapTableIndex>& alter_inverted_indexes,
bool is_drop_op = false);
~IndexBuilder();
@@ -65,7 +64,6 @@ private:
private:
TabletSharedPtr _tablet;
std::vector<TColumn> _columns;
- std::vector<TOlapTableIndex> _exist_indexes;
std::vector<doris::TOlapTableIndex> _alter_inverted_indexes;
bool _is_drop_op;
std::unordered_map<std::string, std::set<int32_t>> _rowset_alter_index_column_ids;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/IndexChangeJob.java b/fe/fe-core/src/main/java/org/apache/doris/alter/IndexChangeJob.java
index fecfe4b99b..6ba3e68ffa 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/IndexChangeJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/IndexChangeJob.java
@@ -290,7 +290,7 @@ public class IndexChangeJob implements Writable {
partitionId, originIndexId, originTabletId,
originSchemaHash, olapTable.getIndexes(),
alterInvertedIndexes, originSchemaColumns,
- isDropOp, taskSignature);
+ isDropOp, taskSignature, jobId);
invertedIndexBatchTask.addTask(alterInvertedIndexTask);
}
} // end for tablet
diff --git a/fe/fe-core/src/main/java/org/apache/doris/task/AlterInvertedIndexTask.java b/fe/fe-core/src/main/java/org/apache/doris/task/AlterInvertedIndexTask.java
index c199d1d482..caf7733165 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/task/AlterInvertedIndexTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/task/AlterInvertedIndexTask.java
@@ -44,11 +44,13 @@ public class AlterInvertedIndexTask extends AgentTask {
private List<Column> schemaColumns;
private List<Index> existIndexes;
private boolean isDropOp = false;
+ private long jobId;
public AlterInvertedIndexTask(long backendId, long dbId, long tableId,
long partitionId, long indexId, long tabletId, int schemaHash,
List<Index> existIndexes, List<Index> alterInvertedIndexes,
- List<Column> schemaColumns, boolean isDropOp, long taskSignature) {
+ List<Column> schemaColumns, boolean isDropOp, long taskSignature,
+ long jobId) {
super(null, backendId, TTaskType.ALTER_INVERTED_INDEX, dbId, tableId,
partitionId, indexId, tabletId, taskSignature);
this.tabletId = tabletId;
@@ -57,6 +59,7 @@ public class AlterInvertedIndexTask extends AgentTask {
this.alterInvertedIndexes = alterInvertedIndexes;
this.schemaColumns = schemaColumns;
this.isDropOp = isDropOp;
+ this.jobId = jobId;
}
public long getTabletId() {
@@ -94,6 +97,8 @@ public class AlterInvertedIndexTask extends AgentTask {
req.setTabletId(tabletId);
req.setSchemaHash(schemaHash);
req.setIsDropOp(isDropOp);
+ // set jonId for debugging in BE
+ req.setJobId(jobId);
if (!alterInvertedIndexes.isEmpty()) {
List<TOlapTableIndex> tIndexes = new ArrayList<>();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org