You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by ai...@apache.org on 2026/01/12 12:42:22 UTC
(doris) branch feature-ann-index-insert-without-index created (now c31eee913f3)
This is an automated email from the ASF dual-hosted git repository.
airborne pushed a change to branch feature-ann-index-insert-without-index
in repository https://gitbox.apache.org/repos/asf/doris.git
at c31eee913f3 [feature](ann-index) Support skip_write_index_on_load for ANN index
This branch includes the following new commits:
new c31eee913f3 [feature](ann-index) Support skip_write_index_on_load for ANN index
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 01/01: [feature](ann-index) Support skip_write_index_on_load for ANN index
Posted by ai...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
airborne pushed a commit to branch feature-ann-index-insert-without-index
in repository https://gitbox.apache.org/repos/asf/doris.git
commit c31eee913f3a6a9b8c1ee884b448cef7ead1086b
Author: airborne12 <ji...@selectdb.com>
AuthorDate: Mon Jan 12 20:40:35 2026 +0800
[feature](ann-index) Support skip_write_index_on_load for ANN index
Enable skip_write_index_on_load property for ANN indexes to skip index
construction during data loading and defer it to compaction time. This
improves data loading performance for tables with ANN indexes.
Key changes:
- FE: Remove restriction that blocked skip_write_index_on_load=true
- BE: Only skip ANN index during TYPE_DIRECT writes, not inverted index
- BE: Handle graceful fallback to brute force when ANN index is missing
- BE: Return INVERTED_INDEX_BYPASS when ANN index file is empty
When skip_write_index_on_load=true:
- ANN index is not built during INSERT operations
- Query falls back to brute force search (l2_distance_approximate == l2_distance)
- ANN index is built during compaction
- After compaction, ANN index is available for approximate search
Co-Authored-By: Claude Opus 4.5 <no...@anthropic.com>
---
.../rowset/segment_v2/ann_index/ann_index_reader.cpp | 11 +++++++++--
be/src/olap/rowset/segment_v2/segment.cpp | 19 +++++++++++++++++--
be/src/olap/rowset/segment_v2/segment_iterator.cpp | 15 ++++++++++++---
be/src/olap/rowset/segment_v2/segment_writer.cpp | 18 +++++++++---------
.../rowset/segment_v2/vertical_segment_writer.cpp | 20 ++++++++++----------
.../apache/doris/common/util/PropertyAnalyzer.java | 3 +--
.../plans/commands/info/ModifyTablePropertiesOp.java | 5 -----
7 files changed, 58 insertions(+), 33 deletions(-)
diff --git a/be/src/olap/rowset/segment_v2/ann_index/ann_index_reader.cpp b/be/src/olap/rowset/segment_v2/ann_index/ann_index_reader.cpp
index ffdd9bf58e7..ba8e0934130 100644
--- a/be/src/olap/rowset/segment_v2/ann_index/ann_index_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/ann_index/ann_index_reader.cpp
@@ -74,8 +74,15 @@ Status AnnIndexReader::load_index(io::IOContext* io_ctx) {
DorisMetrics::instance()->ann_index_load_cnt->increment(1);
try {
- RETURN_IF_ERROR(
- _index_file_reader->init(config::inverted_index_read_buffer_size, io_ctx));
+ auto st = _index_file_reader->init(config::inverted_index_read_buffer_size, io_ctx);
+ // If index file is empty or not found, return INVERTED_INDEX_BYPASS
+ // This can happen when skip_write_index_on_load=true skips ANN index construction
+ if (st.is<ErrorCode::INVERTED_INDEX_BYPASS>() ||
+ st.is<ErrorCode::INVERTED_INDEX_FILE_NOT_FOUND>()) {
+ return Status::Error<ErrorCode::INVERTED_INDEX_BYPASS>(
+ "ANN index file is empty or not found, will fallback to brute force");
+ }
+ RETURN_IF_ERROR(st);
Result<std::unique_ptr<DorisCompoundReader, DirectoryDeleter>> compound_dir;
compound_dir = _index_file_reader->open(&_index_meta, io_ctx);
if (!compound_dir.has_value()) {
diff --git a/be/src/olap/rowset/segment_v2/segment.cpp b/be/src/olap/rowset/segment_v2/segment.cpp
index 561735b49b1..36a509dc96e 100644
--- a/be/src/olap/rowset/segment_v2/segment.cpp
+++ b/be/src/olap/rowset/segment_v2/segment.cpp
@@ -777,9 +777,24 @@ Status Segment::new_index_iterator(const TabletColumn& tablet_column, const Tabl
if (index_meta) {
// call DorisCallOnce.call without check if _index_file_reader is nullptr
// to avoid data race during parallel method calls
- RETURN_IF_ERROR(_index_file_reader_open.call([&] { return _open_index_file_reader(); }));
+ st = _index_file_reader_open.call([&] { return _open_index_file_reader(); });
+ // If index file is empty or not found for ANN index, return OK to fallback to brute force
+ // This can happen when skip_write_index_on_load=true skips index during data loading
+ // but compaction hasn't built the index yet
+ bool is_ann_index = index_meta->index_type() == IndexType::ANN;
+ if (is_ann_index && (st.is<ErrorCode::INVERTED_INDEX_BYPASS>() ||
+ st.is<ErrorCode::INVERTED_INDEX_FILE_NOT_FOUND>())) {
+ return Status::OK();
+ }
+ RETURN_IF_ERROR(st);
// after DorisCallOnce.call, _index_file_reader is guaranteed to be not nullptr
- RETURN_IF_ERROR(reader->new_index_iterator(_index_file_reader, index_meta, iter));
+ st = reader->new_index_iterator(_index_file_reader, index_meta, iter);
+ // If loading index fails due to empty/missing data for ANN index
+ if (is_ann_index && (st.is<ErrorCode::INVERTED_INDEX_BYPASS>() ||
+ st.is<ErrorCode::INVERTED_INDEX_FILE_NOT_FOUND>())) {
+ return Status::OK();
+ }
+ RETURN_IF_ERROR(st);
return Status::OK();
}
return Status::OK();
diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
index 3e52a9815ab..40ac355882a 100644
--- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
@@ -810,9 +810,18 @@ Status SegmentIterator::_apply_ann_topn_predicate() {
vectorized::IColumn::MutablePtr result_column;
std::unique_ptr<std::vector<uint64_t>> result_row_ids;
segment_v2::AnnIndexStats ann_index_stats;
- RETURN_IF_ERROR(_ann_topn_runtime->evaluate_vector_ann_search(ann_index_iterator, &_row_bitmap,
- rows_of_segment, result_column,
- result_row_ids, ann_index_stats));
+ auto st = _ann_topn_runtime->evaluate_vector_ann_search(ann_index_iterator, &_row_bitmap,
+ rows_of_segment, result_column,
+ result_row_ids, ann_index_stats);
+ // If ANN index is not available (empty or missing), fallback to brute force search
+ // This can happen when skip_write_index_on_load=true skips index during data loading
+ if (st.is<ErrorCode::INVERTED_INDEX_BYPASS>() ||
+ st.is<ErrorCode::INVERTED_INDEX_FILE_NOT_FOUND>()) {
+ VLOG_DEBUG << "ANN index not available, fallback to brute force search: " << st.to_string();
+ _need_read_data_indices[src_cid] = true;
+ return Status::OK();
+ }
+ RETURN_IF_ERROR(st);
VLOG_DEBUG << fmt::format("Ann topn filtered {} - {} = {} rows", pre_size,
_row_bitmap.cardinality(), pre_size - _row_bitmap.cardinality());
diff --git a/be/src/olap/rowset/segment_v2/segment_writer.cpp b/be/src/olap/rowset/segment_v2/segment_writer.cpp
index 99efbb8acac..554ee47ecc0 100644
--- a/be/src/olap/rowset/segment_v2/segment_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_writer.cpp
@@ -221,10 +221,6 @@ Status SegmentWriter::_create_column_writer(uint32_t cid, const TabletColumn& co
skip_inverted_index =
_opts.rowset_ctx->columns_to_do_index_compaction.count(column.unique_id()) > 0;
}
- // skip write inverted index on load if skip_write_index_on_load is true
- if (_opts.write_type == DataWriteType::TYPE_DIRECT && schema->skip_write_index_on_load()) {
- skip_inverted_index = true;
- }
// indexes for this column
if (!skip_inverted_index) {
auto inverted_indexs = schema->inverted_indexs(column);
@@ -234,11 +230,15 @@ Status SegmentWriter::_create_column_writer(uint32_t cid, const TabletColumn& co
DCHECK(_index_file_writer != nullptr);
}
}
- // indexes for this column
- if (const auto& index = schema->ann_index(column); index != nullptr) {
- opts.ann_index = index;
- opts.need_ann_index = true;
- DCHECK(_index_file_writer != nullptr);
+ // skip write ann index on load if skip_write_index_on_load is true
+ bool skip_ann_index =
+ _opts.write_type == DataWriteType::TYPE_DIRECT && schema->skip_write_index_on_load();
+ if (!skip_ann_index) {
+ if (const auto& index = schema->ann_index(column); index != nullptr) {
+ opts.ann_index = index;
+ opts.need_ann_index = true;
+ DCHECK(_index_file_writer != nullptr);
+ }
}
opts.index_file_writer = _index_file_writer;
diff --git a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
index c50a03cb845..0e443747aac 100644
--- a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
@@ -220,11 +220,6 @@ Status VerticalSegmentWriter::_create_column_writer(uint32_t cid, const TabletCo
skip_inverted_index =
_opts.rowset_ctx->columns_to_do_index_compaction.contains(column.unique_id());
}
- // skip write inverted index on load if skip_write_index_on_load is true
- if (_opts.write_type == DataWriteType::TYPE_DIRECT &&
- tablet_schema->skip_write_index_on_load()) {
- skip_inverted_index = true;
- }
if (!skip_inverted_index) {
auto inverted_indexs = tablet_schema->inverted_indexs(column);
if (!inverted_indexs.empty()) {
@@ -235,11 +230,16 @@ Status VerticalSegmentWriter::_create_column_writer(uint32_t cid, const TabletCo
}
opts.index_file_writer = _index_file_writer;
- if (const auto& index = tablet_schema->ann_index(column); index != nullptr) {
- opts.ann_index = index;
- opts.need_ann_index = true;
- DCHECK(_index_file_writer != nullptr);
- opts.index_file_writer = _index_file_writer;
+ // skip write ann index on load if skip_write_index_on_load is true
+ bool skip_ann_index = _opts.write_type == DataWriteType::TYPE_DIRECT &&
+ tablet_schema->skip_write_index_on_load();
+ if (!skip_ann_index) {
+ if (const auto& index = tablet_schema->ann_index(column); index != nullptr) {
+ opts.ann_index = index;
+ opts.need_ann_index = true;
+ DCHECK(_index_file_writer != nullptr);
+ opts.index_file_writer = _index_file_writer;
+ }
}
#define DISABLE_INDEX_IF_FIELD_TYPE(TYPE, type_name) \
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
index 655c0fa92e9..524296bc796 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
@@ -932,8 +932,7 @@ public class PropertyAnalyzer {
}
properties.remove(PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD);
if (value.equalsIgnoreCase("true")) {
- throw new AnalysisException("Property " + PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD
- + " is forbidden now.");
+ return true;
} else if (value.equalsIgnoreCase("false")) {
return false;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyTablePropertiesOp.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyTablePropertiesOp.java
index dc55d99d249..f2e53ea4a9c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyTablePropertiesOp.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyTablePropertiesOp.java
@@ -248,11 +248,6 @@ public class ModifyTablePropertiesOp extends AlterTableOp {
this.needTableStable = false;
this.opType = AlterOpType.MODIFY_TABLE_PROPERTY_SYNC;
} else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD)) {
- if (properties.get(PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD).equalsIgnoreCase("true")) {
- throw new AnalysisException(
- "Property "
- + PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD + " is forbidden now");
- }
if (!properties.get(PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD).equalsIgnoreCase("true")
&& !properties.get(PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD)
.equalsIgnoreCase("false")) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org