You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by ai...@apache.org on 2025/04/07 03:09:22 UTC
(doris) branch vector-index-dev updated: [fetura](vector) Extend the index interface to support vector indexing. (#49780)
This is an automated email from the ASF dual-hosted git repository.
airborne pushed a commit to branch vector-index-dev
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/vector-index-dev by this push:
new b2da241940f [fetura](vector) Extend the index interface to support vector indexing. (#49780)
b2da241940f is described below
commit b2da241940fe5d572fcc42f5e5666a0f8222567e
Author: zzzxl <ya...@selectdb.com>
AuthorDate: Mon Apr 7 11:09:14 2025 +0800
[fetura](vector) Extend the index interface to support vector indexing. (#49780)
### What problem does this PR solve?
Issue Number: close #xxx
Related PR: #xxx
Problem Summary:
### Release note
None
### Check List (For Author)
- Test <!-- At least one of them must be included. -->
- [ ] Regression test
- [ ] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason <!-- Add your reason? -->
- Behavior changed:
- [ ] No.
- [ ] Yes. <!-- Explain the behavior change -->
- Does this need documentation?
- [ ] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---
be/src/common/status.h | 2 +-
be/src/index-tools/ann_tool.cpp | 12 +-
be/src/index-tools/index_tool.cpp | 16 +-
be/src/olap/accept_null_predicate.h | 2 +-
be/src/olap/column_predicate.h | 4 +-
be/src/olap/compaction.cpp | 22 +-
be/src/olap/comparison_predicate.h | 18 +-
be/src/olap/in_list_predicate.h | 14 +-
be/src/olap/match_predicate.cpp | 29 +-
be/src/olap/match_predicate.h | 4 +-
be/src/olap/metadata_adder.h | 6 +-
be/src/olap/null_predicate.cpp | 2 +-
be/src/olap/null_predicate.h | 2 +-
be/src/olap/olap_common.h | 2 +-
be/src/olap/rowset/beta_rowset.cpp | 20 +-
...d_index_compaction.h => ann_index_iterator.cpp} | 39 +-
be/src/olap/rowset/segment_v2/ann_index_iterator.h | 53 ++
...ted_index_compaction.h => ann_index_reader.cpp} | 34 +-
be/src/olap/rowset/segment_v2/ann_index_reader.h | 49 ++
be/src/olap/rowset/segment_v2/ann_index_writer.cpp | 17 +
be/src/olap/rowset/segment_v2/column_reader.cpp | 72 +--
be/src/olap/rowset/segment_v2/column_reader.h | 27 +-
be/src/olap/rowset/segment_v2/index_iterator.h | 58 +++
be/src/olap/rowset/segment_v2/index_reader.h | 58 +++
be/src/olap/rowset/segment_v2/index_writer.cpp | 17 +
.../rowset/segment_v2/inverted_index_compaction.h | 2 +-
.../rowset/segment_v2/inverted_index_iterator.cpp | 109 +++++
.../rowset/segment_v2/inverted_index_iterator.h | 56 +++
.../rowset/segment_v2/inverted_index_reader.cpp | 125 ++---
.../olap/rowset/segment_v2/inverted_index_reader.h | 90 ++--
be/src/olap/rowset/segment_v2/segment.cpp | 24 +-
be/src/olap/rowset/segment_v2/segment.h | 14 +-
be/src/olap/rowset/segment_v2/segment_iterator.cpp | 50 +-
be/src/olap/rowset/segment_v2/segment_iterator.h | 15 +-
...dex_file_reader.cpp => x_index_file_reader.cpp} | 22 +-
...d_index_file_reader.h => x_index_file_reader.h} | 8 +-
.../olap/rowset/segment_v2/x_index_file_writer.cpp | 535 ++-------------------
.../olap/rowset/segment_v2/x_index_file_writer.h | 94 +---
.../rowset/segment_v2/x_index_storage_format.cpp | 119 +++++
...index_compaction.h => x_index_storage_format.h} | 49 +-
.../segment_v2/x_index_storage_format_v1.cpp | 190 ++++++++
.../rowset/segment_v2/x_index_storage_format_v1.h | 45 ++
.../segment_v2/x_index_storage_format_v2.cpp | 246 ++++++++++
.../rowset/segment_v2/x_index_storage_format_v2.h | 56 +++
be/src/olap/shared_predicate.h | 2 +-
be/src/olap/task/index_builder.cpp | 18 +-
be/src/olap/task/index_builder.h | 4 +-
be/src/pipeline/exec/olap_scan_operator.cpp | 4 +-
be/src/pipeline/exec/olap_scan_operator.h | 2 +-
be/src/vec/exec/scan/olap_scanner.cpp | 4 +-
be/src/vec/exprs/vexpr.cpp | 2 +-
be/src/vec/exprs/vexpr_context.h | 19 +-
be/src/vec/functions/array/function_array_index.h | 23 +-
.../vec/functions/array/function_arrays_overlap.h | 20 +-
be/src/vec/functions/function.h | 8 +-
be/src/vec/functions/function_ip.h | 38 +-
be/src/vec/functions/function_multi_match.cpp | 16 +-
be/src/vec/functions/function_multi_match.h | 2 +-
be/src/vec/functions/function_tokenize.cpp | 2 +-
be/src/vec/functions/functions_comparison.h | 22 +-
be/src/vec/functions/in.h | 18 +-
be/src/vec/functions/match.cpp | 31 +-
be/src/vec/functions/match.h | 3 +-
be/src/vector/stream_wrapper.h | 17 +
.../compaction/util/index_compaction_utils.cpp | 51 +-
.../segment_v2/inverted_index_array_test.cpp | 6 +-
.../inverted_index_compound_reader_test.cpp | 6 +-
.../segment_v2/inverted_index_writer_test.cpp | 20 +-
68 files changed, 1649 insertions(+), 1117 deletions(-)
diff --git a/be/src/common/status.h b/be/src/common/status.h
index 841d5090699..dc6ece54d8c 100644
--- a/be/src/common/status.h
+++ b/be/src/common/status.h
@@ -278,7 +278,7 @@ namespace ErrorCode {
E(PIP_WAIT_FOR_SC, -3121, false); \
E(ROWSET_ADD_TO_BINLOG_FAILED, -3122, true); \
E(ROWSET_BINLOG_NOT_ONLY_ONE_VERSION, -3123, true); \
- E(INVERTED_INDEX_INVALID_PARAMETERS, -6000, false); \
+ E(INDEX_INVALID_PARAMETERS, -6000, false); \
E(INVERTED_INDEX_NOT_SUPPORTED, -6001, false); \
E(INVERTED_INDEX_CLUCENE_ERROR, -6002, false); \
E(INVERTED_INDEX_FILE_NOT_FOUND, -6003, false); \
diff --git a/be/src/index-tools/ann_tool.cpp b/be/src/index-tools/ann_tool.cpp
index 14ceecaa16d..f56e54fc4c5 100644
--- a/be/src/index-tools/ann_tool.cpp
+++ b/be/src/index-tools/ann_tool.cpp
@@ -50,8 +50,8 @@
#include "olap/rowset/segment_v2/inverted_index/query/conjunction_query.h"
#include "olap/rowset/segment_v2/inverted_index_compound_reader.h"
#include "olap/rowset/segment_v2/inverted_index_desc.h"
-#include "olap/rowset/segment_v2/inverted_index_file_reader.h"
#include "olap/rowset/segment_v2/inverted_index_fs_directory.h"
+#include "olap/rowset/segment_v2/x_index_file_reader.h"
#include "olap/rowset/segment_v2/x_index_file_writer.h"
#include "olap/tablet_schema.h"
#include "util/disk_info.h"
@@ -63,7 +63,7 @@ using doris::segment_v2::DorisCompoundReader;
using doris::segment_v2::DorisFSDirectoryFactory;
using doris::segment_v2::XIndexFileWriter;
using doris::segment_v2::InvertedIndexDescriptor;
-using doris::segment_v2::InvertedIndexFileReader;
+using doris::segment_v2::XIndexFileReader;
using doris::io::FileInfo;
using doris::TabletIndex;
using namespace doris::segment_v2;
@@ -286,9 +286,9 @@ void init_env() {
void test_search() {
auto fs = get_local_file_filesystem();
- auto index_file_reader = std::make_unique<InvertedIndexFileReader>(
- fs, "/home/users/clz/run/test_diskann/123456_0",
- doris::InvertedIndexStorageFormatPB::V2);
+ auto index_file_reader =
+ std::make_unique<XIndexFileReader>(fs, "/home/users/clz/run/test_diskann/123456_0",
+ doris::InvertedIndexStorageFormatPB::V2);
auto st = index_file_reader->init(4096);
if (!st.ok()) {
std::cout << "failed to index_file_reader->init" << st << std::endl;
@@ -301,7 +301,7 @@ void test_search() {
auto ret = index_file_reader->open(&index_meta);
if (!ret.has_value()) {
- std::cerr << "InvertedIndexFileReader open error:" << ret.error() << std::endl;
+ std::cerr << "XIndexFileReader open error:" << ret.error() << std::endl;
return;
}
using T = std::decay_t<decltype(ret)>;
diff --git a/be/src/index-tools/index_tool.cpp b/be/src/index-tools/index_tool.cpp
index ecabbdea0fe..fa6d16c4d12 100644
--- a/be/src/index-tools/index_tool.cpp
+++ b/be/src/index-tools/index_tool.cpp
@@ -45,8 +45,8 @@
#include "olap/rowset/segment_v2/inverted_index/query/conjunction_query.h"
#include "olap/rowset/segment_v2/inverted_index_compound_reader.h"
#include "olap/rowset/segment_v2/inverted_index_desc.h"
-#include "olap/rowset/segment_v2/inverted_index_file_reader.h"
#include "olap/rowset/segment_v2/inverted_index_fs_directory.h"
+#include "olap/rowset/segment_v2/x_index_file_reader.h"
#include "olap/rowset/segment_v2/x_index_file_writer.h"
#include "olap/tablet_schema.h"
@@ -54,7 +54,7 @@ using doris::segment_v2::DorisCompoundReader;
using doris::segment_v2::DorisFSDirectoryFactory;
using doris::segment_v2::XIndexFileWriter;
using doris::segment_v2::InvertedIndexDescriptor;
-using doris::segment_v2::InvertedIndexFileReader;
+using doris::segment_v2::XIndexFileReader;
using doris::io::FileInfo;
using doris::TabletIndex;
using namespace doris::segment_v2;
@@ -610,11 +610,11 @@ int main(int argc, char** argv) {
std::string index_path_prefix = StripSuffixString(FLAGS_idx_file_path, ".idx");
auto fs = doris::io::global_local_filesystem();
try {
- auto index_file_reader = std::make_unique<InvertedIndexFileReader>(
+ auto index_file_reader = std::make_unique<XIndexFileReader>(
fs, index_path_prefix, doris::InvertedIndexStorageFormatPB::V2);
auto st = index_file_reader->init(4096);
if (!st.ok()) {
- std::cerr << "InvertedIndexFileReader init error:" << st.msg() << std::endl;
+ std::cerr << "XIndexFileReader init error:" << st.msg() << std::endl;
return -1;
}
std::cout << "Nested files for " << index_path_prefix << std::endl;
@@ -635,7 +635,7 @@ int main(int argc, char** argv) {
CLuceneError err;
auto ret = index_file_reader->open(&index_meta);
if (!ret.has_value()) {
- std::cerr << "InvertedIndexFileReader open error:" << ret.error() << std::endl;
+ std::cerr << "XIndexFileReader open error:" << ret.error() << std::endl;
return -1;
}
using T = std::decay_t<decltype(ret)>;
@@ -656,11 +656,11 @@ int main(int argc, char** argv) {
std::string index_path_prefix = StripSuffixString(FLAGS_idx_file_path, ".idx");
auto fs = doris::io::global_local_filesystem();
try {
- auto index_file_reader = std::make_unique<InvertedIndexFileReader>(
+ auto index_file_reader = std::make_unique<XIndexFileReader>(
fs, index_path_prefix, doris::InvertedIndexStorageFormatPB::V2);
auto st = index_file_reader->init(4096);
if (!st.ok()) {
- std::cerr << "InvertedIndexFileReader init error:" << st.msg() << std::endl;
+ std::cerr << "XIndexFileReader init error:" << st.msg() << std::endl;
return -1;
}
std::vector<std::string> files;
@@ -673,7 +673,7 @@ int main(int argc, char** argv) {
index_meta.init_from_pb(index_pb);
auto ret = index_file_reader->open(&index_meta);
if (!ret.has_value()) {
- std::cerr << "InvertedIndexFileReader open error:" << ret.error() << std::endl;
+ std::cerr << "XIndexFileReader open error:" << ret.error() << std::endl;
return -1;
}
using T = std::decay_t<decltype(ret)>;
diff --git a/be/src/olap/accept_null_predicate.h b/be/src/olap/accept_null_predicate.h
index 112101f46dd..e3296c6fcc6 100644
--- a/be/src/olap/accept_null_predicate.h
+++ b/be/src/olap/accept_null_predicate.h
@@ -51,7 +51,7 @@ public:
}
Status evaluate(const vectorized::IndexFieldNameAndTypePair& name_with_type,
- InvertedIndexIterator* iterator, uint32_t num_rows,
+ IndexIterator* iterator, uint32_t num_rows,
roaring::Roaring* bitmap) const override {
return _nested->evaluate(name_with_type, iterator, num_rows, bitmap);
}
diff --git a/be/src/olap/column_predicate.h b/be/src/olap/column_predicate.h
index 628b49c6213..43d40cd4f3f 100644
--- a/be/src/olap/column_predicate.h
+++ b/be/src/olap/column_predicate.h
@@ -22,7 +22,7 @@
#include "common/exception.h"
#include "olap/rowset/segment_v2/bitmap_index_reader.h"
#include "olap/rowset/segment_v2/bloom_filter.h"
-#include "olap/rowset/segment_v2/inverted_index_reader.h"
+#include "olap/rowset/segment_v2/inverted_index_iterator.h"
#include "runtime/define_primitive_type.h"
#include "vec/columns/column.h"
#include "vec/exprs/vruntimefilter_wrapper.h"
@@ -168,7 +168,7 @@ public:
//evaluate predicate on inverted
virtual Status evaluate(const vectorized::IndexFieldNameAndTypePair& name_with_type,
- InvertedIndexIterator* iterator, uint32_t num_rows,
+ IndexIterator* iterator, uint32_t num_rows,
roaring::Roaring* bitmap) const {
return Status::NotSupported(
"Not Implemented evaluate with inverted index, please check the predicate");
diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp
index 4d98eaef690..a33ff2f139e 100644
--- a/be/src/olap/compaction.cpp
+++ b/be/src/olap/compaction.cpp
@@ -59,8 +59,8 @@
#include "olap/rowset/rowset_writer_context.h"
#include "olap/rowset/segment_v2/inverted_index_compaction.h"
#include "olap/rowset/segment_v2/inverted_index_desc.h"
-#include "olap/rowset/segment_v2/inverted_index_file_reader.h"
#include "olap/rowset/segment_v2/inverted_index_fs_directory.h"
+#include "olap/rowset/segment_v2/x_index_file_reader.h"
#include "olap/rowset/segment_v2/x_index_file_writer.h"
#include "olap/storage_engine.h"
#include "olap/storage_policy.h"
@@ -634,8 +634,7 @@ Status Compaction::do_inverted_index_compaction() {
}
// src index dirs
- std::vector<std::unique_ptr<InvertedIndexFileReader>> inverted_index_file_readers(
- src_segment_num);
+ std::vector<std::unique_ptr<XIndexFileReader>> index_file_readers(src_segment_num);
for (const auto& m : src_seg_to_id_map) {
const auto& [rowset_id, seg_id] = m.first;
@@ -676,12 +675,12 @@ Status Compaction::do_inverted_index_compaction() {
"get segment path failed. tablet_id={} rowset_id={} seg_id={}",
_tablet->tablet_id(), rowset_id.to_string(), seg_id);
}
- auto inverted_index_file_reader = std::make_unique<InvertedIndexFileReader>(
+ auto index_file_reader = std::make_unique<XIndexFileReader>(
fs,
std::string {InvertedIndexDescriptor::get_index_file_path_prefix(seg_path.value())},
_cur_tablet_schema->get_inverted_index_storage_format(),
rowset->rowset_meta()->inverted_index_file_info(seg_id));
- auto st = inverted_index_file_reader->init(config::inverted_index_read_buffer_size);
+ auto st = index_file_reader->init(config::inverted_index_read_buffer_size);
DBUG_EXECUTE_IF("Compaction::do_inverted_index_compaction_init_inverted_index_file_reader",
{
st = Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>(
@@ -699,7 +698,7 @@ Status Compaction::do_inverted_index_compaction() {
"init inverted index file reader failed. tablet_id={} rowset_id={} seg_id={}",
_tablet->tablet_id(), rowset_id.to_string(), seg_id);
}
- inverted_index_file_readers[m.second] = std::move(inverted_index_file_reader);
+ index_file_readers[m.second] = std::move(index_file_reader);
}
// dest index files
@@ -746,7 +745,7 @@ Status Compaction::do_inverted_index_compaction() {
try {
std::vector<std::unique_ptr<DorisCompoundReader>> src_idx_dirs(src_segment_num);
for (int src_segment_id = 0; src_segment_id < src_segment_num; src_segment_id++) {
- auto res = inverted_index_file_readers[src_segment_id]->open(index_meta);
+ auto res = index_file_readers[src_segment_id]->open(index_meta);
DBUG_EXECUTE_IF("Compaction::open_inverted_index_file_reader", {
res = ResultError(Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>(
"debug point: Compaction::open_index_file_reader error"));
@@ -914,15 +913,14 @@ void Compaction::construct_index_compaction_columns(RowsetWriterContext& ctx) {
std::string index_file_path;
try {
- auto inverted_index_file_reader = std::make_unique<InvertedIndexFileReader>(
+ auto index_file_reader = std::make_unique<XIndexFileReader>(
fs,
std::string {InvertedIndexDescriptor::get_index_file_path_prefix(
seg_path.value())},
_cur_tablet_schema->get_inverted_index_storage_format(),
rowset->rowset_meta()->inverted_index_file_info(i));
- auto st = inverted_index_file_reader->init(
- config::inverted_index_read_buffer_size);
- index_file_path = inverted_index_file_reader->get_index_file_path(index_meta);
+ auto st = index_file_reader->init(config::inverted_index_read_buffer_size);
+ index_file_path = index_file_reader->get_index_file_path(index_meta);
DBUG_EXECUTE_IF(
"Compaction::construct_skip_inverted_index_index_file_reader_init_"
"status_not_ok",
@@ -939,7 +937,7 @@ void Compaction::construct_index_compaction_columns(RowsetWriterContext& ctx) {
}
// check index meta
- auto result = inverted_index_file_reader->open(index_meta);
+ auto result = index_file_reader->open(index_meta);
DBUG_EXECUTE_IF(
"Compaction::construct_skip_inverted_index_index_file_reader_open_"
"error",
diff --git a/be/src/olap/comparison_predicate.h b/be/src/olap/comparison_predicate.h
index d7bf38a6c6a..d8587d6663f 100644
--- a/be/src/olap/comparison_predicate.h
+++ b/be/src/olap/comparison_predicate.h
@@ -68,7 +68,7 @@ public:
}
Status evaluate(const vectorized::IndexFieldNameAndTypePair& name_with_type,
- InvertedIndexIterator* iterator, uint32_t num_rows,
+ IndexIterator* iterator, uint32_t num_rows,
roaring::Roaring* bitmap) const override {
if (iterator == nullptr) {
return Status::OK();
@@ -99,13 +99,17 @@ public:
return Status::InvalidArgument("invalid comparison predicate type {}", PT);
}
- std::shared_ptr<roaring::Roaring> roaring = std::make_shared<roaring::Roaring>();
-
std::unique_ptr<InvertedIndexQueryParamFactory> query_param = nullptr;
RETURN_IF_ERROR(
InvertedIndexQueryParamFactory::create_query_value<Type>(&_value, query_param));
- RETURN_IF_ERROR(iterator->read_from_inverted_index(column_name, query_param->get_value(),
- query_type, num_rows, roaring));
+
+ InvertedIndexParam param;
+ param.column_name = column_name;
+ param.query_value = query_param->get_value();
+ param.query_type = query_type;
+ param.num_rows = num_rows;
+ param.roaring = std::make_shared<roaring::Roaring>();
+ RETURN_IF_ERROR(iterator->read_from_index(¶m));
// mask out null_bitmap, since NULL cmp VALUE will produce NULL
// and be treated as false in WHERE
@@ -120,9 +124,9 @@ public:
}
if constexpr (PT == PredicateType::NE) {
- *bitmap -= *roaring;
+ *bitmap -= *param.roaring;
} else {
- *bitmap &= *roaring;
+ *bitmap &= *param.roaring;
}
return Status::OK();
diff --git a/be/src/olap/in_list_predicate.h b/be/src/olap/in_list_predicate.h
index f5ed4e81da9..a950d9624fd 100644
--- a/be/src/olap/in_list_predicate.h
+++ b/be/src/olap/in_list_predicate.h
@@ -181,7 +181,7 @@ public:
}
Status evaluate(const vectorized::IndexFieldNameAndTypePair& name_with_type,
- InvertedIndexIterator* iterator, uint32_t num_rows,
+ IndexIterator* iterator, uint32_t num_rows,
roaring::Roaring* result) const override {
if (iterator == nullptr) {
return Status::OK();
@@ -197,10 +197,14 @@ public:
RETURN_IF_ERROR(
InvertedIndexQueryParamFactory::create_query_value<Type>(ptr, query_param));
InvertedIndexQueryType query_type = InvertedIndexQueryType::EQUAL_QUERY;
- std::shared_ptr<roaring::Roaring> index = std::make_shared<roaring::Roaring>();
- RETURN_IF_ERROR(iterator->read_from_inverted_index(
- column_name, query_param->get_value(), query_type, num_rows, index));
- indices |= *index;
+ InvertedIndexParam param;
+ param.column_name = column_name;
+ param.query_value = query_param->get_value();
+ param.query_type = query_type;
+ param.num_rows = num_rows;
+ param.roaring = std::make_shared<roaring::Roaring>();
+ RETURN_IF_ERROR(iterator->read_from_index(¶m));
+ indices |= *param.roaring;
iter->next();
}
diff --git a/be/src/olap/match_predicate.cpp b/be/src/olap/match_predicate.cpp
index bed1671ab65..bbbf15b0ed1 100644
--- a/be/src/olap/match_predicate.cpp
+++ b/be/src/olap/match_predicate.cpp
@@ -46,28 +46,33 @@ PredicateType MatchPredicate::type() const {
}
Status MatchPredicate::evaluate(const vectorized::IndexFieldNameAndTypePair& name_with_type,
- InvertedIndexIterator* iterator, uint32_t num_rows,
+ IndexIterator* iterator, uint32_t num_rows,
roaring::Roaring* bitmap) const {
if (iterator == nullptr) {
return Status::OK();
}
if (_check_evaluate(iterator)) {
- return Status::Error<ErrorCode::INVERTED_INDEX_INVALID_PARAMETERS>(
+ return Status::Error<ErrorCode::INDEX_INVALID_PARAMETERS>(
"phrase queries require setting support_phrase = true");
}
auto type = name_with_type.second;
const std::string& name = name_with_type.first;
- std::shared_ptr<roaring::Roaring> roaring = std::make_shared<roaring::Roaring>();
auto inverted_index_query_type = _to_inverted_index_query_type(_match_type);
TypeDescriptor column_desc = type->get_type_as_type_descriptor();
+
+ InvertedIndexParam param;
+ param.column_name = name;
+ param.query_type = inverted_index_query_type;
+ param.num_rows = num_rows;
+ param.roaring = std::make_shared<roaring::Roaring>();
if (is_string_type(column_desc.type) ||
(column_desc.type == TYPE_ARRAY && is_string_type(column_desc.children[0].type))) {
StringRef match_value;
auto length = _value.length();
char* buffer = const_cast<char*>(_value.c_str());
match_value.replace(buffer, int32_t(length)); //is it safe?
- RETURN_IF_ERROR(iterator->read_from_inverted_index(
- name, &match_value, inverted_index_query_type, num_rows, roaring));
+ param.query_value = &match_value;
+ RETURN_IF_ERROR(iterator->read_from_index(¶m));
} else if (column_desc.type == TYPE_ARRAY &&
is_numeric_type(
TabletColumn::get_field_type_by_type(column_desc.children[0].type))) {
@@ -75,8 +80,9 @@ Status MatchPredicate::evaluate(const vectorized::IndexFieldNameAndTypePair& nam
const TypeInfo* type_info = get_scalar_type_info(
TabletColumn::get_field_type_by_type(column_desc.children[0].type));
RETURN_IF_ERROR(type_info->from_string(buf.data(), _value));
- RETURN_IF_ERROR(iterator->read_from_inverted_index(
- name, buf.data(), inverted_index_query_type, num_rows, roaring, true));
+ param.query_value = buf.data();
+ param.skip_try = true;
+ RETURN_IF_ERROR(iterator->read_from_index(¶m));
}
// mask out null_bitmap, since NULL cmp VALUE will produce NULL
@@ -91,7 +97,7 @@ Status MatchPredicate::evaluate(const vectorized::IndexFieldNameAndTypePair& nam
}
}
- *bitmap &= *roaring;
+ *bitmap &= *param.roaring;
return Status::OK();
}
@@ -122,12 +128,11 @@ InvertedIndexQueryType MatchPredicate::_to_inverted_index_query_type(MatchType m
return ret;
}
-bool MatchPredicate::_check_evaluate(InvertedIndexIterator* iterator) const {
+bool MatchPredicate::_check_evaluate(IndexIterator* iterator) const {
if (_match_type == MatchType::MATCH_PHRASE || _match_type == MatchType::MATCH_PHRASE_PREFIX ||
_match_type == MatchType::MATCH_PHRASE_EDGE) {
- if (iterator->get_inverted_index_reader_type() == InvertedIndexReaderType::FULLTEXT &&
- get_parser_phrase_support_string_from_properties(iterator->get_index_properties()) ==
- INVERTED_INDEX_PARSER_PHRASE_SUPPORT_NO) {
+ if (iterator->get_reader()->is_fulltext_index() &&
+ !iterator->get_reader()->is_support_phrase()) {
return true;
}
}
diff --git a/be/src/olap/match_predicate.h b/be/src/olap/match_predicate.h
index 3ff1775fd88..c7565d7e7ca 100644
--- a/be/src/olap/match_predicate.h
+++ b/be/src/olap/match_predicate.h
@@ -60,7 +60,7 @@ public:
//evaluate predicate on inverted
Status evaluate(const vectorized::IndexFieldNameAndTypePair& name_with_type,
- InvertedIndexIterator* iterator, uint32_t num_rows,
+ IndexIterator* iterator, uint32_t num_rows,
roaring::Roaring* bitmap) const override;
bool can_do_apply_safely(PrimitiveType input_type, bool is_null) const override {
@@ -78,7 +78,7 @@ private:
std::string info = "MatchPredicate";
return info;
}
- bool _check_evaluate(InvertedIndexIterator* iterator) const;
+ bool _check_evaluate(IndexIterator* iterator) const;
private:
std::string _value;
diff --git a/be/src/olap/metadata_adder.h b/be/src/olap/metadata_adder.h
index 5b5ba163224..533b93dd98c 100644
--- a/be/src/olap/metadata_adder.h
+++ b/be/src/olap/metadata_adder.h
@@ -90,7 +90,7 @@ class BitmapIndexReader;
class BloomFilterIndexReader;
class IndexPageReader;
class IndexedColumnReader;
-class InvertedIndexReader;
+class IndexReader;
class OrdinalIndexReader;
class ZoneMapIndexReader;
}; // namespace segment_v2
@@ -219,7 +219,7 @@ void MetadataAdder<T>::add_mem_size(int64_t val) {
g_index_page_reader_mem_bytes << val;
} else if constexpr (std::is_same_v<T, segment_v2::IndexedColumnReader>) {
g_indexed_column_reader_mem_bytes << val;
- } else if constexpr (std::is_same_v<T, segment_v2::InvertedIndexReader>) {
+ } else if constexpr (std::is_same_v<T, segment_v2::IndexReader>) {
g_inverted_index_reader_mem_bytes << val;
} else if constexpr (std::is_same_v<T, segment_v2::OrdinalIndexReader>) {
g_ordinal_index_reader_mem_bytes << val;
@@ -260,7 +260,7 @@ void MetadataAdder<T>::add_num(int64_t val) {
g_index_page_reader_num << val;
} else if constexpr (std::is_same_v<T, segment_v2::IndexedColumnReader>) {
g_indexed_column_reader_num << val;
- } else if constexpr (std::is_same_v<T, segment_v2::InvertedIndexReader>) {
+ } else if constexpr (std::is_same_v<T, segment_v2::IndexReader>) {
g_inverted_index_reader_num << val;
} else if constexpr (std::is_same_v<T, segment_v2::OrdinalIndexReader>) {
g_ordinal_index_reader_num << val;
diff --git a/be/src/olap/null_predicate.cpp b/be/src/olap/null_predicate.cpp
index 06ab85324ef..7d6889ccc38 100644
--- a/be/src/olap/null_predicate.cpp
+++ b/be/src/olap/null_predicate.cpp
@@ -54,7 +54,7 @@ Status NullPredicate::evaluate(BitmapIndexIterator* iterator, uint32_t num_rows,
}
Status NullPredicate::evaluate(const vectorized::IndexFieldNameAndTypePair& name_with_type,
- InvertedIndexIterator* iterator, uint32_t num_rows,
+ IndexIterator* iterator, uint32_t num_rows,
roaring::Roaring* bitmap) const {
if (iterator->has_null()) {
InvertedIndexQueryCacheHandle null_bitmap_cache_handle;
diff --git a/be/src/olap/null_predicate.h b/be/src/olap/null_predicate.h
index 8e3fef1ff27..d99b519b90b 100644
--- a/be/src/olap/null_predicate.h
+++ b/be/src/olap/null_predicate.h
@@ -53,7 +53,7 @@ public:
roaring::Roaring* roaring) const override;
Status evaluate(const vectorized::IndexFieldNameAndTypePair& name_with_type,
- InvertedIndexIterator* iterator, uint32_t num_rows,
+ IndexIterator* iterator, uint32_t num_rows,
roaring::Roaring* bitmap) const override;
void evaluate_or(const vectorized::IColumn& column, const uint16_t* sel, uint16_t size,
diff --git a/be/src/olap/olap_common.h b/be/src/olap/olap_common.h
index 616bcf980de..42e31bfbb77 100644
--- a/be/src/olap/olap_common.h
+++ b/be/src/olap/olap_common.h
@@ -414,7 +414,7 @@ struct OlapReaderStatistics {
int64_t segment_iterator_init_timer_ns = 0;
int64_t segment_iterator_init_return_column_iterators_timer_ns = 0;
int64_t segment_iterator_init_bitmap_index_iterators_timer_ns = 0;
- int64_t segment_iterator_init_inverted_index_iterators_timer_ns = 0;
+ int64_t segment_iterator_init_index_iterators_timer_ns = 0;
int64_t segment_create_column_readers_timer_ns = 0;
int64_t segment_load_index_timer_ns = 0;
diff --git a/be/src/olap/rowset/beta_rowset.cpp b/be/src/olap/rowset/beta_rowset.cpp
index b25b261c2bb..898def6c3eb 100644
--- a/be/src/olap/rowset/beta_rowset.cpp
+++ b/be/src/olap/rowset/beta_rowset.cpp
@@ -41,7 +41,7 @@
#include "olap/rowset/beta_rowset_reader.h"
#include "olap/rowset/segment_v2/inverted_index_cache.h"
#include "olap/rowset/segment_v2/inverted_index_desc.h"
-#include "olap/rowset/segment_v2/inverted_index_file_reader.h"
+#include "olap/rowset/segment_v2/x_index_file_reader.h"
#include "olap/segment_loader.h"
#include "olap/tablet_schema.h"
#include "olap/utils.h"
@@ -759,10 +759,10 @@ Status BetaRowset::show_nested_index_file(rapidjson::Value* rowset_value,
auto seg_path = DORIS_TRY(segment_path(seg_id));
auto index_file_path_prefix = InvertedIndexDescriptor::get_index_file_path_prefix(seg_path);
- auto inverted_index_file_reader = std::make_unique<InvertedIndexFileReader>(
+ auto index_file_reader = std::make_unique<XIndexFileReader>(
fs, std::string(index_file_path_prefix), storage_format);
- RETURN_IF_ERROR(inverted_index_file_reader->init());
- auto dirs = inverted_index_file_reader->get_all_directories();
+ RETURN_IF_ERROR(index_file_reader->init());
+ auto dirs = index_file_reader->get_all_directories();
auto add_file_info_to_json = [&](const std::string& path,
rapidjson::Value& json_value) -> Status {
@@ -780,15 +780,15 @@ Status BetaRowset::show_nested_index_file(rapidjson::Value* rowset_value,
return Status::OK();
};
- auto process_files = [&allocator, &inverted_index_file_reader](
- auto& index_meta, rapidjson::Value& indices,
- rapidjson::Value& index) -> Status {
+ auto process_files = [&allocator, &index_file_reader](auto& index_meta,
+ rapidjson::Value& indices,
+ rapidjson::Value& index) -> Status {
rapidjson::Value files_value(rapidjson::kArrayType);
std::vector<std::string> files;
- auto ret = inverted_index_file_reader->open(&index_meta);
+ auto ret = index_file_reader->open(&index_meta);
if (!ret.has_value()) {
- LOG(INFO) << "InvertedIndexFileReader open error:" << ret.error();
- return Status::InternalError("InvertedIndexFileReader open error");
+ LOG(INFO) << "XIndexFileReader open error:" << ret.error();
+ return Status::InternalError("XIndexFileReader open error");
}
using T = std::decay_t<decltype(ret)>;
auto reader = std::forward<T>(ret).value();
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_compaction.h b/be/src/olap/rowset/segment_v2/ann_index_iterator.cpp
similarity index 51%
copy from be/src/olap/rowset/segment_v2/inverted_index_compaction.h
copy to be/src/olap/rowset/segment_v2/ann_index_iterator.cpp
index f344f12ffbf..abee8478eb4 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_compaction.h
+++ b/be/src/olap/rowset/segment_v2/ann_index_iterator.cpp
@@ -14,28 +14,27 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-#pragma once
-#include <CLucene.h>
+#include "ann_index_iterator.h"
-#include <cstdint>
-#include <string_view>
-#include <vector>
+#include <memory>
-#include "common/status.h"
-#include "inverted_index_compound_reader.h"
+namespace doris::segment_v2 {
-namespace doris {
-class TabletIndex;
-namespace segment_v2 {
-class XIndexFileWriter;
-class InvertedIndexFileReader;
+AnnIndexIterator::AnnIndexIterator(const io::IOContext& io_ctx, OlapReaderStatistics* stats,
+ RuntimeState* runtime_state, const IndexReaderPtr& reader)
+ : IndexIterator(io_ctx, stats, runtime_state) {
+ _ann_reader = std::static_pointer_cast<AnnIndexReader>(reader);
+}
-Status compact_column(int64_t index_id,
- std::vector<std::unique_ptr<DorisCompoundReader>>& src_index_dirs,
- std::vector<lucene::store::Directory*>& dest_index_dirs,
- std::string_view tmp_path,
- const std::vector<std::vector<std::pair<uint32_t, uint32_t>>>& trans_vec,
- const std::vector<uint32_t>& dest_segment_num_rows);
-} // namespace segment_v2
-} // namespace doris
+Status AnnIndexIterator::read_from_index(const IndexParam& param) {
+ auto* a_param = std::get<AnnIndexParam*>(param);
+ if (a_param == nullptr) {
+ return Status::Error<ErrorCode::INDEX_INVALID_PARAMETERS>("a_param is null");
+ }
+ (void)a_param->column_name;
+
+ return _ann_reader->query(a_param);
+}
+
+} // namespace doris::segment_v2
\ No newline at end of file
diff --git a/be/src/olap/rowset/segment_v2/ann_index_iterator.h b/be/src/olap/rowset/segment_v2/ann_index_iterator.h
new file mode 100644
index 00000000000..cf2e67c82bb
--- /dev/null
+++ b/be/src/olap/rowset/segment_v2/ann_index_iterator.h
@@ -0,0 +1,53 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include "olap/rowset/segment_v2/ann_index_reader.h"
+#include "olap/rowset/segment_v2/index_iterator.h"
+
+namespace doris::segment_v2 {
+
+struct AnnIndexParam {
+ const std::string& column_name;
+};
+
+class AnnIndexIterator : public IndexIterator {
+public:
+ AnnIndexIterator(const io::IOContext& io_ctx, OlapReaderStatistics* stats,
+ RuntimeState* runtime_state, const IndexReaderPtr& reader);
+ ~AnnIndexIterator() override = default;
+
+ IndexType type() override { return IndexType::ANN; }
+
+ IndexReaderPtr get_reader() override { return _ann_reader; }
+
+ Status read_from_index(const IndexParam& param) override;
+
+ Status read_null_bitmap(InvertedIndexQueryCacheHandle* cache_handle) override {
+ return Status::OK();
+ }
+
+ bool has_null() override { return true; }
+
+private:
+ AnnIndexReaderPtr _ann_reader;
+
+ ENABLE_FACTORY_CREATOR(AnnIndexIterator);
+};
+
+} // namespace doris::segment_v2
\ No newline at end of file
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_compaction.h b/be/src/olap/rowset/segment_v2/ann_index_reader.cpp
similarity index 51%
copy from be/src/olap/rowset/segment_v2/inverted_index_compaction.h
copy to be/src/olap/rowset/segment_v2/ann_index_reader.cpp
index f344f12ffbf..0b54f5c6511 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_compaction.h
+++ b/be/src/olap/rowset/segment_v2/ann_index_reader.cpp
@@ -14,28 +14,22 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-#pragma once
-#include <CLucene.h>
+#include "ann_index_reader.h"
-#include <cstdint>
-#include <string_view>
-#include <vector>
+#include "ann_index_iterator.h"
-#include "common/status.h"
-#include "inverted_index_compound_reader.h"
+namespace doris::segment_v2 {
-namespace doris {
-class TabletIndex;
-namespace segment_v2 {
-class XIndexFileWriter;
-class InvertedIndexFileReader;
+AnnIndexReader::AnnIndexReader(const TabletIndex* index_meta,
+ std::shared_ptr<XIndexFileReader> index_file_reader)
+ : _index_meta(*index_meta), _index_file_reader(std::move(index_file_reader)) {}
-Status compact_column(int64_t index_id,
- std::vector<std::unique_ptr<DorisCompoundReader>>& src_index_dirs,
- std::vector<lucene::store::Directory*>& dest_index_dirs,
- std::string_view tmp_path,
- const std::vector<std::vector<std::pair<uint32_t, uint32_t>>>& trans_vec,
- const std::vector<uint32_t>& dest_segment_num_rows);
-} // namespace segment_v2
-} // namespace doris
+Status AnnIndexReader::new_iterator(const io::IOContext& io_ctx, OlapReaderStatistics* stats,
+ RuntimeState* runtime_state,
+ std::unique_ptr<IndexIterator>* iterator) {
+ *iterator = AnnIndexIterator::create_unique(io_ctx, stats, runtime_state, shared_from_this());
+ return Status::OK();
+}
+
+} // namespace doris::segment_v2
\ No newline at end of file
diff --git a/be/src/olap/rowset/segment_v2/ann_index_reader.h b/be/src/olap/rowset/segment_v2/ann_index_reader.h
new file mode 100644
index 00000000000..3d7edee67c5
--- /dev/null
+++ b/be/src/olap/rowset/segment_v2/ann_index_reader.h
@@ -0,0 +1,49 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include "olap/rowset/segment_v2/index_reader.h"
+#include "olap/tablet_schema.h"
+
+namespace doris::segment_v2 {
+
+struct AnnIndexParam;
+
+class XIndexFileReader;
+class IndexIterator;
+
+class AnnIndexReader : public IndexReader {
+public:
+ AnnIndexReader(const TabletIndex* index_meta,
+ std::shared_ptr<XIndexFileReader> index_file_reader);
+ ~AnnIndexReader() override = default;
+
+ Status query(AnnIndexParam* param) { return Status::OK(); }
+
+ uint64_t get_index_id() const override { return _index_meta.index_id(); }
+
+ Status new_iterator(const io::IOContext& io_ctx, OlapReaderStatistics* stats,
+ RuntimeState* runtime_state,
+ std::unique_ptr<IndexIterator>* iterator) override;
+
+private:
+ TabletIndex _index_meta;
+ std::shared_ptr<XIndexFileReader> _index_file_reader;
+};
+
+} // namespace doris::segment_v2
diff --git a/be/src/olap/rowset/segment_v2/ann_index_writer.cpp b/be/src/olap/rowset/segment_v2/ann_index_writer.cpp
index 5d7b70430f5..5ee10ede59f 100644
--- a/be/src/olap/rowset/segment_v2/ann_index_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/ann_index_writer.cpp
@@ -1,3 +1,20 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
#include "olap/rowset/segment_v2/ann_index_writer.h"
#ifdef BUILD_FAISS
diff --git a/be/src/olap/rowset/segment_v2/column_reader.cpp b/be/src/olap/rowset/segment_v2/column_reader.cpp
index 431ef97dfbf..b7464c1b13e 100644
--- a/be/src/olap/rowset/segment_v2/column_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/column_reader.cpp
@@ -38,13 +38,14 @@
#include "olap/inverted_index_parser.h"
#include "olap/iterators.h"
#include "olap/olap_common.h"
+#include "olap/rowset/segment_v2/ann_index_reader.h"
#include "olap/rowset/segment_v2/binary_dict_page.h" // for BinaryDictPageDecoder
#include "olap/rowset/segment_v2/binary_plain_page.h"
#include "olap/rowset/segment_v2/bitmap_index_reader.h"
#include "olap/rowset/segment_v2/bloom_filter.h"
#include "olap/rowset/segment_v2/bloom_filter_index_reader.h"
#include "olap/rowset/segment_v2/encoding_info.h" // for EncodingInfo
-#include "olap/rowset/segment_v2/inverted_index_file_reader.h"
+#include "olap/rowset/segment_v2/index_reader.h"
#include "olap/rowset/segment_v2/inverted_index_reader.h"
#include "olap/rowset/segment_v2/page_decoder.h"
#include "olap/rowset/segment_v2/page_handle.h" // for PageHandle
@@ -52,6 +53,7 @@
#include "olap/rowset/segment_v2/page_pointer.h" // for PagePointer
#include "olap/rowset/segment_v2/row_ranges.h"
#include "olap/rowset/segment_v2/segment.h"
+#include "olap/rowset/segment_v2/x_index_file_reader.h"
#include "olap/rowset/segment_v2/zone_map_index.h"
#include "olap/tablet_schema.h"
#include "olap/types.h" // for TypeInfo
@@ -338,15 +340,16 @@ Status ColumnReader::new_bitmap_index_iterator(BitmapIndexIterator** iterator) {
return Status::OK();
}
-Status ColumnReader::new_inverted_index_iterator(
- std::shared_ptr<InvertedIndexFileReader> index_file_reader, const TabletIndex* index_meta,
- const StorageReadOptions& read_options, std::unique_ptr<InvertedIndexIterator>* iterator) {
- RETURN_IF_ERROR(_ensure_inverted_index_loaded(std::move(index_file_reader), index_meta));
+Status ColumnReader::new_index_iterator(std::shared_ptr<XIndexFileReader> index_file_reader,
+ const TabletIndex* index_meta,
+ const StorageReadOptions& read_options,
+ std::unique_ptr<IndexIterator>* iterator) {
+ RETURN_IF_ERROR(_ensure_index_loaded(std::move(index_file_reader), index_meta));
{
std::shared_lock<std::shared_mutex> rlock(_load_index_lock);
- if (_inverted_index) {
- RETURN_IF_ERROR(_inverted_index->new_iterator(read_options.io_ctx, read_options.stats,
- read_options.runtime_state, iterator));
+ if (_index_reader) {
+ RETURN_IF_ERROR(_index_reader->new_iterator(read_options.io_ctx, read_options.stats,
+ read_options.runtime_state, iterator));
}
}
return Status::OK();
@@ -624,12 +627,12 @@ Status ColumnReader::_load_bitmap_index(bool use_page_cache, bool kept_in_memory
return Status::OK();
}
-Status ColumnReader::_load_inverted_index_index(
- std::shared_ptr<InvertedIndexFileReader> index_file_reader, const TabletIndex* index_meta) {
+Status ColumnReader::_load_index_index(std::shared_ptr<XIndexFileReader> index_file_reader,
+ const TabletIndex* index_meta) {
std::unique_lock<std::shared_mutex> wlock(_load_index_lock);
- if (_inverted_index && index_meta &&
- _inverted_index->get_index_id() == index_meta->index_id()) {
+ if (_index_reader != nullptr && index_meta &&
+ _index_reader->get_index_id() == index_meta->index_id()) {
return Status::OK();
}
@@ -642,32 +645,37 @@ Status ColumnReader::_load_inverted_index_index(
type = _type_info->type();
}
- if (is_string_type(type)) {
- if (parser_type != InvertedIndexParserType::PARSER_NONE) {
- try {
- _inverted_index = FullTextIndexReader::create_shared(index_meta, index_file_reader);
- } catch (const CLuceneError& e) {
- return Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>(
- "create FullTextIndexReader error: {}", e.what());
+ if (index_meta->index_type() == IndexType::INVERTED) {
+ if (is_string_type(type)) {
+ if (parser_type != InvertedIndexParserType::PARSER_NONE) {
+ try {
+ _index_reader =
+ FullTextIndexReader::create_shared(index_meta, index_file_reader);
+ } catch (const CLuceneError& e) {
+ return Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>(
+ "create FullTextIndexReader error: {}", e.what());
+ }
+ } else {
+ try {
+ _index_reader = StringTypeInvertedIndexReader::create_shared(index_meta,
+ index_file_reader);
+ } catch (const CLuceneError& e) {
+ return Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>(
+ "create StringTypeInvertedIndexReader error: {}", e.what());
+ }
}
- } else {
+ } else if (is_numeric_type(type)) {
try {
- _inverted_index =
- StringTypeInvertedIndexReader::create_shared(index_meta, index_file_reader);
+ _index_reader = BkdIndexReader::create_shared(index_meta, index_file_reader);
} catch (const CLuceneError& e) {
return Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>(
- "create StringTypeInvertedIndexReader error: {}", e.what());
+ "create BkdIndexReader error: {}", e.what());
}
+ } else {
+ _index_reader.reset();
}
- } else if (is_numeric_type(type)) {
- try {
- _inverted_index = BkdIndexReader::create_shared(index_meta, index_file_reader);
- } catch (const CLuceneError& e) {
- return Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>(
- "create BkdIndexReader error: {}", e.what());
- }
- } else {
- _inverted_index.reset();
+ } else if (index_meta->index_type() == IndexType::ANN) {
+ _index_reader = std::make_shared<AnnIndexReader>(index_meta, index_file_reader);
}
// TODO: move has null to inverted_index_reader's query function
//bool has_null = true;
diff --git a/be/src/olap/rowset/segment_v2/column_reader.h b/be/src/olap/rowset/segment_v2/column_reader.h
index c2f3d3822d2..51c8638690f 100644
--- a/be/src/olap/rowset/segment_v2/column_reader.h
+++ b/be/src/olap/rowset/segment_v2/column_reader.h
@@ -37,6 +37,7 @@
#include "io/io_common.h"
#include "olap/olap_common.h"
#include "olap/rowset/segment_v2/common.h"
+#include "olap/rowset/segment_v2/index_reader.h"
#include "olap/rowset/segment_v2/ordinal_page_index.h" // for OrdinalPageIndexIterator
#include "olap/rowset/segment_v2/page_handle.h" // for PageHandle
#include "olap/rowset/segment_v2/page_pointer.h"
@@ -74,10 +75,11 @@ class BitmapIndexIterator;
class BitmapIndexReader;
class InvertedIndexIterator;
class InvertedIndexReader;
-class InvertedIndexFileReader;
+class XIndexFileReader;
class PageDecoder;
class RowRanges;
class ZoneMapIndexReader;
+class IndexIterator;
struct ColumnReaderOptions {
// whether verify checksum when read page
@@ -142,10 +144,9 @@ public:
// Client should delete returned iterator
Status new_bitmap_index_iterator(BitmapIndexIterator** iterator);
- Status new_inverted_index_iterator(std::shared_ptr<InvertedIndexFileReader> index_file_reader,
- const TabletIndex* index_meta,
- const StorageReadOptions& read_options,
- std::unique_ptr<InvertedIndexIterator>* iterator);
+ Status new_index_iterator(std::shared_ptr<XIndexFileReader> index_file_reader,
+ const TabletIndex* index_meta, const StorageReadOptions& read_options,
+ std::unique_ptr<IndexIterator>* iterator);
// Seek to the first entry in the column.
Status seek_to_first(OrdinalPageIndexIterator* iter, const ColumnIteratorOptions& iter_opts);
@@ -213,12 +214,12 @@ private:
io::FileReaderSPtr file_reader);
Status init(const ColumnMetaPB* meta);
- // Read column inverted indexes into memory
+ // Read column indexes into memory
// May be called multiple times, subsequent calls will no op.
- Status _ensure_inverted_index_loaded(std::shared_ptr<InvertedIndexFileReader> index_file_reader,
- const TabletIndex* index_meta) {
+ Status _ensure_index_loaded(std::shared_ptr<XIndexFileReader> index_file_reader,
+ const TabletIndex* index_meta) {
// load inverted index only if not loaded or index_id is changed
- RETURN_IF_ERROR(_load_inverted_index_index(index_file_reader, index_meta));
+ RETURN_IF_ERROR(_load_index_index(index_file_reader, index_meta));
return Status::OK();
}
@@ -227,9 +228,8 @@ private:
[[nodiscard]] Status _load_ordinal_index(bool use_page_cache, bool kept_in_memory,
const ColumnIteratorOptions& iter_opts);
[[nodiscard]] Status _load_bitmap_index(bool use_page_cache, bool kept_in_memory);
- [[nodiscard]] Status _load_inverted_index_index(
- std::shared_ptr<InvertedIndexFileReader> index_file_reader,
- const TabletIndex* index_meta);
+ [[nodiscard]] Status _load_index_index(std::shared_ptr<XIndexFileReader> index_file_reader,
+ const TabletIndex* index_meta);
[[nodiscard]] Status _load_bloom_filter_index(bool use_page_cache, bool kept_in_memory,
const ColumnIteratorOptions& iter_opts);
@@ -283,9 +283,10 @@ private:
std::unique_ptr<ZoneMapIndexReader> _zone_map_index;
std::unique_ptr<OrdinalIndexReader> _ordinal_index;
std::unique_ptr<BitmapIndexReader> _bitmap_index;
- std::shared_ptr<InvertedIndexReader> _inverted_index;
std::shared_ptr<BloomFilterIndexReader> _bloom_filter_index;
+ IndexReaderPtr _index_reader;
+
std::vector<std::unique_ptr<ColumnReader>> _sub_readers;
DorisCallOnce<Status> _set_dict_encoding_type_once;
diff --git a/be/src/olap/rowset/segment_v2/index_iterator.h b/be/src/olap/rowset/segment_v2/index_iterator.h
new file mode 100644
index 00000000000..0fef498340e
--- /dev/null
+++ b/be/src/olap/rowset/segment_v2/index_iterator.h
@@ -0,0 +1,58 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <gen_cpp/olap_file.pb.h>
+
+#include <variant>
+
+#include "common/exception.h"
+#include "common/factory_creator.h"
+#include "olap/rowset/segment_v2/index_reader.h"
+#include "olap/rowset/segment_v2/inverted_index_query_type.h"
+#include "runtime/runtime_state.h"
+
+namespace doris::segment_v2 {
+
+class InvertedIndexQueryCacheHandle;
+
+struct InvertedIndexParam;
+struct AnnIndexParam;
+using IndexParam = std::variant<InvertedIndexParam*, AnnIndexParam*>;
+
+class IndexIterator {
+public:
+ IndexIterator(const io::IOContext& io_ctx, OlapReaderStatistics* stats,
+ RuntimeState* runtime_state)
+ : _io_ctx(io_ctx), _stats(stats), _runtime_state(runtime_state) {}
+
+ virtual ~IndexIterator() = default;
+
+ virtual IndexType type() = 0;
+ virtual IndexReaderPtr get_reader() = 0;
+ virtual Status read_from_index(const IndexParam& param) = 0;
+ virtual Status read_null_bitmap(InvertedIndexQueryCacheHandle* cache_handle) = 0;
+ virtual bool has_null() = 0;
+
+protected:
+ io::IOContext _io_ctx;
+ OlapReaderStatistics* _stats = nullptr;
+ RuntimeState* _runtime_state = nullptr;
+};
+
+} // namespace doris::segment_v2
\ No newline at end of file
diff --git a/be/src/olap/rowset/segment_v2/index_reader.h b/be/src/olap/rowset/segment_v2/index_reader.h
new file mode 100644
index 00000000000..c71e91dae9a
--- /dev/null
+++ b/be/src/olap/rowset/segment_v2/index_reader.h
@@ -0,0 +1,58 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <memory>
+
+#include "olap/metadata_adder.h"
+
+namespace doris {
+class RuntimeState;
+}
+
+namespace doris::segment_v2 {
+
+class IndexIterator;
+
+class InvertedIndexReader;
+using InvertedIndexReaderPtr = std::shared_ptr<InvertedIndexReader>;
+
+class AnnIndexReader;
+using AnnIndexReaderPtr = std::shared_ptr<AnnIndexReader>;
+
+class IndexReader : public std::enable_shared_from_this<IndexReader>,
+ public MetadataAdder<IndexReader> {
+public:
+ IndexReader() = default;
+ ~IndexReader() override = default;
+
+ virtual uint64_t get_index_id() const = 0;
+
+ virtual Status new_iterator(const io::IOContext& io_ctx, OlapReaderStatistics* stats,
+ RuntimeState* runtime_state,
+ std::unique_ptr<IndexIterator>* iterator) = 0;
+
+ virtual bool is_fulltext_index() { return false; }
+ virtual bool is_string_index() { return false; }
+ virtual bool is_bkd_index() { return false; }
+
+ virtual bool is_support_phrase() { return false; }
+};
+using IndexReaderPtr = std::shared_ptr<IndexReader>;
+
+} // namespace doris::segment_v2
\ No newline at end of file
diff --git a/be/src/olap/rowset/segment_v2/index_writer.cpp b/be/src/olap/rowset/segment_v2/index_writer.cpp
index 63cebbec4ca..3f5a6201552 100644
--- a/be/src/olap/rowset/segment_v2/index_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/index_writer.cpp
@@ -1,3 +1,20 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
#include "olap/rowset/segment_v2/index_writer.h"
#include <CLucene.h> // IWYU pragma: keep
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_compaction.h b/be/src/olap/rowset/segment_v2/inverted_index_compaction.h
index f344f12ffbf..9955ff91841 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_compaction.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index_compaction.h
@@ -29,7 +29,7 @@ namespace doris {
class TabletIndex;
namespace segment_v2 {
class XIndexFileWriter;
-class InvertedIndexFileReader;
+class XIndexFileReader;
Status compact_column(int64_t index_id,
std::vector<std::unique_ptr<DorisCompoundReader>>& src_index_dirs,
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_iterator.cpp b/be/src/olap/rowset/segment_v2/inverted_index_iterator.cpp
new file mode 100644
index 00000000000..dc5fbe0791b
--- /dev/null
+++ b/be/src/olap/rowset/segment_v2/inverted_index_iterator.cpp
@@ -0,0 +1,109 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "inverted_index_iterator.h"
+
+#include <memory>
+
+#include "olap/rowset/segment_v2/inverted_index_cache.h"
+#include "olap/rowset/segment_v2/inverted_index_reader.h"
+
+namespace doris::segment_v2 {
+
+InvertedIndexIterator::InvertedIndexIterator(const io::IOContext& io_ctx,
+ OlapReaderStatistics* stats,
+ RuntimeState* runtime_state,
+ const IndexReaderPtr& reader)
+ : IndexIterator(io_ctx, stats, runtime_state) {
+ _index_reader = std::static_pointer_cast<InvertedIndexReader>(reader);
+}
+
+Status InvertedIndexIterator::read_from_index(const IndexParam& param) {
+ auto* i_param = std::get<InvertedIndexParam*>(param);
+ if (i_param == nullptr) {
+ return Status::Error<ErrorCode::INDEX_INVALID_PARAMETERS>("i_param is null");
+ }
+ DBUG_EXECUTE_IF("return_inverted_index_bypass", {
+ return Status::Error<ErrorCode::INVERTED_INDEX_BYPASS>("inverted index bypass");
+ });
+ if (UNLIKELY(_index_reader == nullptr)) {
+ throw CLuceneError(CL_ERR_NullPointer, "bkd index reader is null", false);
+ }
+ if (!i_param->skip_try && _index_reader->type() == InvertedIndexReaderType::BKD) {
+ if (_runtime_state != nullptr &&
+ _runtime_state->query_options().inverted_index_skip_threshold > 0 &&
+ _runtime_state->query_options().inverted_index_skip_threshold < 100) {
+ auto query_bkd_limit_percent =
+ _runtime_state->query_options().inverted_index_skip_threshold;
+ uint32_t hit_count = 0;
+ RETURN_IF_ERROR(try_read_from_inverted_index(i_param->column_name, i_param->query_value,
+ i_param->query_type, &hit_count));
+ if (hit_count > i_param->num_rows * query_bkd_limit_percent / 100) {
+ return Status::Error<ErrorCode::INVERTED_INDEX_BYPASS>(
+ "hit count: {}, bkd inverted reached limit {}% , segment num "
+ "rows:{}", // add blackspace after % to avoid log4j format bug
+ hit_count, query_bkd_limit_percent, i_param->num_rows);
+ }
+ }
+ }
+
+ auto execute_query = [&]() {
+ return _index_reader->query(&_io_ctx, _stats, _runtime_state, i_param->column_name,
+ i_param->query_value, i_param->query_type, i_param->roaring);
+ };
+
+ if (_runtime_state->query_options().enable_profile) {
+ InvertedIndexQueryStatistics query_stats;
+ {
+ SCOPED_RAW_TIMER(&query_stats.exec_time);
+ RETURN_IF_ERROR(execute_query());
+ }
+ query_stats.column_name = i_param->column_name;
+ query_stats.hit_rows = i_param->roaring->cardinality();
+ _stats->inverted_index_stats.stats.emplace_back(query_stats);
+ } else {
+ RETURN_IF_ERROR(execute_query());
+ }
+
+ return Status::OK();
+}
+
+Status InvertedIndexIterator::read_null_bitmap(InvertedIndexQueryCacheHandle* cache_handle) {
+ return _index_reader->read_null_bitmap(&_io_ctx, _stats, cache_handle, nullptr);
+}
+
+bool InvertedIndexIterator::has_null() {
+ return _index_reader->has_null();
+}
+
+Status InvertedIndexIterator::try_read_from_inverted_index(const std::string& column_name,
+ const void* query_value,
+ InvertedIndexQueryType query_type,
+ uint32_t* count) {
+ // NOTE: only bkd index support try read now.
+ if (query_type == InvertedIndexQueryType::GREATER_EQUAL_QUERY ||
+ query_type == InvertedIndexQueryType::GREATER_THAN_QUERY ||
+ query_type == InvertedIndexQueryType::LESS_EQUAL_QUERY ||
+ query_type == InvertedIndexQueryType::LESS_THAN_QUERY ||
+ query_type == InvertedIndexQueryType::EQUAL_QUERY) {
+ RETURN_IF_ERROR(_index_reader->try_query(&_io_ctx, _stats, _runtime_state, column_name,
+ query_value, query_type, count));
+ }
+ return Status::OK();
+}
+
+} // namespace doris::segment_v2
\ No newline at end of file
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_iterator.h b/be/src/olap/rowset/segment_v2/inverted_index_iterator.h
new file mode 100644
index 00000000000..39d6d4f6e6d
--- /dev/null
+++ b/be/src/olap/rowset/segment_v2/inverted_index_iterator.h
@@ -0,0 +1,56 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include "olap/rowset/segment_v2/index_iterator.h"
+#include "olap/rowset/segment_v2/inverted_index_reader.h"
+
+namespace doris::segment_v2 {
+
+struct InvertedIndexParam {
+ std::string column_name;
+ const void* query_value;
+ InvertedIndexQueryType query_type;
+ uint32_t num_rows;
+ std::shared_ptr<roaring::Roaring> roaring;
+ bool skip_try = false;
+};
+
+class InvertedIndexIterator : public IndexIterator {
+public:
+ InvertedIndexIterator(const io::IOContext& io_ctx, OlapReaderStatistics* stats,
+ RuntimeState* runtime_state, const IndexReaderPtr& reader);
+ ~InvertedIndexIterator() override = default;
+
+ IndexType type() override { return IndexType::INVERTED; }
+ IndexReaderPtr get_reader() override { return _index_reader; }
+
+ Status read_from_index(const IndexParam& param) override;
+ Status read_null_bitmap(InvertedIndexQueryCacheHandle* cache_handle) override;
+ bool has_null() override;
+
+private:
+ Status try_read_from_inverted_index(const std::string& column_name, const void* query_value,
+ InvertedIndexQueryType query_type, uint32_t* count);
+
+ InvertedIndexReaderPtr _index_reader;
+
+ ENABLE_FACTORY_CREATOR(InvertedIndexIterator);
+};
+
+} // namespace doris::segment_v2
\ No newline at end of file
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp b/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp
index 4caffbf96ee..9754a54dd70 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp
@@ -45,9 +45,10 @@
#include "olap/rowset/segment_v2/inverted_index/query/phrase_query.h"
#include "olap/rowset/segment_v2/inverted_index/query/query_factory.h"
#include "olap/rowset/segment_v2/inverted_index_cache.h"
-#include "olap/rowset/segment_v2/inverted_index_file_reader.h"
#include "olap/rowset/segment_v2/inverted_index_fs_directory.h"
+#include "olap/rowset/segment_v2/inverted_index_iterator.h"
#include "olap/rowset/segment_v2/inverted_index_searcher.h"
+#include "olap/rowset/segment_v2/x_index_file_reader.h"
#include "olap/types.h"
#include "runtime/runtime_state.h"
#include "util/faststring.h"
@@ -98,7 +99,7 @@ CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_IPV4)
CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_IPV6)
std::string InvertedIndexReader::get_index_file_path() {
- return _inverted_index_file_reader->get_index_file_path(&_index_meta);
+ return _index_file_reader->get_index_file_path(&_index_meta);
}
Status InvertedIndexReader::read_null_bitmap(const io::IOContext* io_ctx,
@@ -110,7 +111,7 @@ Status InvertedIndexReader::read_null_bitmap(const io::IOContext* io_ctx,
bool owned_dir = false;
try {
// try to get query bitmap result from cache and return immediately on cache hit
- auto index_file_key = _inverted_index_file_reader->get_index_file_cache_key(&_index_meta);
+ auto index_file_key = _index_file_reader->get_index_file_cache_key(&_index_meta);
InvertedIndexQueryCache::CacheKey cache_key {
index_file_key, "", InvertedIndexQueryType::UNKNOWN_QUERY, "null_bitmap"};
auto* cache = InvertedIndexQueryCache::instance();
@@ -120,13 +121,12 @@ Status InvertedIndexReader::read_null_bitmap(const io::IOContext* io_ctx,
if (!dir) {
// TODO: ugly code here, try to refact.
- auto st = _inverted_index_file_reader->init(config::inverted_index_read_buffer_size,
- io_ctx);
+ auto st = _index_file_reader->init(config::inverted_index_read_buffer_size, io_ctx);
if (!st.ok()) {
LOG(WARNING) << st;
return st;
}
- auto directory = DORIS_TRY(_inverted_index_file_reader->open(&_index_meta, io_ctx));
+ auto directory = DORIS_TRY(_index_file_reader->open(&_index_meta, io_ctx));
dir = directory.release();
owned_dir = true;
}
@@ -190,7 +190,7 @@ Status InvertedIndexReader::handle_query_cache(RuntimeState* runtime_state,
Status InvertedIndexReader::handle_searcher_cache(
RuntimeState* runtime_state, InvertedIndexCacheHandle* inverted_index_cache_handle,
const io::IOContext* io_ctx, OlapReaderStatistics* stats) {
- auto index_file_key = _inverted_index_file_reader->get_index_file_cache_key(&_index_meta);
+ auto index_file_key = _index_file_reader->get_index_file_cache_key(&_index_meta);
InvertedIndexSearcherCache::CacheKey searcher_cache_key(index_file_key);
const auto& query_options = runtime_state->query_options();
if (query_options.enable_inverted_index_searcher_cache &&
@@ -211,13 +211,12 @@ Status InvertedIndexReader::handle_searcher_cache(
SCOPED_RAW_TIMER(&stats->inverted_index_searcher_open_timer);
IndexSearcherPtr searcher;
- auto st =
- _inverted_index_file_reader->init(config::inverted_index_read_buffer_size, io_ctx);
+ auto st = _index_file_reader->init(config::inverted_index_read_buffer_size, io_ctx);
if (!st.ok()) {
LOG(WARNING) << st;
return st;
}
- auto dir = DORIS_TRY(_inverted_index_file_reader->open(&_index_meta, io_ctx));
+ auto dir = DORIS_TRY(_index_file_reader->open(&_index_meta, io_ctx));
DBUG_EXECUTE_IF("InvertedIndexReader.handle_searcher_cache.io_ctx", ({
if (dir) {
@@ -279,7 +278,7 @@ Status InvertedIndexReader::match_index_search(
SCOPED_RAW_TIMER(&stats->inverted_index_searcher_search_timer);
auto query = QueryFactory::create(query_type, index_searcher, queryOptions, io_ctx);
if (!query) {
- return Status::Error<ErrorCode::INVERTED_INDEX_INVALID_PARAMETERS>(
+ return Status::Error<ErrorCode::INDEX_INVALID_PARAMETERS>(
"query type " + query_type_to_string(query_type) + ", query is nullptr");
}
query->add(query_info);
@@ -291,9 +290,14 @@ Status InvertedIndexReader::match_index_search(
return Status::OK();
}
+bool InvertedIndexReader::is_support_phrase() {
+ return get_parser_phrase_support_string_from_properties(get_index_properties()) ==
+ INVERTED_INDEX_PARSER_PHRASE_SUPPORT_YES;
+}
+
Status FullTextIndexReader::new_iterator(const io::IOContext& io_ctx, OlapReaderStatistics* stats,
RuntimeState* runtime_state,
- std::unique_ptr<InvertedIndexIterator>* iterator) {
+ std::unique_ptr<IndexIterator>* iterator) {
*iterator =
InvertedIndexIterator::create_unique(io_ctx, stats, runtime_state, shared_from_this());
return Status::OK();
@@ -313,7 +317,7 @@ Status FullTextIndexReader::query(const io::IOContext* io_ctx, OlapReaderStatist
try {
InvertedIndexQueryInfo query_info;
InvertedIndexQueryCache::CacheKey cache_key;
- auto index_file_key = _inverted_index_file_reader->get_index_file_cache_key(&_index_meta);
+ auto index_file_key = _index_file_reader->get_index_file_cache_key(&_index_meta);
// terms
if (query_type == InvertedIndexQueryType::MATCH_REGEXP_QUERY) {
@@ -387,9 +391,10 @@ InvertedIndexReaderType FullTextIndexReader::type() {
return InvertedIndexReaderType::FULLTEXT;
}
-Status StringTypeInvertedIndexReader::new_iterator(
- const io::IOContext& io_ctx, OlapReaderStatistics* stats, RuntimeState* runtime_state,
- std::unique_ptr<InvertedIndexIterator>* iterator) {
+Status StringTypeInvertedIndexReader::new_iterator(const io::IOContext& io_ctx,
+ OlapReaderStatistics* stats,
+ RuntimeState* runtime_state,
+ std::unique_ptr<IndexIterator>* iterator) {
*iterator =
InvertedIndexIterator::create_unique(io_ctx, stats, runtime_state, shared_from_this());
return Status::OK();
@@ -420,7 +425,7 @@ Status StringTypeInvertedIndexReader::query(const io::IOContext* io_ctx,
VLOG_DEBUG << "begin to query the inverted index from clucene"
<< ", column_name: " << column_name << ", search_str: " << search_str;
- auto index_file_key = _inverted_index_file_reader->get_index_file_cache_key(&_index_meta);
+ auto index_file_key = _index_file_reader->get_index_file_cache_key(&_index_meta);
// try to get query bitmap result from cache and return immediately on cache hit
InvertedIndexQueryCache::CacheKey cache_key {index_file_key, column_name, query_type,
search_str};
@@ -527,7 +532,7 @@ InvertedIndexReaderType StringTypeInvertedIndexReader::type() {
Status BkdIndexReader::new_iterator(const io::IOContext& io_ctx, OlapReaderStatistics* stats,
RuntimeState* runtime_state,
- std::unique_ptr<InvertedIndexIterator>* iterator) {
+ std::unique_ptr<IndexIterator>* iterator) {
*iterator =
InvertedIndexIterator::create_unique(io_ctx, stats, runtime_state, shared_from_this());
return Status::OK();
@@ -666,14 +671,14 @@ Status BkdIndexReader::try_query(const io::IOContext* io_ctx, OlapReaderStatisti
auto st = get_bkd_reader(r, io_ctx, stats, runtime_state);
if (!st.ok()) {
LOG(WARNING) << "get bkd reader for "
- << _inverted_index_file_reader->get_index_file_path(&_index_meta)
+ << _index_file_reader->get_index_file_path(&_index_meta)
<< " failed: " << st;
return st;
}
std::string query_str;
_value_key_coder->full_encode_ascending(query_value, &query_str);
- auto index_file_key = _inverted_index_file_reader->get_index_file_cache_key(&_index_meta);
+ auto index_file_key = _index_file_reader->get_index_file_cache_key(&_index_meta);
InvertedIndexQueryCache::CacheKey cache_key {index_file_key, column_name, query_type,
query_str};
auto* cache = InvertedIndexQueryCache::instance();
@@ -707,14 +712,14 @@ Status BkdIndexReader::query(const io::IOContext* io_ctx, OlapReaderStatistics*
auto st = get_bkd_reader(r, io_ctx, stats, runtime_state);
if (!st.ok()) {
LOG(WARNING) << "get bkd reader for "
- << _inverted_index_file_reader->get_index_file_path(&_index_meta)
+ << _index_file_reader->get_index_file_path(&_index_meta)
<< " failed: " << st;
return st;
}
std::string query_str;
_value_key_coder->full_encode_ascending(query_value, &query_str);
- auto index_file_key = _inverted_index_file_reader->get_index_file_cache_key(&_index_meta);
+ auto index_file_key = _index_file_reader->get_index_file_cache_key(&_index_meta);
InvertedIndexQueryCache::CacheKey cache_key {index_file_key, column_name, query_type,
query_str};
auto* cache = InvertedIndexQueryCache::instance();
@@ -734,8 +739,8 @@ Status BkdIndexReader::query(const io::IOContext* io_ctx, OlapReaderStatistics*
return Status::OK();
} catch (const CLuceneError& e) {
- LOG(ERROR) << "BKD Query CLuceneError Occurred, error msg: " << e.what() << " file_path:"
- << _inverted_index_file_reader->get_index_file_path(&_index_meta);
+ LOG(ERROR) << "BKD Query CLuceneError Occurred, error msg: " << e.what()
+ << " file_path:" << _index_file_reader->get_index_file_path(&_index_meta);
return Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>(
"BKD Query CLuceneError Occurred, error msg: {}", e.what());
}
@@ -1150,78 +1155,6 @@ lucene::util::bkd::relation InvertedIndexVisitor<QT>::compare(std::vector<uint8_
}
}
-Status InvertedIndexIterator::read_from_inverted_index(
- const std::string& column_name, const void* query_value, InvertedIndexQueryType query_type,
- uint32_t segment_num_rows, std::shared_ptr<roaring::Roaring>& bit_map, bool skip_try) {
- DBUG_EXECUTE_IF("return_inverted_index_bypass", {
- return Status::Error<ErrorCode::INVERTED_INDEX_BYPASS>("inverted index bypass");
- });
- if (UNLIKELY(_reader == nullptr)) {
- throw CLuceneError(CL_ERR_NullPointer, "bkd index reader is null", false);
- }
- if (!skip_try && _reader->type() == InvertedIndexReaderType::BKD) {
- if (_runtime_state != nullptr &&
- _runtime_state->query_options().inverted_index_skip_threshold > 0 &&
- _runtime_state->query_options().inverted_index_skip_threshold < 100) {
- auto query_bkd_limit_percent =
- _runtime_state->query_options().inverted_index_skip_threshold;
- uint32_t hit_count = 0;
- RETURN_IF_ERROR(
- try_read_from_inverted_index(column_name, query_value, query_type, &hit_count));
- if (hit_count > segment_num_rows * query_bkd_limit_percent / 100) {
- return Status::Error<ErrorCode::INVERTED_INDEX_BYPASS>(
- "hit count: {}, bkd inverted reached limit {}% , segment num "
- "rows:{}", // add blackspace after % to avoid log4j format bug
- hit_count, query_bkd_limit_percent, segment_num_rows);
- }
- }
- }
-
- auto execute_query = [&]() {
- return _reader->query(&_io_ctx, _stats, _runtime_state, column_name, query_value,
- query_type, bit_map);
- };
-
- if (_runtime_state->query_options().enable_profile) {
- InvertedIndexQueryStatistics query_stats;
- {
- SCOPED_RAW_TIMER(&query_stats.exec_time);
- RETURN_IF_ERROR(execute_query());
- }
- query_stats.column_name = column_name;
- query_stats.hit_rows = bit_map->cardinality();
- _stats->inverted_index_stats.stats.emplace_back(query_stats);
- } else {
- RETURN_IF_ERROR(execute_query());
- }
-
- return Status::OK();
-}
-
-Status InvertedIndexIterator::try_read_from_inverted_index(const std::string& column_name,
- const void* query_value,
- InvertedIndexQueryType query_type,
- uint32_t* count) {
- // NOTE: only bkd index support try read now.
- if (query_type == InvertedIndexQueryType::GREATER_EQUAL_QUERY ||
- query_type == InvertedIndexQueryType::GREATER_THAN_QUERY ||
- query_type == InvertedIndexQueryType::LESS_EQUAL_QUERY ||
- query_type == InvertedIndexQueryType::LESS_THAN_QUERY ||
- query_type == InvertedIndexQueryType::EQUAL_QUERY) {
- RETURN_IF_ERROR(_reader->try_query(&_io_ctx, _stats, _runtime_state, column_name,
- query_value, query_type, count));
- }
- return Status::OK();
-}
-
-InvertedIndexReaderType InvertedIndexIterator::get_inverted_index_reader_type() const {
- return _reader->type();
-}
-
-const std::map<string, string>& InvertedIndexIterator::get_index_properties() const {
- return _reader->get_index_properties();
-}
-
template class InvertedIndexVisitor<InvertedIndexQueryType::LESS_THAN_QUERY>;
template class InvertedIndexVisitor<InvertedIndexQueryType::EQUAL_QUERY>;
template class InvertedIndexVisitor<InvertedIndexQueryType::LESS_EQUAL_QUERY>;
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_reader.h b/be/src/olap/rowset/segment_v2/inverted_index_reader.h
index 6ee29354743..7aa94dc18f5 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_reader.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index_reader.h
@@ -28,6 +28,7 @@
#include "io/fs/file_system.h"
#include "io/fs/path.h"
#include "olap/inverted_index_parser.h"
+#include "olap/rowset/segment_v2/index_reader.h"
#include "olap/rowset/segment_v2/inverted_index_cache.h"
#include "olap/rowset/segment_v2/inverted_index_compound_reader.h"
#include "olap/rowset/segment_v2/inverted_index_desc.h"
@@ -71,8 +72,10 @@ namespace segment_v2 {
class InvertedIndexIterator;
class InvertedIndexQueryCacheHandle;
-class InvertedIndexFileReader;
+class XIndexFileReader;
struct InvertedIndexQueryInfo;
+class IndexIterator;
+
class InvertedIndexResultBitmap {
private:
std::shared_ptr<roaring::Roaring> _data_bitmap = nullptr;
@@ -171,20 +174,13 @@ public:
bool is_empty() const { return (_data_bitmap == nullptr && _null_bitmap == nullptr); }
};
-class InvertedIndexReader : public std::enable_shared_from_this<InvertedIndexReader>,
- public MetadataAdder<InvertedIndexReader> {
+class InvertedIndexReader : public IndexReader {
public:
- explicit InvertedIndexReader(
- const TabletIndex* index_meta,
- std::shared_ptr<InvertedIndexFileReader> inverted_index_file_reader)
- : _inverted_index_file_reader(std::move(inverted_index_file_reader)),
- _index_meta(*index_meta) {}
+ explicit InvertedIndexReader(const TabletIndex* index_meta,
+ std::shared_ptr<XIndexFileReader> index_file_reader)
+ : _index_file_reader(std::move(index_file_reader)), _index_meta(*index_meta) {}
virtual ~InvertedIndexReader() = default;
- // create a new column iterator. Client should delete returned iterator
- virtual Status new_iterator(const io::IOContext& io_ctx, OlapReaderStatistics* stats,
- RuntimeState* runtime_state,
- std::unique_ptr<InvertedIndexIterator>* iterator) = 0;
virtual Status query(const io::IOContext* io_ctx, OlapReaderStatistics* stats,
RuntimeState* runtime_state, const std::string& column_name,
const void* query_value, InvertedIndexQueryType query_type,
@@ -200,7 +196,7 @@ public:
virtual InvertedIndexReaderType type() = 0;
- [[nodiscard]] uint64_t get_index_id() const { return _index_meta.index_id(); }
+ [[nodiscard]] uint64_t get_index_id() const override { return _index_meta.index_id(); }
[[nodiscard]] const std::map<string, string>& get_index_properties() const {
return _index_meta.properties();
@@ -223,6 +219,12 @@ public:
lucene::store::Directory* dir, IndexSearcherPtr* searcher,
size_t& reader_size);
+ bool is_fulltext_index() override { return type() == InvertedIndexReaderType::FULLTEXT; }
+ bool is_string_index() override { return type() == InvertedIndexReaderType::STRING_TYPE; }
+ bool is_bkd_index() override { return type() == InvertedIndexReaderType::BKD; }
+
+ bool is_support_phrase() override;
+
protected:
Status match_index_search(const io::IOContext* io_ctx, OlapReaderStatistics* stats,
RuntimeState* runtime_state, InvertedIndexQueryType query_type,
@@ -231,7 +233,7 @@ protected:
const std::shared_ptr<roaring::Roaring>& term_match_bitmap);
friend class InvertedIndexIterator;
- std::shared_ptr<InvertedIndexFileReader> _inverted_index_file_reader;
+ std::shared_ptr<XIndexFileReader> _index_file_reader;
TabletIndex _index_meta;
bool _has_null = true;
};
@@ -241,15 +243,14 @@ class FullTextIndexReader : public InvertedIndexReader {
ENABLE_FACTORY_CREATOR(FullTextIndexReader);
public:
- explicit FullTextIndexReader(
- const TabletIndex* index_meta,
- std::shared_ptr<InvertedIndexFileReader>& inverted_index_file_reader)
- : InvertedIndexReader(index_meta, inverted_index_file_reader) {}
+ explicit FullTextIndexReader(const TabletIndex* index_meta,
+ std::shared_ptr<XIndexFileReader>& index_file_reader)
+ : InvertedIndexReader(index_meta, index_file_reader) {}
~FullTextIndexReader() override = default;
Status new_iterator(const io::IOContext& io_ctx, OlapReaderStatistics* stats,
RuntimeState* runtime_state,
- std::unique_ptr<InvertedIndexIterator>* iterator) override;
+ std::unique_ptr<IndexIterator>* iterator) override;
Status query(const io::IOContext* io_ctx, OlapReaderStatistics* stats,
RuntimeState* runtime_state, const std::string& column_name,
const void* query_value, InvertedIndexQueryType query_type,
@@ -269,15 +270,14 @@ class StringTypeInvertedIndexReader : public InvertedIndexReader {
ENABLE_FACTORY_CREATOR(StringTypeInvertedIndexReader);
public:
- explicit StringTypeInvertedIndexReader(
- const TabletIndex* index_meta,
- std::shared_ptr<InvertedIndexFileReader>& inverted_index_file_reader)
- : InvertedIndexReader(index_meta, inverted_index_file_reader) {}
+ explicit StringTypeInvertedIndexReader(const TabletIndex* index_meta,
+ std::shared_ptr<XIndexFileReader>& index_file_reader)
+ : InvertedIndexReader(index_meta, index_file_reader) {}
~StringTypeInvertedIndexReader() override = default;
Status new_iterator(const io::IOContext& io_ctx, OlapReaderStatistics* stats,
RuntimeState* runtime_state,
- std::unique_ptr<InvertedIndexIterator>* iterator) override;
+ std::unique_ptr<IndexIterator>* iterator) override;
Status query(const io::IOContext* io_ctx, OlapReaderStatistics* stats,
RuntimeState* runtime_state, const std::string& column_name,
const void* query_value, InvertedIndexQueryType query_type,
@@ -332,13 +332,13 @@ class BkdIndexReader : public InvertedIndexReader {
public:
explicit BkdIndexReader(const TabletIndex* index_meta,
- std::shared_ptr<InvertedIndexFileReader>& inverted_index_file_reader)
- : InvertedIndexReader(index_meta, inverted_index_file_reader) {}
+ std::shared_ptr<XIndexFileReader>& index_file_reader)
+ : InvertedIndexReader(index_meta, index_file_reader) {}
~BkdIndexReader() override = default;
Status new_iterator(const io::IOContext& io_ctx, OlapReaderStatistics* stats,
RuntimeState* runtime_state,
- std::unique_ptr<InvertedIndexIterator>* iterator) override;
+ std::unique_ptr<IndexIterator>* iterator) override;
Status query(const io::IOContext* io_ctx, OlapReaderStatistics* stats,
RuntimeState* runtime_state, const std::string& column_name,
@@ -444,41 +444,5 @@ private:
storage_val _value;
};
-class InvertedIndexIterator {
- ENABLE_FACTORY_CREATOR(InvertedIndexIterator);
-
-public:
- InvertedIndexIterator(const io::IOContext& io_ctx, OlapReaderStatistics* stats,
- RuntimeState* runtime_state, std::shared_ptr<InvertedIndexReader> reader)
- : _io_ctx(io_ctx),
- _stats(stats),
- _runtime_state(runtime_state),
- _reader(std::move(reader)) {}
-
- Status read_from_inverted_index(const std::string& column_name, const void* query_value,
- InvertedIndexQueryType query_type, uint32_t segment_num_rows,
- std::shared_ptr<roaring::Roaring>& bit_map,
- bool skip_try = false);
- Status try_read_from_inverted_index(const std::string& column_name, const void* query_value,
- InvertedIndexQueryType query_type, uint32_t* count);
-
- Status read_null_bitmap(InvertedIndexQueryCacheHandle* cache_handle,
- lucene::store::Directory* dir = nullptr) {
- return _reader->read_null_bitmap(&_io_ctx, _stats, cache_handle, dir);
- }
-
- [[nodiscard]] InvertedIndexReaderType get_inverted_index_reader_type() const;
- [[nodiscard]] const std::map<string, string>& get_index_properties() const;
- [[nodiscard]] bool has_null() { return _reader->has_null(); };
-
- const InvertedIndexReaderPtr& reader() { return _reader; }
-
-private:
- io::IOContext _io_ctx;
- OlapReaderStatistics* _stats = nullptr;
- RuntimeState* _runtime_state = nullptr;
- std::shared_ptr<InvertedIndexReader> _reader;
-};
-
} // namespace segment_v2
} // namespace doris
diff --git a/be/src/olap/rowset/segment_v2/segment.cpp b/be/src/olap/rowset/segment_v2/segment.cpp
index 90135b5826d..3e1120b1d71 100644
--- a/be/src/olap/rowset/segment_v2/segment.cpp
+++ b/be/src/olap/rowset/segment_v2/segment.cpp
@@ -46,12 +46,12 @@
#include "olap/rowset/segment_v2/empty_segment_iterator.h"
#include "olap/rowset/segment_v2/hierarchical_data_reader.h"
#include "olap/rowset/segment_v2/indexed_column_reader.h"
-#include "olap/rowset/segment_v2/inverted_index_file_reader.h"
#include "olap/rowset/segment_v2/page_io.h"
#include "olap/rowset/segment_v2/page_pointer.h"
#include "olap/rowset/segment_v2/segment_iterator.h"
#include "olap/rowset/segment_v2/segment_writer.h" // k_segment_magic_length
#include "olap/rowset/segment_v2/stream_reader.h"
+#include "olap/rowset/segment_v2/x_index_file_reader.h"
#include "olap/schema.h"
#include "olap/short_key_index.h"
#include "olap/tablet_schema.h"
@@ -210,7 +210,7 @@ Status Segment::_open() {
}
Status Segment::_open_inverted_index() {
- _inverted_index_file_reader = std::make_shared<InvertedIndexFileReader>(
+ _index_file_reader = std::make_shared<XIndexFileReader>(
_fs,
std::string {InvertedIndexDescriptor::get_index_file_path_prefix(
_file_reader->path().native())},
@@ -553,8 +553,8 @@ Status Segment::healthy_status() {
if (_create_column_readers_once_call.has_called()) {
RETURN_IF_ERROR(_create_column_readers_once_call.stored_result());
}
- if (_inverted_index_file_reader_open.has_called()) {
- RETURN_IF_ERROR(_inverted_index_file_reader_open.stored_result());
+ if (_index_file_reader_open.has_called()) {
+ RETURN_IF_ERROR(_index_file_reader_open.stored_result());
}
// This status is set by running time, for example, if there is something wrong during read segment iterator.
return _healthy_status.status();
@@ -953,23 +953,21 @@ Status Segment::new_bitmap_index_iterator(const TabletColumn& tablet_column,
return Status::OK();
}
-Status Segment::new_inverted_index_iterator(const TabletColumn& tablet_column,
- const TabletIndex* index_meta,
- const StorageReadOptions& read_options,
- std::unique_ptr<InvertedIndexIterator>* iter) {
+Status Segment::new_index_iterator(const TabletColumn& tablet_column, const TabletIndex* index_meta,
+ const StorageReadOptions& read_options,
+ std::unique_ptr<IndexIterator>* iter) {
if (read_options.runtime_state != nullptr) {
_be_exec_version = read_options.runtime_state->be_exec_version();
}
RETURN_IF_ERROR(_create_column_readers_once(read_options.stats));
ColumnReader* reader = _get_column_reader(tablet_column);
if (reader != nullptr && index_meta) {
- // call DorisCallOnce.call without check if _inverted_index_file_reader is nullptr
+ // call DorisCallOnce.call without check if _index_file_reader is nullptr
// to avoid data race during parallel method calls
+ RETURN_IF_ERROR(_index_file_reader_open.call([&] { return _open_inverted_index(); }));
+ // after DorisCallOnce.call, _index_file_reader is guaranteed to be not nullptr
RETURN_IF_ERROR(
- _inverted_index_file_reader_open.call([&] { return _open_inverted_index(); }));
- // after DorisCallOnce.call, _inverted_index_file_reader is guaranteed to be not nullptr
- RETURN_IF_ERROR(reader->new_inverted_index_iterator(_inverted_index_file_reader, index_meta,
- read_options, iter));
+ reader->new_index_iterator(_index_file_reader, index_meta, read_options, iter));
return Status::OK();
}
return Status::OK();
diff --git a/be/src/olap/rowset/segment_v2/segment.h b/be/src/olap/rowset/segment_v2/segment.h
index 382db37976c..245c87cc0bd 100644
--- a/be/src/olap/rowset/segment_v2/segment.h
+++ b/be/src/olap/rowset/segment_v2/segment.h
@@ -67,7 +67,8 @@ namespace segment_v2 {
class BitmapIndexIterator;
class Segment;
class InvertedIndexIterator;
-class InvertedIndexFileReader;
+class XIndexFileReader;
+class IndexIterator;
using SegmentSharedPtr = std::shared_ptr<Segment>;
// A Segment is used to represent a segment in memory format. When segment is
@@ -122,10 +123,9 @@ public:
const StorageReadOptions& read_options,
std::unique_ptr<BitmapIndexIterator>* iter);
- Status new_inverted_index_iterator(const TabletColumn& tablet_column,
- const TabletIndex* index_meta,
- const StorageReadOptions& read_options,
- std::unique_ptr<InvertedIndexIterator>* iter);
+ Status new_index_iterator(const TabletColumn& tablet_column, const TabletIndex* index_meta,
+ const StorageReadOptions& read_options,
+ std::unique_ptr<IndexIterator>* iter);
const ShortKeyIndexDecoder* get_short_key_index() const {
DCHECK(_load_index_once.has_called() && _load_index_once.stored_result().ok());
@@ -305,8 +305,8 @@ private:
std::unique_ptr<PrimaryKeyIndexReader> _pk_index_reader;
std::mutex _open_lock;
// inverted index file reader
- std::shared_ptr<InvertedIndexFileReader> _inverted_index_file_reader;
- DorisCallOnce<Status> _inverted_index_file_reader_open;
+ std::shared_ptr<XIndexFileReader> _index_file_reader;
+ DorisCallOnce<Status> _index_file_reader_open;
InvertedIndexFileInfo _idx_file_info;
diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
index 00306091876..7789efaa6f8 100644
--- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
@@ -50,11 +50,12 @@
#include "olap/primary_key_index.h"
#include "olap/rowset/segment_v2/bitmap_index_reader.h"
#include "olap/rowset/segment_v2/column_reader.h"
+#include "olap/rowset/segment_v2/index_iterator.h"
#include "olap/rowset/segment_v2/indexed_column_reader.h"
-#include "olap/rowset/segment_v2/inverted_index_file_reader.h"
#include "olap/rowset/segment_v2/inverted_index_reader.h"
#include "olap/rowset/segment_v2/row_ranges.h"
#include "olap/rowset/segment_v2/segment.h"
+#include "olap/rowset/segment_v2/x_index_file_reader.h"
#include "olap/schema.h"
#include "olap/short_key_index.h"
#include "olap/tablet_schema.h"
@@ -261,7 +262,7 @@ SegmentIterator::SegmentIterator(std::shared_ptr<Segment> segment, SchemaSPtr sc
_schema(schema),
_column_iterators(_schema->num_columns()),
_bitmap_index_iterators(_schema->num_columns()),
- _inverted_index_iterators(_schema->num_columns()),
+ _index_iterators(_schema->num_columns()),
_cur_rowid(0),
_lazy_materialization_read(false),
_lazy_inited(false),
@@ -368,7 +369,7 @@ void SegmentIterator::_initialize_predicate_results() {
Status SegmentIterator::init_iterators() {
RETURN_IF_ERROR(_init_return_column_iterators());
RETURN_IF_ERROR(_init_bitmap_index_iterators());
- RETURN_IF_ERROR(_init_inverted_index_iterators());
+ RETURN_IF_ERROR(_init_index_iterators());
return Status::OK();
}
@@ -508,7 +509,7 @@ Status SegmentIterator::_get_row_ranges_by_column_conditions() {
{
if (_opts.runtime_state &&
_opts.runtime_state->query_options().enable_inverted_index_query &&
- has_inverted_index_in_iterators()) {
+ has_index_in_iterators()) {
SCOPED_RAW_TIMER(&_opts.stats->inverted_index_filter_timer);
size_t input_rows = _row_bitmap.cardinality();
RETURN_IF_ERROR(_apply_inverted_index());
@@ -755,7 +756,7 @@ bool SegmentIterator::_check_apply_by_inverted_index(ColumnPredicate* pred) {
return false;
}
auto pred_column_id = pred->column_id();
- if (_inverted_index_iterators[pred_column_id] == nullptr) {
+ if (_index_iterators[pred_column_id] == nullptr) {
//this column without inverted index
return false;
}
@@ -773,10 +774,12 @@ bool SegmentIterator::_check_apply_by_inverted_index(ColumnPredicate* pred) {
// UNTOKENIZED strings exceed ignore_above, they are written as null, causing range query errors
if (PredicateTypeTraits::is_range(pred->type()) &&
- _inverted_index_iterators[pred_column_id] != nullptr &&
- _inverted_index_iterators[pred_column_id]->get_inverted_index_reader_type() ==
- InvertedIndexReaderType::STRING_TYPE) {
- return false;
+ _index_iterators[pred_column_id] != nullptr) {
+ if (_index_iterators[pred_column_id]->type() == IndexType::INVERTED) {
+ if (_index_iterators[pred_column_id]->get_reader()->is_string_index()) {
+ return false;
+ }
+ }
}
// Function filter no apply inverted index
@@ -852,9 +855,12 @@ bool SegmentIterator::_downgrade_without_index(Status res, bool need_remaining)
}
bool SegmentIterator::_column_has_fulltext_index(int32_t cid) {
- bool has_fulltext_index = _inverted_index_iterators[cid] != nullptr &&
- _inverted_index_iterators[cid]->get_inverted_index_reader_type() ==
- InvertedIndexReaderType::FULLTEXT;
+ if (_index_iterators[cid]->type() != IndexType::INVERTED) {
+ return false;
+ }
+
+ bool has_fulltext_index = _index_iterators[cid] != nullptr &&
+ _index_iterators[cid]->get_reader()->is_fulltext_index();
return has_fulltext_index;
}
@@ -871,9 +877,9 @@ Status SegmentIterator::_apply_inverted_index_on_column_predicate(
} else {
bool need_remaining_after_evaluate = _column_has_fulltext_index(pred->column_id()) &&
PredicateTypeTraits::is_equal_or_list(pred->type());
- Status res = pred->evaluate(_storage_name_and_type[pred->column_id()],
- _inverted_index_iterators[pred->column_id()].get(), num_rows(),
- &_row_bitmap);
+ Status res =
+ pred->evaluate(_storage_name_and_type[pred->column_id()],
+ _index_iterators[pred->column_id()].get(), num_rows(), &_row_bitmap);
if (!res.ok()) {
if (_downgrade_without_index(res, need_remaining_after_evaluate)) {
remaining_predicates.emplace_back(pred);
@@ -1068,24 +1074,24 @@ Status SegmentIterator::_init_bitmap_index_iterators() {
return Status::OK();
}
-Status SegmentIterator::_init_inverted_index_iterators() {
- SCOPED_RAW_TIMER(&_opts.stats->segment_iterator_init_inverted_index_iterators_timer_ns);
+Status SegmentIterator::_init_index_iterators() {
+ SCOPED_RAW_TIMER(&_opts.stats->segment_iterator_init_index_iterators_timer_ns);
if (_cur_rowid >= num_rows()) {
return Status::OK();
}
for (auto cid : _schema->column_ids()) {
// Use segment’s own index_meta, for compatibility with future indexing needs to default to lowercase.
- if (_inverted_index_iterators[cid] == nullptr) {
+ if (_index_iterators[cid] == nullptr) {
// In the _opts.tablet_schema, the sub-column type information for the variant is FieldType::OLAP_FIELD_TYPE_VARIANT.
// This is because the sub-column is created in create_materialized_variant_column.
// We use this column to locate the metadata for the inverted index, which requires a unique_id and path.
const auto& column = _opts.tablet_schema->column(cid);
int32_t col_unique_id =
column.is_extracted_column() ? column.parent_unique_id() : column.unique_id();
- RETURN_IF_ERROR(_segment->new_inverted_index_iterator(
+ RETURN_IF_ERROR(_segment->new_index_iterator(
column,
_segment->_tablet_schema->inverted_index(col_unique_id, column.suffix_path()),
- _opts, &_inverted_index_iterators[cid]));
+ _opts, &_index_iterators[cid]));
}
}
return Status::OK();
@@ -2010,7 +2016,7 @@ Status SegmentIterator::copy_column_data_by_selector(vectorized::IColumn* input_
void SegmentIterator::_clear_iterators() {
_column_iterators.clear();
_bitmap_index_iterators.clear();
- _inverted_index_iterators.clear();
+ _index_iterators.clear();
}
Status SegmentIterator::_next_batch_internal(vectorized::Block* block) {
@@ -2440,7 +2446,7 @@ Status SegmentIterator::current_block_row_locations(std::vector<RowLocation>* bl
Status SegmentIterator::_construct_compound_expr_context() {
auto inverted_index_context = std::make_shared<vectorized::InvertedIndexContext>(
- _schema->column_ids(), _inverted_index_iterators, _storage_name_and_type,
+ _schema->column_ids(), _index_iterators, _storage_name_and_type,
_common_expr_inverted_index_status);
for (const auto& expr_ctx : _opts.common_expr_ctxs_push_down) {
vectorized::VExprContextSPtr context;
diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.h b/be/src/olap/rowset/segment_v2/segment_iterator.h
index e684ec372d7..774a88fbf7d 100644
--- a/be/src/olap/rowset/segment_v2/segment_iterator.h
+++ b/be/src/olap/rowset/segment_v2/segment_iterator.h
@@ -41,6 +41,7 @@
#include "olap/row_cursor.h"
#include "olap/row_cursor_cell.h"
#include "olap/rowset/segment_v2/common.h"
+#include "olap/rowset/segment_v2/index_iterator.h"
#include "olap/rowset/segment_v2/segment.h"
#include "olap/schema.h"
#include "util/runtime_profile.h"
@@ -69,6 +70,8 @@ class BitmapIndexIterator;
class ColumnIterator;
class InvertedIndexIterator;
class RowRanges;
+class IndexIterator;
+
struct ColumnPredicateInfo {
ColumnPredicateInfo() = default;
@@ -149,12 +152,8 @@ public:
}
}
- std::vector<std::unique_ptr<InvertedIndexIterator>>& inverted_index_iterators() {
- return _inverted_index_iterators;
- }
-
- bool has_inverted_index_in_iterators() const {
- return std::any_of(_inverted_index_iterators.begin(), _inverted_index_iterators.end(),
+ bool has_index_in_iterators() const {
+ return std::any_of(_index_iterators.begin(), _index_iterators.end(),
[](const auto& iterator) { return iterator != nullptr; });
}
@@ -178,7 +177,7 @@ private:
[[nodiscard]] Status _init_impl(const StorageReadOptions& opts);
[[nodiscard]] Status _init_return_column_iterators();
[[nodiscard]] Status _init_bitmap_index_iterators();
- [[nodiscard]] Status _init_inverted_index_iterators();
+ [[nodiscard]] Status _init_index_iterators();
// calculate row ranges that fall into requested key ranges using short key index
[[nodiscard]] Status _get_row_ranges_by_keys();
[[nodiscard]] Status _prepare_seek(const StorageReadOptions::KeyRange& key_range);
@@ -392,7 +391,7 @@ private:
// vector idx -> column iterarator
std::vector<std::unique_ptr<ColumnIterator>> _column_iterators;
std::vector<std::unique_ptr<BitmapIndexIterator>> _bitmap_index_iterators;
- std::vector<std::unique_ptr<InvertedIndexIterator>> _inverted_index_iterators;
+ std::vector<std::unique_ptr<IndexIterator>> _index_iterators;
// after init(), `_row_bitmap` contains all rowid to scan
roaring::Roaring _row_bitmap;
// an iterator for `_row_bitmap` that can be used to extract row range to scan
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_file_reader.cpp b/be/src/olap/rowset/segment_v2/x_index_file_reader.cpp
similarity index 93%
rename from be/src/olap/rowset/segment_v2/inverted_index_file_reader.cpp
rename to be/src/olap/rowset/segment_v2/x_index_file_reader.cpp
index 30c3e178732..08a8ee074a5 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_file_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/x_index_file_reader.cpp
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-#include "olap/rowset/segment_v2/inverted_index_file_reader.h"
+#include "olap/rowset/segment_v2/x_index_file_reader.h"
#include <memory>
#include <utility>
@@ -27,7 +27,7 @@
namespace doris::segment_v2 {
-Status InvertedIndexFileReader::init(int32_t read_buffer_size, const io::IOContext* io_ctx) {
+Status XIndexFileReader::init(int32_t read_buffer_size, const io::IOContext* io_ctx) {
std::unique_lock<std::shared_mutex> lock(_mutex); // Lock for writing
if (!_inited) {
_read_buffer_size = read_buffer_size;
@@ -39,7 +39,7 @@ Status InvertedIndexFileReader::init(int32_t read_buffer_size, const io::IOConte
return Status::OK();
}
-Status InvertedIndexFileReader::_init_from(int32_t read_buffer_size, const io::IOContext* io_ctx) {
+Status XIndexFileReader::_init_from(int32_t read_buffer_size, const io::IOContext* io_ctx) {
auto index_file_full_path = InvertedIndexDescriptor::get_index_file_path_v2(_index_path_prefix);
try {
@@ -135,7 +135,7 @@ Status InvertedIndexFileReader::_init_from(int32_t read_buffer_size, const io::I
return Status::OK();
}
-Result<InvertedIndexDirectoryMap> InvertedIndexFileReader::get_all_directories() {
+Result<InvertedIndexDirectoryMap> XIndexFileReader::get_all_directories() {
InvertedIndexDirectoryMap res;
std::shared_lock<std::shared_mutex> lock(_mutex); // Lock for reading
for (auto& [index, _] : _indices_entries) {
@@ -150,7 +150,7 @@ Result<InvertedIndexDirectoryMap> InvertedIndexFileReader::get_all_directories()
return res;
}
-Result<std::unique_ptr<DorisCompoundReader>> InvertedIndexFileReader::_open(
+Result<std::unique_ptr<DorisCompoundReader>> XIndexFileReader::_open(
int64_t index_id, const std::string& index_suffix, const io::IOContext* io_ctx) const {
std::unique_ptr<DorisCompoundReader> compound_reader;
@@ -226,19 +226,19 @@ Result<std::unique_ptr<DorisCompoundReader>> InvertedIndexFileReader::_open(
return compound_reader;
}
-Result<std::unique_ptr<DorisCompoundReader>> InvertedIndexFileReader::open(
+Result<std::unique_ptr<DorisCompoundReader>> XIndexFileReader::open(
const TabletIndex* index_meta, const io::IOContext* io_ctx) const {
auto index_id = index_meta->index_id();
auto index_suffix = index_meta->get_index_suffix();
return _open(index_id, index_suffix, io_ctx);
}
-std::string InvertedIndexFileReader::get_index_file_cache_key(const TabletIndex* index_meta) const {
+std::string XIndexFileReader::get_index_file_cache_key(const TabletIndex* index_meta) const {
return InvertedIndexDescriptor::get_index_file_cache_key(
_index_path_prefix, index_meta->index_id(), index_meta->get_index_suffix());
}
-std::string InvertedIndexFileReader::get_index_file_path(const TabletIndex* index_meta) const {
+std::string XIndexFileReader::get_index_file_path(const TabletIndex* index_meta) const {
if (_storage_format == InvertedIndexStorageFormatPB::V1) {
return InvertedIndexDescriptor::get_index_file_path_v1(
_index_path_prefix, index_meta->index_id(), index_meta->get_index_suffix());
@@ -246,7 +246,7 @@ std::string InvertedIndexFileReader::get_index_file_path(const TabletIndex* inde
return InvertedIndexDescriptor::get_index_file_path_v2(_index_path_prefix);
}
-Status InvertedIndexFileReader::index_file_exist(const TabletIndex* index_meta, bool* res) const {
+Status XIndexFileReader::index_file_exist(const TabletIndex* index_meta, bool* res) const {
if (_storage_format == InvertedIndexStorageFormatPB::V1) {
auto index_file_path = InvertedIndexDescriptor::get_index_file_path_v1(
_index_path_prefix, index_meta->index_id(), index_meta->get_index_suffix());
@@ -271,7 +271,7 @@ Status InvertedIndexFileReader::index_file_exist(const TabletIndex* index_meta,
return Status::OK();
}
-Status InvertedIndexFileReader::has_null(const TabletIndex* index_meta, bool* res) const {
+Status XIndexFileReader::has_null(const TabletIndex* index_meta, bool* res) const {
if (_storage_format == InvertedIndexStorageFormatPB::V1) {
*res = true;
return Status::OK();
@@ -305,7 +305,7 @@ Status InvertedIndexFileReader::has_null(const TabletIndex* index_meta, bool* re
return Status::OK();
}
-void InvertedIndexFileReader::debug_file_entries() {
+void XIndexFileReader::debug_file_entries() {
std::shared_lock<std::shared_mutex> lock(_mutex); // Lock for reading
for (auto& index : _indices_entries) {
LOG(INFO) << "index_id:" << index.first.first;
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_file_reader.h b/be/src/olap/rowset/segment_v2/x_index_file_reader.h
similarity index 92%
rename from be/src/olap/rowset/segment_v2/inverted_index_file_reader.h
rename to be/src/olap/rowset/segment_v2/x_index_file_reader.h
index 640336a2ef9..d8fb566c289 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_file_reader.h
+++ b/be/src/olap/rowset/segment_v2/x_index_file_reader.h
@@ -40,7 +40,7 @@ namespace segment_v2 {
class ReaderFileEntry;
class DorisCompoundReader;
-class InvertedIndexFileReader {
+class XIndexFileReader {
public:
using EntriesType =
lucene::util::CLHashMap<char*, ReaderFileEntry*, lucene::util::Compare::Char,
@@ -50,9 +50,9 @@ public:
using IndicesEntriesMap =
std::map<std::pair<int64_t, std::string>, std::unique_ptr<EntriesType>>;
- InvertedIndexFileReader(io::FileSystemSPtr fs, std::string index_path_prefix,
- InvertedIndexStorageFormatPB storage_format,
- InvertedIndexFileInfo idx_file_info = InvertedIndexFileInfo())
+ XIndexFileReader(io::FileSystemSPtr fs, std::string index_path_prefix,
+ InvertedIndexStorageFormatPB storage_format,
+ InvertedIndexFileInfo idx_file_info = InvertedIndexFileInfo())
: _fs(std::move(fs)),
_index_path_prefix(std::move(index_path_prefix)),
_storage_format(storage_format),
diff --git a/be/src/olap/rowset/segment_v2/x_index_file_writer.cpp b/be/src/olap/rowset/segment_v2/x_index_file_writer.cpp
index 6aafe26557d..cb364782c5a 100644
--- a/be/src/olap/rowset/segment_v2/x_index_file_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/x_index_file_writer.cpp
@@ -22,18 +22,40 @@
#include <algorithm>
#include <filesystem>
-#include "common/exception.h"
#include "common/status.h"
#include "io/fs/stream_sink_file_writer.h"
#include "olap/rowset/segment_v2/inverted_index_compound_reader.h"
#include "olap/rowset/segment_v2/inverted_index_desc.h"
-#include "olap/rowset/segment_v2/inverted_index_file_reader.h"
#include "olap/rowset/segment_v2/inverted_index_fs_directory.h"
#include "olap/rowset/segment_v2/inverted_index_reader.h"
+#include "olap/rowset/segment_v2/x_index_file_reader.h"
+#include "olap/rowset/segment_v2/x_index_storage_format_v1.h"
+#include "olap/rowset/segment_v2/x_index_storage_format_v2.h"
#include "olap/tablet_schema.h"
namespace doris::segment_v2 {
+XIndexFileWriter::XIndexFileWriter(io::FileSystemSPtr fs, std::string index_path_prefix,
+ std::string rowset_id, int64_t seg_id,
+ InvertedIndexStorageFormatPB storage_format,
+ io::FileWriterPtr file_writer, bool can_use_ram_dir)
+ : _fs(std::move(fs)),
+ _index_path_prefix(std::move(index_path_prefix)),
+ _rowset_id(std::move(rowset_id)),
+ _seg_id(seg_id),
+ _storage_format(storage_format),
+ _local_fs(io::global_local_filesystem()),
+ _idx_v2_writer(std::move(file_writer)),
+ _can_use_ram_dir(can_use_ram_dir) {
+ auto tmp_file_dir = ExecEnv::GetInstance()->get_tmp_file_dirs()->get_tmp_file_dir();
+ _tmp_dir = tmp_file_dir.native();
+ if (_storage_format == InvertedIndexStorageFormatPB::V1) {
+ _index_storage_format = std::make_unique<XIndexStorageFormatV1>(this);
+ } else {
+ _index_storage_format = std::make_unique<XIndexStorageFormatV2>(this);
+ }
+}
+
Status XIndexFileWriter::initialize(InvertedIndexDirectoryMap& indices_dirs) {
_indices_dirs = std::move(indices_dirs);
return Status::OK();
@@ -95,38 +117,13 @@ Status XIndexFileWriter::delete_index(const TabletIndex* index_meta) {
return Status::OK();
}
-int64_t XIndexFileWriter::headerLength() {
- int64_t header_size = 0;
- header_size +=
- sizeof(int32_t) * 2; // Account for the size of the version number and number of indices
-
- for (const auto& entry : _indices_dirs) {
- const auto& suffix = entry.first.second;
- header_size += sizeof(int64_t); // index id
- header_size += sizeof(int32_t); // index suffix name size
- header_size += suffix.length(); // index suffix name
- header_size += sizeof(int32_t); // index file count
- const auto& dir = entry.second;
- std::vector<std::string> files;
- dir->list(&files);
-
- for (const auto& file : files) {
- header_size += sizeof(int32_t); // file name size
- header_size += file.length(); // file name
- header_size += sizeof(int64_t); // file offset
- header_size += sizeof(int64_t); // file size
- }
- }
- return header_size;
-}
-
Status XIndexFileWriter::add_into_searcher_cache() {
- auto inverted_index_file_reader =
- std::make_unique<InvertedIndexFileReader>(_fs, _index_path_prefix, _storage_format);
- auto st = inverted_index_file_reader->init();
+ auto index_file_reader =
+ std::make_unique<XIndexFileReader>(_fs, _index_path_prefix, _storage_format);
+ auto st = index_file_reader->init();
if (!st.ok()) {
if (dynamic_cast<io::StreamSinkFileWriter*>(_idx_v2_writer.get()) != nullptr) {
- //StreamSinkFileWriter not found file is normal.
+ // StreamSinkFileWriter not found file is normal.
return Status::OK();
}
LOG(WARNING) << "InvertedIndexFileWriter::add_into_searcher_cache for "
@@ -135,8 +132,7 @@ Status XIndexFileWriter::add_into_searcher_cache() {
}
for (const auto& entry : _indices_dirs) {
auto index_meta = entry.first;
- auto dir =
- DORIS_TRY(inverted_index_file_reader->_open(index_meta.first, index_meta.second));
+ auto dir = DORIS_TRY(index_file_reader->_open(index_meta.first, index_meta.second));
auto index_file_key = InvertedIndexDescriptor::get_index_file_cache_key(
_index_path_prefix, index_meta.first, index_meta.second);
InvertedIndexSearcherCache::CacheKey searcher_cache_key(index_file_key);
@@ -189,33 +185,21 @@ Status XIndexFileWriter::close() {
"must be v2");
}
})
- if (_storage_format == InvertedIndexStorageFormatPB::V1) {
- try {
- RETURN_IF_ERROR(write_v1());
- for (const auto& entry : _indices_dirs) {
- const auto& dir = entry.second;
- // delete index path, which contains separated inverted index files
- if (std::strcmp(dir->getObjectName(), "DorisFSDirectory") == 0) {
- auto* compound_dir = static_cast<DorisFSDirectory*>(dir.get());
- compound_dir->deleteDirectory();
- }
+ try {
+ RETURN_IF_ERROR(_index_storage_format->write());
+ for (const auto& entry : _indices_dirs) {
+ const auto& dir = entry.second;
+ // delete index path, which contains separated inverted index files
+ if (std::strcmp(dir->getObjectName(), "DorisFSDirectory") == 0) {
+ auto* compound_dir = static_cast<DorisFSDirectory*>(dir.get());
+ compound_dir->deleteDirectory();
}
- } catch (CLuceneError& err) {
+ }
+ } catch (CLuceneError& err) {
+ if (_storage_format == InvertedIndexStorageFormatPB::V1) {
return Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>(
"CLuceneError occur when close, error msg: {}", err.what());
- }
- } else {
- try {
- RETURN_IF_ERROR(write());
- for (const auto& entry : _indices_dirs) {
- const auto& dir = entry.second;
- // delete index path, which contains separated inverted index files
- if (std::strcmp(dir->getObjectName(), "DorisFSDirectory") == 0) {
- auto* compound_dir = static_cast<DorisFSDirectory*>(dir.get());
- compound_dir->deleteDirectory();
- }
- }
- } catch (CLuceneError& err) {
+ } else {
return Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>(
"CLuceneError occur when close idx file {}, error msg: {}",
InvertedIndexDescriptor::get_index_file_path_v2(_index_path_prefix),
@@ -228,431 +212,18 @@ Status XIndexFileWriter::close() {
return Status::OK();
}
-void XIndexFileWriter::sort_files(std::vector<FileInfo>& file_infos) {
- auto file_priority = [](const std::string& filename) {
- for (const auto& entry : InvertedIndexDescriptor::index_file_info_map) {
- if (filename.find(entry.first) != std::string::npos) {
- return entry.second;
- }
- }
- return std::numeric_limits<int32_t>::max(); // Other files
- };
-
- std::sort(file_infos.begin(), file_infos.end(), [&](const FileInfo& a, const FileInfo& b) {
- int32_t priority_a = file_priority(a.filename);
- int32_t priority_b = file_priority(b.filename);
- if (priority_a != priority_b) {
- return priority_a < priority_b;
- }
- return a.filesize < b.filesize;
- });
-}
-
-void XIndexFileWriter::copyFile(const char* fileName, lucene::store::Directory* dir,
- lucene::store::IndexOutput* output, uint8_t* buffer,
- int64_t bufferLength) {
- lucene::store::IndexInput* tmp = nullptr;
- CLuceneError err;
- auto open = dir->openInput(fileName, tmp, err);
- DBUG_EXECUTE_IF("XIndexFileWriter::copyFile_openInput_error", {
- open = false;
- err.set(CL_ERR_IO, "debug point: copyFile_openInput_error");
- });
- if (!open) {
- throw err;
- }
-
- std::unique_ptr<lucene::store::IndexInput> input(tmp);
- int64_t start_ptr = output->getFilePointer();
- int64_t length = input->length();
- int64_t remainder = length;
- int64_t chunk = bufferLength;
-
- while (remainder > 0) {
- int64_t len = std::min({chunk, length, remainder});
- input->readBytes(buffer, len);
- output->writeBytes(buffer, len);
- remainder -= len;
- }
- DBUG_EXECUTE_IF("XIndexFileWriter::copyFile_remainder_is_not_zero", { remainder = 10; });
- if (remainder != 0) {
- std::ostringstream errMsg;
- errMsg << "Non-zero remainder length after copying: " << remainder << " (id: " << fileName
- << ", length: " << length << ", buffer size: " << chunk << ")";
- err.set(CL_ERR_IO, errMsg.str().c_str());
- throw err;
- }
-
- int64_t end_ptr = output->getFilePointer();
- int64_t diff = end_ptr - start_ptr;
- DBUG_EXECUTE_IF("XIndexFileWriter::copyFile_diff_not_equals_length", { diff = length - 10; });
- if (diff != length) {
- std::ostringstream errMsg;
- errMsg << "Difference in the output file offsets " << diff
- << " does not match the original file length " << length;
- err.set(CL_ERR_IO, errMsg.str().c_str());
- throw err;
- }
- input->close();
-}
-
-Status XIndexFileWriter::write_v1() {
- int64_t total_size = 0;
- std::unique_ptr<lucene::store::Directory, DirectoryDeleter> out_dir = nullptr;
- std::unique_ptr<lucene::store::IndexOutput> output = nullptr;
- ErrorContext error_context;
- for (const auto& entry : _indices_dirs) {
- const int64_t index_id = entry.first.first;
- const auto& index_suffix = entry.first.second;
- try {
- const auto& directory = entry.second.get();
-
- // Prepare sorted file list
- auto sorted_files = prepare_sorted_files(directory);
-
- // Calculate header length
- auto [header_length, header_file_count] =
- calculate_header_length(sorted_files, directory);
-
- // Create output stream
- auto result = create_output_stream_v1(index_id, index_suffix);
- out_dir = std::move(result.first);
- output = std::move(result.second);
-
- size_t start = output->getFilePointer();
- // Write header and data
- write_header_and_data_v1(output.get(), sorted_files, directory, header_length,
- header_file_count);
-
- // Collect file information
- auto compound_file_size = output->getFilePointer() - start;
- total_size += compound_file_size;
- add_index_info(index_id, index_suffix, compound_file_size);
- } catch (CLuceneError& err) {
- error_context.eptr = std::current_exception();
- auto index_path = InvertedIndexDescriptor::get_index_file_path_v1(
- _index_path_prefix, index_id, index_suffix);
- error_context.err_msg.append("CLuceneError occur when write_v1 idx file: ");
- error_context.err_msg.append(index_path);
- error_context.err_msg.append(", error msg: ");
- error_context.err_msg.append(err.what());
- LOG(ERROR) << error_context.err_msg;
- }
- FINALLY({
- FINALLY_CLOSE(output);
- FINALLY_CLOSE(out_dir);
- })
- }
-
- _total_file_size = total_size;
- return Status::OK();
-}
-
-Status XIndexFileWriter::write() {
- std::unique_ptr<lucene::store::Directory, DirectoryDeleter> out_dir = nullptr;
- std::unique_ptr<lucene::store::IndexOutput> compound_file_output = nullptr;
- ErrorContext error_context;
- try {
- // Calculate header length and initialize offset
- int64_t current_offset = headerLength();
- // Prepare file metadata
- auto file_metadata = prepare_file_metadata(current_offset);
-
- // Create output stream
- auto result = create_output_stream();
- out_dir = std::move(result.first);
- compound_file_output = std::move(result.second);
-
- // Write version and number of indices
- write_version_and_indices_count(compound_file_output.get());
-
- // Write index headers and file metadata
- write_index_headers_and_metadata(compound_file_output.get(), file_metadata);
-
- // Copy file data
- copy_files_data(compound_file_output.get(), file_metadata);
-
- _total_file_size = compound_file_output->getFilePointer();
- _file_info.set_index_size(_total_file_size);
- } catch (CLuceneError& err) {
- error_context.eptr = std::current_exception();
- auto index_path = InvertedIndexDescriptor::get_index_file_path_v2(_index_path_prefix);
- error_context.err_msg.append("CLuceneError occur when close idx file: ");
- error_context.err_msg.append(index_path);
- error_context.err_msg.append(", error msg: ");
- error_context.err_msg.append(err.what());
- LOG(ERROR) << error_context.err_msg;
- }
- FINALLY({
- FINALLY_CLOSE(compound_file_output);
- FINALLY_CLOSE(out_dir);
- })
-
- return Status::OK();
-}
-
-// Helper function implementations
-std::vector<FileInfo> XIndexFileWriter::prepare_sorted_files(lucene::store::Directory* directory) {
- std::vector<std::string> files;
- directory->list(&files);
-
- // Remove write.lock file
- files.erase(std::remove(files.begin(), files.end(), DorisFSDirectory::WRITE_LOCK_FILE),
- files.end());
-
- std::vector<FileInfo> sorted_files;
- for (const auto& file : files) {
- FileInfo file_info;
- file_info.filename = file;
- file_info.filesize = directory->fileLength(file.c_str());
- sorted_files.push_back(std::move(file_info));
- }
-
- // Sort the files
- sort_files(sorted_files);
- return sorted_files;
-}
-
-void XIndexFileWriter::add_index_info(int64_t index_id, const std::string& index_suffix,
- int64_t compound_file_size) {
- InvertedIndexFileInfo_IndexInfo index_info;
- index_info.set_index_id(index_id);
- index_info.set_index_suffix(index_suffix);
- index_info.set_index_file_size(compound_file_size);
- auto* new_index_info = _file_info.add_index_info();
- *new_index_info = index_info;
-}
-
-std::pair<int64_t, int32_t> XIndexFileWriter::calculate_header_length(
- const std::vector<FileInfo>& sorted_files, lucene::store::Directory* directory) {
- // Use RAMDirectory to calculate header length
- lucene::store::RAMDirectory ram_dir;
- auto* out_idx = ram_dir.createOutput("temp_idx");
- DBUG_EXECUTE_IF("XIndexFileWriter::calculate_header_length_ram_output_is_nullptr",
- { out_idx = nullptr; })
- if (out_idx == nullptr) {
- LOG(WARNING) << "XIndexFileWriter::calculate_header_length error: RAMDirectory "
- "output is nullptr.";
- _CLTHROWA(CL_ERR_IO, "Create RAMDirectory output error");
- }
- std::unique_ptr<lucene::store::IndexOutput> ram_output(out_idx);
- int32_t file_count = sorted_files.size();
- ram_output->writeVInt(file_count);
-
- int64_t header_file_length = 0;
- const int64_t buffer_length = 16384;
- uint8_t ram_buffer[buffer_length];
- int32_t header_file_count = 0;
- for (const auto& file : sorted_files) {
- ram_output->writeString(file.filename);
- ram_output->writeLong(0);
- ram_output->writeLong(file.filesize);
- header_file_length += file.filesize;
-
- if (header_file_length <= DorisFSDirectory::MAX_HEADER_DATA_SIZE) {
- copyFile(file.filename.c_str(), directory, ram_output.get(), ram_buffer, buffer_length);
- header_file_count++;
- }
- }
-
- int64_t header_length = ram_output->getFilePointer();
- ram_output->close();
- ram_dir.close();
- return {header_length, header_file_count};
-}
-
-std::pair<std::unique_ptr<lucene::store::Directory, DirectoryDeleter>,
- std::unique_ptr<lucene::store::IndexOutput>>
-XIndexFileWriter::create_output_stream_v1(int64_t index_id, const std::string& index_suffix) {
- io::Path cfs_path(InvertedIndexDescriptor::get_index_file_path_v1(_index_path_prefix, index_id,
- index_suffix));
- auto idx_path = cfs_path.parent_path();
- std::string idx_name = cfs_path.filename();
-
- auto* out_dir = DorisFSDirectoryFactory::getDirectory(_fs, idx_path.c_str());
- out_dir->set_file_writer_opts(_opts);
- std::unique_ptr<lucene::store::Directory, DirectoryDeleter> out_dir_ptr(out_dir);
-
- auto* out = out_dir->createOutput(idx_name.c_str());
- DBUG_EXECUTE_IF("XIndexFileWriter::write_v1_out_dir_createOutput_nullptr", { out = nullptr; });
- if (out == nullptr) {
- LOG(WARNING) << "XIndexFileWriter::create_output_stream_v1 error: CompoundDirectory "
- "output is nullptr.";
- _CLTHROWA(CL_ERR_IO, "Create CompoundDirectory output error");
+std::string XIndexFileWriter::debug_string() const {
+ std::stringstream indices_dirs;
+ for (const auto& [index, dir] : _indices_dirs) {
+ indices_dirs << "index id is: " << index.first << " , index suffix is: " << index.second
+ << " , index dir is: " << dir->toString();
}
- std::unique_ptr<lucene::store::IndexOutput> output(out);
-
- return {std::move(out_dir_ptr), std::move(output)};
-}
-
-void XIndexFileWriter::write_header_and_data_v1(lucene::store::IndexOutput* output,
- const std::vector<FileInfo>& sorted_files,
- lucene::store::Directory* directory,
- int64_t header_length, int32_t header_file_count) {
- output->writeVInt(sorted_files.size());
- int64_t data_offset = header_length;
- const int64_t buffer_length = 16384;
- uint8_t buffer[buffer_length];
-
- for (int i = 0; i < sorted_files.size(); ++i) {
- auto file = sorted_files[i];
- output->writeString(file.filename);
-
- // DataOffset
- if (i < header_file_count) {
- // file data write in header, so we set its offset to -1.
- output->writeLong(-1);
- } else {
- output->writeLong(data_offset);
- }
- output->writeLong(file.filesize); // FileLength
- if (i < header_file_count) {
- // append data
- copyFile(file.filename.c_str(), directory, output, buffer, buffer_length);
- } else {
- data_offset += file.filesize;
- }
- }
-
- for (size_t i = header_file_count; i < sorted_files.size(); ++i) {
- copyFile(sorted_files[i].filename.c_str(), directory, output, buffer, buffer_length);
- }
-}
-
-std::pair<std::unique_ptr<lucene::store::Directory, DirectoryDeleter>,
- std::unique_ptr<lucene::store::IndexOutput>>
-XIndexFileWriter::create_output_stream() {
- io::Path index_path {InvertedIndexDescriptor::get_index_file_path_v2(_index_path_prefix)};
-
- auto* out_dir = DorisFSDirectoryFactory::getDirectory(_fs, index_path.parent_path().c_str());
- out_dir->set_file_writer_opts(_opts);
- std::unique_ptr<lucene::store::Directory, DirectoryDeleter> out_dir_ptr(out_dir);
-
- DCHECK(_idx_v2_writer != nullptr) << "inverted index file writer v2 is nullptr";
- auto compound_file_output = std::unique_ptr<lucene::store::IndexOutput>(
- out_dir->createOutputV2(_idx_v2_writer.get()));
-
- return {std::move(out_dir_ptr), std::move(compound_file_output)};
-}
-
-void XIndexFileWriter::write_version_and_indices_count(lucene::store::IndexOutput* output) {
- // Write the version number
- output->writeInt(_storage_format);
-
- // Write the number of indices
- const auto num_indices = static_cast<uint32_t>(_indices_dirs.size());
- output->writeInt(num_indices);
+ return fmt::format(
+ "inverted index file writer debug string: index storage format is: {}, index path "
+ "prefix is: {}, rowset id is: {}, seg id is: {}, closed is: {}, total file size "
+ "is: {}, index dirs is: {}",
+ _storage_format, _index_path_prefix, _rowset_id, _seg_id, _closed, _total_file_size,
+ indices_dirs.str());
}
-std::vector<XIndexFileWriter::FileMetadata> XIndexFileWriter::prepare_file_metadata(
- int64_t& current_offset) {
- std::vector<FileMetadata> file_metadata;
- std::vector<FileMetadata> meta_files;
- std::vector<FileMetadata> normal_files;
- for (const auto& entry : _indices_dirs) {
- const int64_t index_id = entry.first.first;
- const auto& index_suffix = entry.first.second;
- auto* dir = entry.second.get();
-
- auto sorted_files = prepare_sorted_files(dir);
-
- for (const auto& file : sorted_files) {
- bool is_meta = false;
-
- for (const auto& entry : InvertedIndexDescriptor::index_file_info_map) {
- if (file.filename.find(entry.first) != std::string::npos) {
- meta_files.emplace_back(index_id, index_suffix, file.filename, 0, file.filesize,
- dir);
- is_meta = true;
- break;
- }
- }
-
- if (!is_meta) {
- normal_files.emplace_back(index_id, index_suffix, file.filename, 0, file.filesize,
- dir);
- }
- }
- }
-
- file_metadata.reserve(meta_files.size() + normal_files.size());
-
- // meta file
- for (auto& entry : meta_files) {
- entry.offset = current_offset;
- file_metadata.emplace_back(std::move(entry));
- current_offset += entry.length;
- }
- // normal file
- for (auto& entry : normal_files) {
- entry.offset = current_offset;
- file_metadata.emplace_back(std::move(entry));
- current_offset += entry.length;
- }
-
- DBUG_EXECUTE_IF("CSIndexInput.readInternal", {
- bool is_meta_file = true;
- for (const auto& entry : file_metadata) {
- bool is_meta = false;
- for (const auto& map_entry : InvertedIndexDescriptor::index_file_info_map) {
- if (entry.filename.find(map_entry.first) != std::string::npos) {
- is_meta = true;
- break;
- }
- }
-
- if (is_meta_file && !is_meta) {
- is_meta_file = false;
- } else if (!is_meta_file && is_meta) {
- throw Exception(ErrorCode::INTERNAL_ERROR,
- "Invalid file order: meta files must be at the beginning of the "
- "file_metadata structure.");
- }
- }
- });
-
- return file_metadata;
-}
-
-void XIndexFileWriter::write_index_headers_and_metadata(
- lucene::store::IndexOutput* output, const std::vector<FileMetadata>& file_metadata) {
- // Group files by index_id and index_suffix
- std::map<std::pair<int64_t, std::string>, std::vector<FileMetadata>> indices;
-
- for (const auto& meta : file_metadata) {
- indices[{meta.index_id, meta.index_suffix}].push_back(meta);
- }
-
- for (const auto& index_entry : indices) {
- int64_t index_id = index_entry.first.first;
- const std::string& index_suffix = index_entry.first.second;
- const auto& files = index_entry.second;
-
- // Write the index ID and the number of files
- output->writeLong(index_id);
- output->writeInt(static_cast<int32_t>(index_suffix.length()));
- output->writeBytes(reinterpret_cast<const uint8_t*>(index_suffix.data()),
- index_suffix.length());
- output->writeInt(static_cast<int32_t>(files.size()));
-
- // Write file metadata
- for (const auto& file : files) {
- output->writeInt(static_cast<int32_t>(file.filename.length()));
- output->writeBytes(reinterpret_cast<const uint8_t*>(file.filename.data()),
- file.filename.length());
- output->writeLong(file.offset);
- output->writeLong(file.length);
- }
- }
-}
-
-void XIndexFileWriter::copy_files_data(lucene::store::IndexOutput* output,
- const std::vector<FileMetadata>& file_metadata) {
- const int64_t buffer_length = 16384;
- uint8_t buffer[buffer_length];
-
- for (const auto& meta : file_metadata) {
- copyFile(meta.filename.c_str(), meta.directory, output, buffer, buffer_length);
- }
-}
-} // namespace doris::segment_v2
+} // namespace doris::segment_v2
\ No newline at end of file
diff --git a/be/src/olap/rowset/segment_v2/x_index_file_writer.h b/be/src/olap/rowset/segment_v2/x_index_file_writer.h
index 294d2a91441..97ba8956940 100644
--- a/be/src/olap/rowset/segment_v2/x_index_file_writer.h
+++ b/be/src/olap/rowset/segment_v2/x_index_file_writer.h
@@ -24,59 +24,38 @@
#include <string>
#include <utility>
-#include <vector>
#include "io/fs/file_system.h"
#include "io/fs/file_writer.h"
#include "io/fs/local_file_system.h"
#include "olap/rowset/segment_v2/inverted_index_common.h"
#include "olap/rowset/segment_v2/inverted_index_compound_reader.h"
-#include "olap/rowset/segment_v2/inverted_index_desc.h"
#include "olap/rowset/segment_v2/inverted_index_searcher.h"
-#include "runtime/exec_env.h"
+#include "olap/rowset/segment_v2/x_index_storage_format.h"
namespace doris {
class TabletIndex;
namespace segment_v2 {
class DorisFSDirectory;
+
using InvertedIndexDirectoryMap =
std::map<std::pair<int64_t, std::string>, std::shared_ptr<lucene::store::Directory>>;
class XIndexFileWriter;
using XIndexFileWriterPtr = std::unique_ptr<XIndexFileWriter>;
-class FileInfo {
-public:
- std::string filename;
- int64_t filesize;
-};
-
class XIndexFileWriter {
public:
XIndexFileWriter(io::FileSystemSPtr fs, std::string index_path_prefix, std::string rowset_id,
int64_t seg_id, InvertedIndexStorageFormatPB storage_format,
- io::FileWriterPtr file_writer = nullptr, bool can_use_ram_dir = true)
- : _fs(std::move(fs)),
- _index_path_prefix(std::move(index_path_prefix)),
- _rowset_id(std::move(rowset_id)),
- _seg_id(seg_id),
- _storage_format(storage_format),
- _local_fs(io::global_local_filesystem()),
- _idx_v2_writer(std::move(file_writer)),
- _can_use_ram_dir(can_use_ram_dir) {
- auto tmp_file_dir = ExecEnv::GetInstance()->get_tmp_file_dirs()->get_tmp_file_dir();
- _tmp_dir = tmp_file_dir.native();
- }
-
+ io::FileWriterPtr file_writer = nullptr, bool can_use_ram_dir = true);
virtual ~XIndexFileWriter() = default;
Result<std::shared_ptr<DorisFSDirectory>> open(const TabletIndex* index_meta);
Status delete_index(const TabletIndex* index_meta);
Status initialize(InvertedIndexDirectoryMap& indices_dirs);
Status add_into_searcher_cache();
- Status write();
- Status write_v1();
Status close();
const InvertedIndexFileInfo* get_index_file_info() const {
DCHECK(_closed) << debug_string();
@@ -89,71 +68,14 @@ public:
const io::FileSystemSPtr& get_fs() const { return _fs; }
InvertedIndexStorageFormatPB get_storage_format() const { return _storage_format; }
void set_file_writer_opts(const io::FileWriterOptions& opts) { _opts = opts; }
-
- std::string debug_string() const {
- std::stringstream indices_dirs;
- for (const auto& [index, dir] : _indices_dirs) {
- indices_dirs << "index id is: " << index.first << " , index suffix is: " << index.second
- << " , index dir is: " << dir->toString();
- }
- return fmt::format(
- "inverted index file writer debug string: index storage format is: {}, index path "
- "prefix is: {}, rowset id is: {}, seg id is: {}, closed is: {}, total file size "
- "is: {}, index dirs is: {}",
- _storage_format, _index_path_prefix, _rowset_id, _seg_id, _closed, _total_file_size,
- indices_dirs.str());
- }
+ std::string debug_string() const;
private:
- // Helper functions shared between write_v1 and write_v2
- std::vector<FileInfo> prepare_sorted_files(lucene::store::Directory* directory);
- void sort_files(std::vector<FileInfo>& file_infos);
- void copyFile(const char* fileName, lucene::store::Directory* dir,
- lucene::store::IndexOutput* output, uint8_t* buffer, int64_t bufferLength);
- void add_index_info(int64_t index_id, const std::string& index_suffix,
- int64_t compound_file_size);
- int64_t headerLength();
- // Helper functions specific to write_v1
- std::pair<int64_t, int32_t> calculate_header_length(const std::vector<FileInfo>& sorted_files,
- lucene::store::Directory* directory);
- virtual std::pair<std::unique_ptr<lucene::store::Directory, DirectoryDeleter>,
- std::unique_ptr<lucene::store::IndexOutput>>
- create_output_stream_v1(int64_t index_id, const std::string& index_suffix);
- virtual void write_header_and_data_v1(lucene::store::IndexOutput* output,
- const std::vector<FileInfo>& sorted_files,
- lucene::store::Directory* directory,
- int64_t header_length, int32_t header_file_count);
- // Helper functions specific to write_v2
- virtual std::pair<std::unique_ptr<lucene::store::Directory, DirectoryDeleter>,
- std::unique_ptr<lucene::store::IndexOutput>>
- create_output_stream();
- void write_version_and_indices_count(lucene::store::IndexOutput* output);
- struct FileMetadata {
- int64_t index_id;
- std::string index_suffix;
- std::string filename;
- int64_t offset;
- int64_t length;
- lucene::store::Directory* directory;
-
- FileMetadata(int64_t id, const std::string& suffix, const std::string& file, int64_t off,
- int64_t len, lucene::store::Directory* dir)
- : index_id(id),
- index_suffix(suffix),
- filename(file),
- offset(off),
- length(len),
- directory(dir) {}
- };
- std::vector<FileMetadata> prepare_file_metadata(int64_t& current_offset);
- virtual void write_index_headers_and_metadata(lucene::store::IndexOutput* output,
- const std::vector<FileMetadata>& file_metadata);
- void copy_files_data(lucene::store::IndexOutput* output,
- const std::vector<FileMetadata>& file_metadata);
Status _insert_directory_into_map(int64_t index_id, const std::string& index_suffix,
std::shared_ptr<DorisFSDirectory> dir);
virtual Result<std::unique_ptr<IndexSearcherBuilder>> _construct_index_searcher_builder(
const DorisCompoundReader* dir);
+
// Member variables...
InvertedIndexDirectoryMap _indices_dirs;
const io::FileSystemSPtr _fs;
@@ -176,6 +98,12 @@ private:
// only once
bool _closed = false;
bool _can_use_ram_dir = true;
+
+ XIndexStorageFormatPtr _index_storage_format;
+
+ friend class XIndexStorageFormatV1;
+ friend class XIndexStorageFormatV2;
};
+
} // namespace segment_v2
} // namespace doris
diff --git a/be/src/olap/rowset/segment_v2/x_index_storage_format.cpp b/be/src/olap/rowset/segment_v2/x_index_storage_format.cpp
new file mode 100644
index 00000000000..ec0c4d7d35a
--- /dev/null
+++ b/be/src/olap/rowset/segment_v2/x_index_storage_format.cpp
@@ -0,0 +1,119 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "x_index_storage_format.h"
+
+#include "olap/rowset/segment_v2/inverted_index_desc.h"
+#include "olap/rowset/segment_v2/inverted_index_fs_directory.h"
+#include "util/debug_points.h"
+
+namespace doris::segment_v2 {
+
+XIndexStorageFormat::XIndexStorageFormat(XIndexFileWriter* x_file_writer)
+ : _x_file_writer(x_file_writer) {}
+
+void XIndexStorageFormat::sort_files(std::vector<FileInfo>& file_infos) {
+ auto file_priority = [](const std::string& filename) {
+ for (const auto& entry : InvertedIndexDescriptor::index_file_info_map) {
+ if (filename.find(entry.first) != std::string::npos) {
+ return entry.second;
+ }
+ }
+ return std::numeric_limits<int32_t>::max(); // Other files
+ };
+
+ std::sort(file_infos.begin(), file_infos.end(), [&](const FileInfo& a, const FileInfo& b) {
+ int32_t priority_a = file_priority(a.filename);
+ int32_t priority_b = file_priority(b.filename);
+ if (priority_a != priority_b) {
+ return priority_a < priority_b;
+ }
+ return a.filesize < b.filesize;
+ });
+}
+
+std::vector<FileInfo> XIndexStorageFormat::prepare_sorted_files(
+ lucene::store::Directory* directory) {
+ std::vector<std::string> files;
+ directory->list(&files);
+
+ // Remove write.lock file
+ files.erase(std::remove(files.begin(), files.end(), DorisFSDirectory::WRITE_LOCK_FILE),
+ files.end());
+
+ std::vector<FileInfo> sorted_files;
+ for (const auto& file : files) {
+ FileInfo file_info;
+ file_info.filename = file;
+ file_info.filesize = directory->fileLength(file.c_str());
+ sorted_files.push_back(std::move(file_info));
+ }
+
+ // Sort the files
+ sort_files(sorted_files);
+ return sorted_files;
+}
+
+void XIndexStorageFormat::copy_file(const char* fileName, lucene::store::Directory* dir,
+ lucene::store::IndexOutput* output, uint8_t* buffer,
+ int64_t bufferLength) {
+ lucene::store::IndexInput* tmp = nullptr;
+ CLuceneError err;
+ auto open = dir->openInput(fileName, tmp, err);
+ DBUG_EXECUTE_IF("XIndexFileWriter::copyFile_openInput_error", {
+ open = false;
+ err.set(CL_ERR_IO, "debug point: copyFile_openInput_error");
+ });
+ if (!open) {
+ throw err;
+ }
+
+ std::unique_ptr<lucene::store::IndexInput> input(tmp);
+ int64_t start_ptr = output->getFilePointer();
+ int64_t length = input->length();
+ int64_t remainder = length;
+ int64_t chunk = bufferLength;
+
+ while (remainder > 0) {
+ int64_t len = std::min({chunk, length, remainder});
+ input->readBytes(buffer, len);
+ output->writeBytes(buffer, len);
+ remainder -= len;
+ }
+ DBUG_EXECUTE_IF("XIndexFileWriter::copyFile_remainder_is_not_zero", { remainder = 10; });
+ if (remainder != 0) {
+ std::ostringstream errMsg;
+ errMsg << "Non-zero remainder length after copying: " << remainder << " (id: " << fileName
+ << ", length: " << length << ", buffer size: " << chunk << ")";
+ err.set(CL_ERR_IO, errMsg.str().c_str());
+ throw err;
+ }
+
+ int64_t end_ptr = output->getFilePointer();
+ int64_t diff = end_ptr - start_ptr;
+ DBUG_EXECUTE_IF("XIndexFileWriter::copyFile_diff_not_equals_length", { diff = length - 10; });
+ if (diff != length) {
+ std::ostringstream errMsg;
+ errMsg << "Difference in the output file offsets " << diff
+ << " does not match the original file length " << length;
+ err.set(CL_ERR_IO, errMsg.str().c_str());
+ throw err;
+ }
+ input->close();
+}
+
+} // namespace doris::segment_v2
\ No newline at end of file
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_compaction.h b/be/src/olap/rowset/segment_v2/x_index_storage_format.h
similarity index 51%
copy from be/src/olap/rowset/segment_v2/inverted_index_compaction.h
copy to be/src/olap/rowset/segment_v2/x_index_storage_format.h
index f344f12ffbf..0b296f8de14 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_compaction.h
+++ b/be/src/olap/rowset/segment_v2/x_index_storage_format.h
@@ -14,28 +14,37 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-#pragma once
-
-#include <CLucene.h>
-#include <cstdint>
-#include <string_view>
-#include <vector>
+#pragma once
#include "common/status.h"
-#include "inverted_index_compound_reader.h"
+#include "olap/rowset/segment_v2/inverted_index_common.h"
+
+namespace doris::segment_v2 {
-namespace doris {
-class TabletIndex;
-namespace segment_v2 {
class XIndexFileWriter;
-class InvertedIndexFileReader;
-
-Status compact_column(int64_t index_id,
- std::vector<std::unique_ptr<DorisCompoundReader>>& src_index_dirs,
- std::vector<lucene::store::Directory*>& dest_index_dirs,
- std::string_view tmp_path,
- const std::vector<std::vector<std::pair<uint32_t, uint32_t>>>& trans_vec,
- const std::vector<uint32_t>& dest_segment_num_rows);
-} // namespace segment_v2
-} // namespace doris
+
+class FileInfo {
+public:
+ std::string filename;
+ int64_t filesize;
+};
+
+class XIndexStorageFormat {
+public:
+ XIndexStorageFormat(XIndexFileWriter* x_file_writer);
+ virtual ~XIndexStorageFormat() = default;
+
+ virtual Status write() = 0;
+
+ void sort_files(std::vector<FileInfo>& file_infos);
+ std::vector<FileInfo> prepare_sorted_files(lucene::store::Directory* directory);
+ void copy_file(const char* fileName, lucene::store::Directory* dir,
+ lucene::store::IndexOutput* output, uint8_t* buffer, int64_t bufferLength);
+
+protected:
+ XIndexFileWriter* _x_file_writer = nullptr;
+};
+using XIndexStorageFormatPtr = std::unique_ptr<XIndexStorageFormat>;
+
+} // namespace doris::segment_v2
\ No newline at end of file
diff --git a/be/src/olap/rowset/segment_v2/x_index_storage_format_v1.cpp b/be/src/olap/rowset/segment_v2/x_index_storage_format_v1.cpp
new file mode 100644
index 00000000000..5bc93eeddf5
--- /dev/null
+++ b/be/src/olap/rowset/segment_v2/x_index_storage_format_v1.cpp
@@ -0,0 +1,190 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "x_index_storage_format_v1.h"
+
+#include "olap/rowset/segment_v2/inverted_index_desc.h"
+#include "olap/rowset/segment_v2/inverted_index_fs_directory.h"
+#include "olap/rowset/segment_v2/x_index_file_writer.h"
+#include "util/debug_points.h"
+
+namespace doris::segment_v2 {
+
+XIndexStorageFormatV1::XIndexStorageFormatV1(XIndexFileWriter* file_writer)
+ : XIndexStorageFormat(file_writer) {}
+
+Status XIndexStorageFormatV1::write() {
+ int64_t total_size = 0;
+ std::unique_ptr<lucene::store::Directory, DirectoryDeleter> out_dir = nullptr;
+ std::unique_ptr<lucene::store::IndexOutput> output = nullptr;
+ ErrorContext error_context;
+ for (const auto& entry : _x_file_writer->_indices_dirs) {
+ const int64_t index_id = entry.first.first;
+ const auto& index_suffix = entry.first.second;
+ try {
+ const auto& directory = entry.second.get();
+
+ // Prepare sorted file list
+ auto sorted_files = prepare_sorted_files(directory);
+
+ // Calculate header length
+ auto [header_length, header_file_count] =
+ calculate_header_length(sorted_files, directory);
+
+ // Create output stream
+ auto result = create_output_stream(index_id, index_suffix);
+ out_dir = std::move(result.first);
+ output = std::move(result.second);
+
+ size_t start = output->getFilePointer();
+ // Write header and data
+ write_header_and_data(output.get(), sorted_files, directory, header_length,
+ header_file_count);
+
+ // Collect file information
+ auto compound_file_size = output->getFilePointer() - start;
+ total_size += compound_file_size;
+ add_index_info(index_id, index_suffix, compound_file_size);
+ } catch (CLuceneError& err) {
+ error_context.eptr = std::current_exception();
+ auto index_path = InvertedIndexDescriptor::get_index_file_path_v1(
+ _x_file_writer->_index_path_prefix, index_id, index_suffix);
+ error_context.err_msg.append("CLuceneError occur when write_v1 idx file: ");
+ error_context.err_msg.append(index_path);
+ error_context.err_msg.append(", error msg: ");
+ error_context.err_msg.append(err.what());
+ LOG(ERROR) << error_context.err_msg;
+ }
+ FINALLY({
+ FINALLY_CLOSE(output);
+ FINALLY_CLOSE(out_dir);
+ })
+ }
+
+ _x_file_writer->_total_file_size = total_size;
+ return Status::OK();
+}
+
+std::pair<int64_t, int32_t> XIndexStorageFormatV1::calculate_header_length(
+ const std::vector<FileInfo>& sorted_files, lucene::store::Directory* directory) {
+ // Use RAMDirectory to calculate header length
+ lucene::store::RAMDirectory ram_dir;
+ auto* out_idx = ram_dir.createOutput("temp_idx");
+ DBUG_EXECUTE_IF("XIndexFileWriter::calculate_header_length_ram_output_is_nullptr",
+ { out_idx = nullptr; })
+ if (out_idx == nullptr) {
+ LOG(WARNING) << "XIndexFileWriter::calculate_header_length error: RAMDirectory "
+ "output is nullptr.";
+ _CLTHROWA(CL_ERR_IO, "Create RAMDirectory output error");
+ }
+ std::unique_ptr<lucene::store::IndexOutput> ram_output(out_idx);
+ int32_t file_count = sorted_files.size();
+ ram_output->writeVInt(file_count);
+
+ int64_t header_file_length = 0;
+ const int64_t buffer_length = 16384;
+ uint8_t ram_buffer[buffer_length];
+ int32_t header_file_count = 0;
+ for (const auto& file : sorted_files) {
+ ram_output->writeString(file.filename);
+ ram_output->writeLong(0);
+ ram_output->writeLong(file.filesize);
+ header_file_length += file.filesize;
+
+ if (header_file_length <= DorisFSDirectory::MAX_HEADER_DATA_SIZE) {
+ copy_file(file.filename.c_str(), directory, ram_output.get(), ram_buffer,
+ buffer_length);
+ header_file_count++;
+ }
+ }
+
+ int64_t header_length = ram_output->getFilePointer();
+ ram_output->close();
+ ram_dir.close();
+ return {header_length, header_file_count};
+}
+
+std::pair<std::unique_ptr<lucene::store::Directory, DirectoryDeleter>,
+ std::unique_ptr<lucene::store::IndexOutput>>
+XIndexStorageFormatV1::create_output_stream(int64_t index_id, const std::string& index_suffix) {
+ io::Path cfs_path(InvertedIndexDescriptor::get_index_file_path_v1(
+ _x_file_writer->_index_path_prefix, index_id, index_suffix));
+ auto idx_path = cfs_path.parent_path();
+ std::string idx_name = cfs_path.filename();
+
+ auto* out_dir = DorisFSDirectoryFactory::getDirectory(_x_file_writer->_fs, idx_path.c_str());
+ out_dir->set_file_writer_opts(_x_file_writer->_opts);
+ std::unique_ptr<lucene::store::Directory, DirectoryDeleter> out_dir_ptr(out_dir);
+
+ auto* out = out_dir->createOutput(idx_name.c_str());
+ DBUG_EXECUTE_IF("XIndexFileWriter::write_v1_out_dir_createOutput_nullptr", { out = nullptr; });
+ if (out == nullptr) {
+ LOG(WARNING) << "XIndexFileWriter::create_output_stream_v1 error: CompoundDirectory "
+ "output is nullptr.";
+ _CLTHROWA(CL_ERR_IO, "Create CompoundDirectory output error");
+ }
+ std::unique_ptr<lucene::store::IndexOutput> output(out);
+
+ return {std::move(out_dir_ptr), std::move(output)};
+}
+
+void XIndexStorageFormatV1::write_header_and_data(lucene::store::IndexOutput* output,
+ const std::vector<FileInfo>& sorted_files,
+ lucene::store::Directory* directory,
+ int64_t header_length,
+ int32_t header_file_count) {
+ output->writeVInt(sorted_files.size());
+ int64_t data_offset = header_length;
+ const int64_t buffer_length = 16384;
+ uint8_t buffer[buffer_length];
+
+ for (int i = 0; i < sorted_files.size(); ++i) {
+ auto file = sorted_files[i];
+ output->writeString(file.filename);
+
+ // DataOffset
+ if (i < header_file_count) {
+ // file data write in header, so we set its offset to -1.
+ output->writeLong(-1);
+ } else {
+ output->writeLong(data_offset);
+ }
+ output->writeLong(file.filesize); // FileLength
+ if (i < header_file_count) {
+ // append data
+ copy_file(file.filename.c_str(), directory, output, buffer, buffer_length);
+ } else {
+ data_offset += file.filesize;
+ }
+ }
+
+ for (size_t i = header_file_count; i < sorted_files.size(); ++i) {
+ copy_file(sorted_files[i].filename.c_str(), directory, output, buffer, buffer_length);
+ }
+}
+
+void XIndexStorageFormatV1::add_index_info(int64_t index_id, const std::string& index_suffix,
+ int64_t compound_file_size) {
+ InvertedIndexFileInfo_IndexInfo index_info;
+ index_info.set_index_id(index_id);
+ index_info.set_index_suffix(index_suffix);
+ index_info.set_index_file_size(compound_file_size);
+ auto* new_index_info = _x_file_writer->_file_info.add_index_info();
+ *new_index_info = index_info;
+}
+
+} // namespace doris::segment_v2
\ No newline at end of file
diff --git a/be/src/olap/rowset/segment_v2/x_index_storage_format_v1.h b/be/src/olap/rowset/segment_v2/x_index_storage_format_v1.h
new file mode 100644
index 00000000000..33b07f36716
--- /dev/null
+++ b/be/src/olap/rowset/segment_v2/x_index_storage_format_v1.h
@@ -0,0 +1,45 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include "x_index_storage_format.h"
+
+namespace doris::segment_v2 {
+
+class XIndexStorageFormatV1 : public XIndexStorageFormat {
+public:
+ XIndexStorageFormatV1(XIndexFileWriter* file_writer);
+ ~XIndexStorageFormatV1() override = default;
+
+ Status write() override;
+
+private:
+ std::pair<int64_t, int32_t> calculate_header_length(const std::vector<FileInfo>& sorted_files,
+ lucene::store::Directory* directory);
+ std::pair<std::unique_ptr<lucene::store::Directory, DirectoryDeleter>,
+ std::unique_ptr<lucene::store::IndexOutput>>
+ create_output_stream(int64_t index_id, const std::string& index_suffix);
+ void write_header_and_data(lucene::store::IndexOutput* output,
+ const std::vector<FileInfo>& sorted_files,
+ lucene::store::Directory* directory, int64_t header_length,
+ int32_t header_file_count);
+ void add_index_info(int64_t index_id, const std::string& index_suffix,
+ int64_t compound_file_size);
+};
+
+} // namespace doris::segment_v2
\ No newline at end of file
diff --git a/be/src/olap/rowset/segment_v2/x_index_storage_format_v2.cpp b/be/src/olap/rowset/segment_v2/x_index_storage_format_v2.cpp
new file mode 100644
index 00000000000..2cc5ad0d773
--- /dev/null
+++ b/be/src/olap/rowset/segment_v2/x_index_storage_format_v2.cpp
@@ -0,0 +1,246 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "x_index_storage_format_v2.h"
+
+#include "olap/rowset/segment_v2/inverted_index_desc.h"
+#include "olap/rowset/segment_v2/inverted_index_fs_directory.h"
+#include "olap/rowset/segment_v2/x_index_file_writer.h"
+#include "util/debug_points.h"
+
+namespace doris::segment_v2 {
+
+FileMetadata::FileMetadata(int64_t id, std::string suffix, std::string file, int64_t off,
+ int64_t len, lucene::store::Directory* dir)
+ : index_id(id),
+ index_suffix(std::move(suffix)),
+ filename(std::move(file)),
+ offset(off),
+ length(len),
+ directory(dir) {}
+
+XIndexStorageFormatV2::XIndexStorageFormatV2(XIndexFileWriter* x_file_writer)
+ : XIndexStorageFormat(x_file_writer) {}
+
+Status XIndexStorageFormatV2::write() {
+ std::unique_ptr<lucene::store::Directory, DirectoryDeleter> out_dir = nullptr;
+ std::unique_ptr<lucene::store::IndexOutput> compound_file_output = nullptr;
+ ErrorContext error_context;
+ try {
+ // Calculate header length and initialize offset
+ int64_t current_offset = header_length();
+ // Prepare file metadata
+ auto file_metadata = prepare_file_metadata(current_offset);
+
+ // Create output stream
+ auto result = create_output_stream();
+ out_dir = std::move(result.first);
+ compound_file_output = std::move(result.second);
+
+ // Write version and number of indices
+ write_version_and_indices_count(compound_file_output.get());
+
+ // Write index headers and file metadata
+ write_index_headers_and_metadata(compound_file_output.get(), file_metadata);
+
+ // Copy file data
+ copy_files_data(compound_file_output.get(), file_metadata);
+
+ _x_file_writer->_total_file_size = compound_file_output->getFilePointer();
+ _x_file_writer->_file_info.set_index_size(_x_file_writer->_total_file_size);
+ } catch (CLuceneError& err) {
+ error_context.eptr = std::current_exception();
+ auto index_path =
+ InvertedIndexDescriptor::get_index_file_path_v2(_x_file_writer->_index_path_prefix);
+ error_context.err_msg.append("CLuceneError occur when close idx file: ");
+ error_context.err_msg.append(index_path);
+ error_context.err_msg.append(", error msg: ");
+ error_context.err_msg.append(err.what());
+ LOG(ERROR) << error_context.err_msg;
+ }
+ FINALLY({
+ FINALLY_CLOSE(compound_file_output);
+ FINALLY_CLOSE(out_dir);
+ })
+
+ return Status::OK();
+}
+
+int64_t XIndexStorageFormatV2::header_length() {
+ int64_t header_size = 0;
+ header_size +=
+ sizeof(int32_t) * 2; // Account for the size of the version number and number of indices
+
+ for (const auto& entry : _x_file_writer->_indices_dirs) {
+ const auto& suffix = entry.first.second;
+ header_size += sizeof(int64_t); // index id
+ header_size += sizeof(int32_t); // index suffix name size
+ header_size += suffix.length(); // index suffix name
+ header_size += sizeof(int32_t); // index file count
+ const auto& dir = entry.second;
+ std::vector<std::string> files;
+ dir->list(&files);
+
+ for (const auto& file : files) {
+ header_size += sizeof(int32_t); // file name size
+ header_size += file.length(); // file name
+ header_size += sizeof(int64_t); // file offset
+ header_size += sizeof(int64_t); // file size
+ }
+ }
+ return header_size;
+}
+
+std::vector<FileMetadata> XIndexStorageFormatV2::prepare_file_metadata(int64_t& current_offset) {
+ std::vector<FileMetadata> file_metadata;
+ std::vector<FileMetadata> meta_files;
+ std::vector<FileMetadata> normal_files;
+ for (const auto& entry : _x_file_writer->_indices_dirs) {
+ const int64_t index_id = entry.first.first;
+ const auto& index_suffix = entry.first.second;
+ auto* dir = entry.second.get();
+
+ auto sorted_files = prepare_sorted_files(dir);
+
+ for (const auto& file : sorted_files) {
+ bool is_meta = false;
+
+ for (const auto& entry : InvertedIndexDescriptor::index_file_info_map) {
+ if (file.filename.find(entry.first) != std::string::npos) {
+ meta_files.emplace_back(index_id, index_suffix, file.filename, 0, file.filesize,
+ dir);
+ is_meta = true;
+ break;
+ }
+ }
+
+ if (!is_meta) {
+ normal_files.emplace_back(index_id, index_suffix, file.filename, 0, file.filesize,
+ dir);
+ }
+ }
+ }
+
+ file_metadata.reserve(meta_files.size() + normal_files.size());
+
+ // meta file
+ for (auto& entry : meta_files) {
+ entry.offset = current_offset;
+ file_metadata.emplace_back(std::move(entry));
+ current_offset += entry.length;
+ }
+ // normal file
+ for (auto& entry : normal_files) {
+ entry.offset = current_offset;
+ file_metadata.emplace_back(std::move(entry));
+ current_offset += entry.length;
+ }
+
+ DBUG_EXECUTE_IF("CSIndexInput.readInternal", {
+ bool is_meta_file = true;
+ for (const auto& entry : file_metadata) {
+ bool is_meta = false;
+ for (const auto& map_entry : InvertedIndexDescriptor::index_file_info_map) {
+ if (entry.filename.find(map_entry.first) != std::string::npos) {
+ is_meta = true;
+ break;
+ }
+ }
+
+ if (is_meta_file && !is_meta) {
+ is_meta_file = false;
+ } else if (!is_meta_file && is_meta) {
+ throw Exception(ErrorCode::INTERNAL_ERROR,
+ "Invalid file order: meta files must be at the beginning of the "
+ "file_metadata structure.");
+ }
+ }
+ });
+
+ return file_metadata;
+}
+
+std::pair<std::unique_ptr<lucene::store::Directory, DirectoryDeleter>,
+ std::unique_ptr<lucene::store::IndexOutput>>
+XIndexStorageFormatV2::create_output_stream() {
+ io::Path index_path {
+ InvertedIndexDescriptor::get_index_file_path_v2(_x_file_writer->_index_path_prefix)};
+
+ auto* out_dir = DorisFSDirectoryFactory::getDirectory(_x_file_writer->_fs,
+ index_path.parent_path().c_str());
+ out_dir->set_file_writer_opts(_x_file_writer->_opts);
+ std::unique_ptr<lucene::store::Directory, DirectoryDeleter> out_dir_ptr(out_dir);
+
+ DCHECK(_x_file_writer->_idx_v2_writer != nullptr) << "inverted index file writer v2 is nullptr";
+ auto compound_file_output = std::unique_ptr<lucene::store::IndexOutput>(
+ out_dir->createOutputV2(_x_file_writer->_idx_v2_writer.get()));
+
+ return {std::move(out_dir_ptr), std::move(compound_file_output)};
+}
+
+void XIndexStorageFormatV2::write_version_and_indices_count(lucene::store::IndexOutput* output) {
+ // Write the version number
+ output->writeInt(_x_file_writer->_storage_format);
+
+ // Write the number of indices
+ const auto num_indices = static_cast<uint32_t>(_x_file_writer->_indices_dirs.size());
+ output->writeInt(num_indices);
+}
+
+void XIndexStorageFormatV2::write_index_headers_and_metadata(
+ lucene::store::IndexOutput* output, const std::vector<FileMetadata>& file_metadata) {
+ // Group files by index_id and index_suffix
+ std::map<std::pair<int64_t, std::string>, std::vector<FileMetadata>> indices;
+
+ for (const auto& meta : file_metadata) {
+ indices[{meta.index_id, meta.index_suffix}].push_back(meta);
+ }
+
+ for (const auto& index_entry : indices) {
+ int64_t index_id = index_entry.first.first;
+ const std::string& index_suffix = index_entry.first.second;
+ const auto& files = index_entry.second;
+
+ // Write the index ID and the number of files
+ output->writeLong(index_id);
+ output->writeInt(static_cast<int32_t>(index_suffix.length()));
+ output->writeBytes(reinterpret_cast<const uint8_t*>(index_suffix.data()),
+ index_suffix.length());
+ output->writeInt(static_cast<int32_t>(files.size()));
+
+ // Write file metadata
+ for (const auto& file : files) {
+ output->writeInt(static_cast<int32_t>(file.filename.length()));
+ output->writeBytes(reinterpret_cast<const uint8_t*>(file.filename.data()),
+ file.filename.length());
+ output->writeLong(file.offset);
+ output->writeLong(file.length);
+ }
+ }
+}
+
+void XIndexStorageFormatV2::copy_files_data(lucene::store::IndexOutput* output,
+ const std::vector<FileMetadata>& file_metadata) {
+ const int64_t buffer_length = 16384;
+ uint8_t buffer[buffer_length];
+
+ for (const auto& meta : file_metadata) {
+ copy_file(meta.filename.c_str(), meta.directory, output, buffer, buffer_length);
+ }
+}
+
+} // namespace doris::segment_v2
\ No newline at end of file
diff --git a/be/src/olap/rowset/segment_v2/x_index_storage_format_v2.h b/be/src/olap/rowset/segment_v2/x_index_storage_format_v2.h
new file mode 100644
index 00000000000..40c4e47e396
--- /dev/null
+++ b/be/src/olap/rowset/segment_v2/x_index_storage_format_v2.h
@@ -0,0 +1,56 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include "x_index_storage_format.h"
+
+namespace doris::segment_v2 {
+
+struct FileMetadata {
+ int64_t index_id;
+ std::string index_suffix;
+ std::string filename;
+ int64_t offset;
+ int64_t length;
+ lucene::store::Directory* directory;
+
+ FileMetadata(int64_t id, std::string suffix, std::string file, int64_t off, int64_t len,
+ lucene::store::Directory* dir);
+};
+
+class XIndexStorageFormatV2 : public XIndexStorageFormat {
+public:
+ XIndexStorageFormatV2(XIndexFileWriter* file_writer);
+ ~XIndexStorageFormatV2() override = default;
+
+ Status write() override;
+
+private:
+ int64_t header_length();
+ std::vector<FileMetadata> prepare_file_metadata(int64_t& current_offset);
+ std::pair<std::unique_ptr<lucene::store::Directory, DirectoryDeleter>,
+ std::unique_ptr<lucene::store::IndexOutput>>
+ create_output_stream();
+ void write_version_and_indices_count(lucene::store::IndexOutput* output);
+ void write_index_headers_and_metadata(lucene::store::IndexOutput* output,
+ const std::vector<FileMetadata>& file_metadata);
+ void copy_files_data(lucene::store::IndexOutput* output,
+ const std::vector<FileMetadata>& file_metadata);
+};
+
+} // namespace doris::segment_v2
\ No newline at end of file
diff --git a/be/src/olap/shared_predicate.h b/be/src/olap/shared_predicate.h
index 2a83f7ef434..064aa72e582 100644
--- a/be/src/olap/shared_predicate.h
+++ b/be/src/olap/shared_predicate.h
@@ -62,7 +62,7 @@ public:
}
Status evaluate(const vectorized::IndexFieldNameAndTypePair& name_with_type,
- InvertedIndexIterator* iterator, uint32_t num_rows,
+ IndexIterator* iterator, uint32_t num_rows,
roaring::Roaring* bitmap) const override {
std::shared_lock<std::shared_mutex> lock(_mtx);
if (!_nested) {
diff --git a/be/src/olap/task/index_builder.cpp b/be/src/olap/task/index_builder.cpp
index 0d010785a46..74e27f1914b 100644
--- a/be/src/olap/task/index_builder.cpp
+++ b/be/src/olap/task/index_builder.cpp
@@ -25,8 +25,8 @@
#include "olap/rowset/rowset_writer_context.h"
#include "olap/rowset/segment_v2/index_writer.h"
#include "olap/rowset/segment_v2/inverted_index_desc.h"
-#include "olap/rowset/segment_v2/inverted_index_file_reader.h"
#include "olap/rowset/segment_v2/inverted_index_fs_directory.h"
+#include "olap/rowset/segment_v2/x_index_file_reader.h"
#include "olap/rowset/segment_v2/x_index_file_writer.h"
#include "olap/segment_loader.h"
#include "olap/storage_engine.h"
@@ -230,7 +230,7 @@ Status IndexBuilder::update_inverted_index_info() {
} else {
for (int seg_id = 0; seg_id < num_segments; seg_id++) {
auto seg_path = DORIS_TRY(input_rowset->segment_path(seg_id));
- auto idx_file_reader = std::make_unique<InvertedIndexFileReader>(
+ auto idx_file_reader = std::make_unique<XIndexFileReader>(
context.fs(),
std::string {InvertedIndexDescriptor::get_index_file_path_prefix(seg_path)},
output_rs_tablet_schema->get_inverted_index_storage_format());
@@ -243,7 +243,7 @@ Status IndexBuilder::update_inverted_index_info() {
if (!st.ok() && !st.is<ErrorCode::INVERTED_INDEX_FILE_NOT_FOUND>()) {
return st;
}
- _inverted_index_file_readers.emplace(
+ _index_file_readers.emplace(
std::make_pair(output_rs_writer->rowset_id().to_string(), seg_id),
std::move(idx_file_reader));
}
@@ -292,11 +292,11 @@ Status IndexBuilder::handle_single_rowset(RowsetMetaSharedPtr output_rowset_meta
const auto& output_rowset_schema = output_rowset_meta->tablet_schema();
size_t inverted_index_size = 0;
for (auto& seg_ptr : segments) {
- auto idx_file_reader_iter = _inverted_index_file_readers.find(
+ auto idx_file_reader_iter = _index_file_readers.find(
std::make_pair(output_rowset_meta->rowset_id().to_string(), seg_ptr->id()));
DBUG_EXECUTE_IF("IndexBuilder::handle_single_rowset_can_not_find_reader_drop_op",
- { idx_file_reader_iter = _inverted_index_file_readers.end(); })
- if (idx_file_reader_iter == _inverted_index_file_readers.end()) {
+ { idx_file_reader_iter = _index_file_readers.end(); })
+ if (idx_file_reader_iter == _index_file_readers.end()) {
LOG(ERROR) << "idx_file_reader_iter" << output_rowset_meta->rowset_id() << ":"
<< seg_ptr->id() << " cannot be found";
continue;
@@ -363,11 +363,11 @@ Status IndexBuilder::handle_single_rowset(RowsetMetaSharedPtr output_rowset_meta
std::unique_ptr<XIndexFileWriter> x_index_file_writer = nullptr;
if (output_rowset_schema->get_inverted_index_storage_format() >=
InvertedIndexStorageFormatPB::V2) {
- auto idx_file_reader_iter = _inverted_index_file_readers.find(
+ auto idx_file_reader_iter = _index_file_readers.find(
std::make_pair(output_rowset_meta->rowset_id().to_string(), seg_ptr->id()));
DBUG_EXECUTE_IF("IndexBuilder::handle_single_rowset_can_not_find_reader",
- { idx_file_reader_iter = _inverted_index_file_readers.end(); })
- if (idx_file_reader_iter == _inverted_index_file_readers.end()) {
+ { idx_file_reader_iter = _index_file_readers.end(); })
+ if (idx_file_reader_iter == _index_file_readers.end()) {
LOG(ERROR) << "idx_file_reader_iter" << output_rowset_meta->rowset_id() << ":"
<< seg_ptr->id() << " cannot be found";
continue;
diff --git a/be/src/olap/task/index_builder.h b/be/src/olap/task/index_builder.h
index 113cbc60564..cf4e3b7176b 100644
--- a/be/src/olap/task/index_builder.h
+++ b/be/src/olap/task/index_builder.h
@@ -86,8 +86,8 @@ private:
_inverted_index_builders;
std::unordered_map<int64_t, std::unique_ptr<XIndexFileWriter>> _x_index_file_writers;
// <rowset_id, segment_id>
- std::unordered_map<std::pair<std::string, int64_t>, std::unique_ptr<InvertedIndexFileReader>>
- _inverted_index_file_readers;
+ std::unordered_map<std::pair<std::string, int64_t>, std::unique_ptr<XIndexFileReader>>
+ _index_file_readers;
};
using IndexBuilderSharedPtr = std::shared_ptr<IndexBuilder>;
diff --git a/be/src/pipeline/exec/olap_scan_operator.cpp b/be/src/pipeline/exec/olap_scan_operator.cpp
index b93c22274d9..1422537761a 100644
--- a/be/src/pipeline/exec/olap_scan_operator.cpp
+++ b/be/src/pipeline/exec/olap_scan_operator.cpp
@@ -193,8 +193,8 @@ Status OlapScanLocalState::_init_profile() {
ADD_TIMER(_scanner_profile, "SegmentIteratorInitReturnColumnIteratorsTimer");
_segment_iterator_init_bitmap_index_iterators_timer =
ADD_TIMER(_scanner_profile, "SegmentIteratorInitBitmapIndexIteratorsTimer");
- _segment_iterator_init_inverted_index_iterators_timer =
- ADD_TIMER(_scanner_profile, "SegmentIteratorInitInvertedIndexIteratorsTimer");
+ _segment_iterator_init_index_iterators_timer =
+ ADD_TIMER(_scanner_profile, "SegmentIteratorInitIndexIteratorsTimer");
_segment_create_column_readers_timer =
ADD_TIMER(_scanner_profile, "SegmentCreateColumnReadersTimer");
diff --git a/be/src/pipeline/exec/olap_scan_operator.h b/be/src/pipeline/exec/olap_scan_operator.h
index f6275df046e..2e6cf4b182a 100644
--- a/be/src/pipeline/exec/olap_scan_operator.h
+++ b/be/src/pipeline/exec/olap_scan_operator.h
@@ -212,7 +212,7 @@ private:
RuntimeProfile::Counter* _segment_iterator_init_timer = nullptr;
RuntimeProfile::Counter* _segment_iterator_init_return_column_iterators_timer = nullptr;
RuntimeProfile::Counter* _segment_iterator_init_bitmap_index_iterators_timer = nullptr;
- RuntimeProfile::Counter* _segment_iterator_init_inverted_index_iterators_timer = nullptr;
+ RuntimeProfile::Counter* _segment_iterator_init_index_iterators_timer = nullptr;
RuntimeProfile::Counter* _segment_create_column_readers_timer = nullptr;
RuntimeProfile::Counter* _segment_load_index_timer = nullptr;
diff --git a/be/src/vec/exec/scan/olap_scanner.cpp b/be/src/vec/exec/scan/olap_scanner.cpp
index d623a54f4f9..039c9fa08dd 100644
--- a/be/src/vec/exec/scan/olap_scanner.cpp
+++ b/be/src/vec/exec/scan/olap_scanner.cpp
@@ -677,8 +677,8 @@ void OlapScanner::_collect_profile_before_close() {
stats.segment_iterator_init_return_column_iterators_timer_ns);
COUNTER_UPDATE(local_state->_segment_iterator_init_bitmap_index_iterators_timer,
stats.segment_iterator_init_bitmap_index_iterators_timer_ns);
- COUNTER_UPDATE(local_state->_segment_iterator_init_inverted_index_iterators_timer,
- stats.segment_iterator_init_inverted_index_iterators_timer_ns);
+ COUNTER_UPDATE(local_state->_segment_iterator_init_index_iterators_timer,
+ stats.segment_iterator_init_index_iterators_timer_ns);
COUNTER_UPDATE(local_state->_segment_create_column_readers_timer,
stats.segment_create_column_readers_timer_ns);
diff --git a/be/src/vec/exprs/vexpr.cpp b/be/src/vec/exprs/vexpr.cpp
index 0a7c8330983..4eaf761311f 100644
--- a/be/src/vec/exprs/vexpr.cpp
+++ b/be/src/vec/exprs/vexpr.cpp
@@ -624,7 +624,7 @@ Status VExpr::get_result_from_const(vectorized::Block* block, const std::string&
Status VExpr::_evaluate_inverted_index(VExprContext* context, const FunctionBasePtr& function,
uint32_t segment_num_rows) {
// Pre-allocate vectors based on an estimated or known size
- std::vector<segment_v2::InvertedIndexIterator*> iterators;
+ std::vector<segment_v2::IndexIterator*> iterators;
std::vector<vectorized::IndexFieldNameAndTypePair> data_type_with_names;
std::vector<int> column_ids;
vectorized::ColumnsWithTypeAndName arguments;
diff --git a/be/src/vec/exprs/vexpr_context.h b/be/src/vec/exprs/vexpr_context.h
index 3fb6a1a3b74..9a88fd26488 100644
--- a/be/src/vec/exprs/vexpr_context.h
+++ b/be/src/vec/exprs/vexpr_context.h
@@ -44,29 +44,27 @@ class InvertedIndexContext {
public:
InvertedIndexContext(
const std::vector<ColumnId>& col_ids,
- const std::vector<std::unique_ptr<segment_v2::InvertedIndexIterator>>&
- inverted_index_iterators,
+ const std::vector<std::unique_ptr<segment_v2::IndexIterator>>& index_iterators,
const std::vector<vectorized::IndexFieldNameAndTypePair>& storage_name_and_type_vec,
std::unordered_map<ColumnId, std::unordered_map<const vectorized::VExpr*, bool>>&
common_expr_inverted_index_status)
: _col_ids(col_ids),
- _inverted_index_iterators(inverted_index_iterators),
+ _index_iterators(index_iterators),
_storage_name_and_type(storage_name_and_type_vec),
_expr_inverted_index_status(common_expr_inverted_index_status) {}
- segment_v2::InvertedIndexIterator* get_inverted_index_iterator_by_column_id(
- int column_index) const {
+ segment_v2::IndexIterator* get_inverted_index_iterator_by_column_id(int column_index) const {
if (column_index < 0 || column_index >= _col_ids.size()) {
return nullptr;
}
const auto& column_id = _col_ids[column_index];
- if (column_id >= _inverted_index_iterators.size()) {
+ if (column_id >= _index_iterators.size()) {
return nullptr;
}
- if (!_inverted_index_iterators[column_id]) {
+ if (!_index_iterators[column_id]) {
return nullptr;
}
- return _inverted_index_iterators[column_id].get();
+ return _index_iterators[column_id].get();
}
const vectorized::IndexFieldNameAndTypePair* get_storage_name_and_type_by_column_id(
@@ -129,9 +127,8 @@ private:
// A reference to a vector of column IDs for the current expression's output columns.
const std::vector<ColumnId>& _col_ids;
- // A reference to a vector of unique pointers to inverted index iterators.
- const std::vector<std::unique_ptr<segment_v2::InvertedIndexIterator>>&
- _inverted_index_iterators;
+ // A reference to a vector of unique pointers to index iterators.
+ const std::vector<std::unique_ptr<segment_v2::IndexIterator>>& _index_iterators;
// A reference to a vector of storage name and type pairs related to schema.
const std::vector<vectorized::IndexFieldNameAndTypePair>& _storage_name_and_type;
diff --git a/be/src/vec/functions/array/function_array_index.h b/be/src/vec/functions/array/function_array_index.h
index 7f08bfdd4b4..b190f9e424e 100644
--- a/be/src/vec/functions/array/function_array_index.h
+++ b/be/src/vec/functions/array/function_array_index.h
@@ -118,7 +118,7 @@ public:
Status evaluate_inverted_index(
const ColumnsWithTypeAndName& arguments,
const std::vector<vectorized::IndexFieldNameAndTypePair>& data_type_with_names,
- std::vector<segment_v2::InvertedIndexIterator*> iterators, uint32_t num_rows,
+ std::vector<segment_v2::IndexIterator*> iterators, uint32_t num_rows,
segment_v2::InvertedIndexResultBitmap& bitmap_result) const override {
DCHECK(arguments.size() == 1);
DCHECK(data_type_with_names.size() == 1);
@@ -128,8 +128,7 @@ public:
if (iter == nullptr) {
return Status::OK();
}
- if (iter->get_inverted_index_reader_type() ==
- segment_v2::InvertedIndexReaderType::FULLTEXT) {
+ if (iter->get_reader()->is_fulltext_index()) {
// parser is not none we can not make sure the result is correct in expr combination
// for example, filter: !array_index(array, 'tall:120cm, weight: 35kg')
// here we have rows [tall:120cm, weight: 35kg, hobbies: reading book] which be tokenized
@@ -147,7 +146,6 @@ public:
return Status::OK();
}
- std::shared_ptr<roaring::Roaring> roaring = std::make_shared<roaring::Roaring>();
std::shared_ptr<roaring::Roaring> null_bitmap = std::make_shared<roaring::Roaring>();
if (iter->has_null()) {
segment_v2::InvertedIndexQueryCacheHandle null_bitmap_cache_handle;
@@ -157,9 +155,14 @@ public:
std::unique_ptr<InvertedIndexQueryParamFactory> query_param = nullptr;
RETURN_IF_ERROR(InvertedIndexQueryParamFactory::create_query_value(param_type, ¶m_value,
query_param));
- RETURN_IF_ERROR(iter->read_from_inverted_index(
- data_type_with_name.first, query_param->get_value(),
- segment_v2::InvertedIndexQueryType::EQUAL_QUERY, num_rows, roaring));
+ InvertedIndexParam param;
+ param.column_name = data_type_with_name.first;
+ param.query_value = query_param->get_value();
+ param.query_type = segment_v2::InvertedIndexQueryType::EQUAL_QUERY;
+ param.num_rows = num_rows;
+ param.roaring = std::make_shared<roaring::Roaring>();
+ ;
+ RETURN_IF_ERROR(iter->read_from_index(¶m));
// here debug for check array_contains function really filter rows by inverted index correctly
DBUG_EXECUTE_IF("array_func.array_contains", {
auto result_bitmap = DebugPoints::instance()->get_debug_param_or_default<int32_t>(
@@ -168,14 +171,14 @@ public:
return Status::Error<ErrorCode::INTERNAL_ERROR>(
"result_bitmap count cannot be negative");
}
- if (roaring->cardinality() != result_bitmap) {
+ if (param.roaring->cardinality() != result_bitmap) {
return Status::Error<ErrorCode::INTERNAL_ERROR>(
"array_contains really filtered {} by inverted index not equal to expected "
"{}",
- roaring->cardinality(), result_bitmap);
+ param.roaring->cardinality(), result_bitmap);
}
})
- segment_v2::InvertedIndexResultBitmap result(roaring, null_bitmap);
+ segment_v2::InvertedIndexResultBitmap result(param.roaring, null_bitmap);
bitmap_result = result;
bitmap_result.mask_out_null();
diff --git a/be/src/vec/functions/array/function_arrays_overlap.h b/be/src/vec/functions/array/function_arrays_overlap.h
index c144940cc46..90577aafcfd 100644
--- a/be/src/vec/functions/array/function_arrays_overlap.h
+++ b/be/src/vec/functions/array/function_arrays_overlap.h
@@ -165,7 +165,7 @@ public:
Status evaluate_inverted_index(
const ColumnsWithTypeAndName& arguments,
const std::vector<vectorized::IndexFieldNameAndTypePair>& data_type_with_names,
- std::vector<segment_v2::InvertedIndexIterator*> iterators, uint32_t num_rows,
+ std::vector<segment_v2::IndexIterator*> iterators, uint32_t num_rows,
segment_v2::InvertedIndexResultBitmap& bitmap_result) const override {
DCHECK(arguments.size() == 1);
DCHECK(data_type_with_names.size() == 1);
@@ -175,8 +175,7 @@ public:
return Status::OK();
}
auto data_type_with_name = data_type_with_names[0];
- if (iter->get_inverted_index_reader_type() ==
- segment_v2::InvertedIndexReaderType::FULLTEXT) {
+ if (iter->get_reader()->is_fulltext_index()) {
return Status::Error<ErrorCode::INVERTED_INDEX_EVALUATE_SKIPPED>(
"Inverted index evaluate skipped, FULLTEXT reader can not support "
"array_overlap");
@@ -213,19 +212,24 @@ public:
}
std::unique_ptr<InvertedIndexQueryParamFactory> query_param = nullptr;
const Array& query_val = param_value.get<Array>();
+
+ InvertedIndexParam param;
+ param.column_name = data_type_with_name.first;
+ param.query_type = segment_v2::InvertedIndexQueryType::EQUAL_QUERY;
+ param.num_rows = num_rows;
for (auto nested_query_val : query_val) {
// any element inside array is NULL, return NULL
// by current arrays_overlap execute logic.
if (nested_query_val.is_null()) {
return Status::OK();
}
- std::shared_ptr<roaring::Roaring> single_res = std::make_shared<roaring::Roaring>();
RETURN_IF_ERROR(InvertedIndexQueryParamFactory::create_query_value(
nested_param_type, &nested_query_val, query_param));
- RETURN_IF_ERROR(iter->read_from_inverted_index(
- data_type_with_name.first, query_param->get_value(),
- segment_v2::InvertedIndexQueryType::EQUAL_QUERY, num_rows, single_res));
- *roaring |= *single_res;
+ param.query_value = query_param->get_value();
+ param.roaring = std::make_shared<roaring::Roaring>();
+ ;
+ RETURN_IF_ERROR(iter->read_from_index(¶m));
+ *roaring |= *param.roaring;
}
segment_v2::InvertedIndexResultBitmap result(roaring, null_bitmap);
diff --git a/be/src/vec/functions/function.h b/be/src/vec/functions/function.h
index 92282c48394..8636e795bac 100644
--- a/be/src/vec/functions/function.h
+++ b/be/src/vec/functions/function.h
@@ -30,7 +30,7 @@
#include "common/exception.h"
#include "common/status.h"
-#include "olap/rowset/segment_v2/inverted_index_reader.h"
+#include "olap/rowset/segment_v2/inverted_index_iterator.h"
#include "udf/udf.h"
#include "vec/core/block.h"
#include "vec/core/column_numbers.h"
@@ -196,7 +196,7 @@ public:
virtual Status evaluate_inverted_index(
const ColumnsWithTypeAndName& arguments,
const std::vector<vectorized::IndexFieldNameAndTypePair>& data_type_with_names,
- std::vector<segment_v2::InvertedIndexIterator*> iterators, uint32_t num_rows,
+ std::vector<segment_v2::IndexIterator*> iterators, uint32_t num_rows,
segment_v2::InvertedIndexResultBitmap& bitmap_result) const {
return Status::OK();
}
@@ -437,7 +437,7 @@ protected:
Status evaluate_inverted_index(
const ColumnsWithTypeAndName& arguments,
const std::vector<vectorized::IndexFieldNameAndTypePair>& data_type_with_names,
- std::vector<segment_v2::InvertedIndexIterator*> iterators, uint32_t num_rows,
+ std::vector<segment_v2::IndexIterator*> iterators, uint32_t num_rows,
segment_v2::InvertedIndexResultBitmap& bitmap_result) const {
return function->evaluate_inverted_index(arguments, data_type_with_names, iterators,
num_rows, bitmap_result);
@@ -503,7 +503,7 @@ public:
Status evaluate_inverted_index(
const ColumnsWithTypeAndName& args,
const std::vector<vectorized::IndexFieldNameAndTypePair>& data_type_with_names,
- std::vector<segment_v2::InvertedIndexIterator*> iterators, uint32_t num_rows,
+ std::vector<segment_v2::IndexIterator*> iterators, uint32_t num_rows,
segment_v2::InvertedIndexResultBitmap& bitmap_result) const override {
return function->evaluate_inverted_index(args, data_type_with_names, iterators, num_rows,
bitmap_result);
diff --git a/be/src/vec/functions/function_ip.h b/be/src/vec/functions/function_ip.h
index 5fed82d6e10..35ac98ee40f 100644
--- a/be/src/vec/functions/function_ip.h
+++ b/be/src/vec/functions/function_ip.h
@@ -648,7 +648,7 @@ public:
Status evaluate_inverted_index(
const ColumnsWithTypeAndName& arguments,
const std::vector<vectorized::IndexFieldNameAndTypePair>& data_type_with_names,
- std::vector<segment_v2::InvertedIndexIterator*> iterators, uint32_t num_rows,
+ std::vector<segment_v2::IndexIterator*> iterators, uint32_t num_rows,
segment_v2::InvertedIndexResultBitmap& bitmap_result) const override {
DCHECK(arguments.size() == 1);
DCHECK(data_type_with_names.size() == 1);
@@ -659,7 +659,7 @@ public:
return Status::OK();
}
- if (iter->get_inverted_index_reader_type() != segment_v2::InvertedIndexReaderType::BKD) {
+ if (!iter->get_reader()->is_bkd_index()) {
// Not support only bkd index
return Status::Error<ErrorCode::INVERTED_INDEX_EVALUATE_SKIPPED>(
"Inverted index evaluate skipped, ip range reader can only support by bkd "
@@ -707,40 +707,48 @@ public:
" can not support this cidr " + arg_column->get_data_at(0).to_string());
}
// apply for inverted index
- std::shared_ptr<roaring::Roaring> res_roaring = std::make_shared<roaring::Roaring>();
- std::shared_ptr<roaring::Roaring> max_roaring = std::make_shared<roaring::Roaring>();
std::shared_ptr<roaring::Roaring> null_bitmap = std::make_shared<roaring::Roaring>();
auto param_type = data_type_with_name.second->get_type_as_type_descriptor().type;
std::unique_ptr<segment_v2::InvertedIndexQueryParamFactory> query_param = nullptr;
+
// >= min ip
+ segment_v2::InvertedIndexParam res_param;
+ res_param.column_name = data_type_with_name.first;
+ res_param.query_type = segment_v2::InvertedIndexQueryType::GREATER_EQUAL_QUERY;
+ res_param.query_value = query_param->get_value();
+ res_param.num_rows = num_rows;
+ res_param.roaring = std::make_shared<roaring::Roaring>();
RETURN_IF_ERROR(segment_v2::InvertedIndexQueryParamFactory::create_query_value(
param_type, &min_ip, query_param));
- RETURN_IF_ERROR(iter->read_from_inverted_index(
- data_type_with_name.first, query_param->get_value(),
- segment_v2::InvertedIndexQueryType::GREATER_EQUAL_QUERY, num_rows, res_roaring));
+ RETURN_IF_ERROR(iter->read_from_index(&res_param));
+
// <= max ip
+ segment_v2::InvertedIndexParam max_param;
+ res_param.column_name = data_type_with_name.first;
+ res_param.query_type = segment_v2::InvertedIndexQueryType::LESS_EQUAL_QUERY;
+ res_param.query_value = query_param->get_value();
+ res_param.num_rows = num_rows;
+ res_param.roaring = std::make_shared<roaring::Roaring>();
RETURN_IF_ERROR(segment_v2::InvertedIndexQueryParamFactory::create_query_value(
param_type, &max_ip, query_param));
- RETURN_IF_ERROR(iter->read_from_inverted_index(
- data_type_with_name.first, query_param->get_value(),
- segment_v2::InvertedIndexQueryType::LESS_EQUAL_QUERY, num_rows, max_roaring));
+ RETURN_IF_ERROR(iter->read_from_index(&res_param));
DBUG_EXECUTE_IF("ip.inverted_index_filtered", {
auto req_id = DebugPoints::instance()->get_debug_param_or_default<int32_t>(
"ip.inverted_index_filtered", "req_id", 0);
LOG(INFO) << "execute inverted index req_id: " << req_id
- << " min: " << res_roaring->cardinality();
+ << " min: " << res_param.roaring->cardinality();
});
- *res_roaring &= *max_roaring;
+ *res_param.roaring &= *max_param.roaring;
DBUG_EXECUTE_IF("ip.inverted_index_filtered", {
auto req_id = DebugPoints::instance()->get_debug_param_or_default<int32_t>(
"ip.inverted_index_filtered", "req_id", 0);
LOG(INFO) << "execute inverted index req_id: " << req_id
- << " max: " << max_roaring->cardinality()
- << " result: " << res_roaring->cardinality();
+ << " max: " << max_param.roaring->cardinality()
+ << " result: " << res_param.roaring->cardinality();
});
- segment_v2::InvertedIndexResultBitmap result(res_roaring, null_bitmap);
+ segment_v2::InvertedIndexResultBitmap result(res_param.roaring, null_bitmap);
bitmap_result = result;
bitmap_result.mask_out_null();
return Status::OK();
diff --git a/be/src/vec/functions/function_multi_match.cpp b/be/src/vec/functions/function_multi_match.cpp
index 9ccf7bf7513..3d993dd43e8 100644
--- a/be/src/vec/functions/function_multi_match.cpp
+++ b/be/src/vec/functions/function_multi_match.cpp
@@ -56,7 +56,7 @@ InvertedIndexQueryType get_query_type(const std::string& query_type) {
Status FunctionMultiMatch::evaluate_inverted_index(
const ColumnsWithTypeAndName& arguments,
const std::vector<vectorized::IndexFieldNameAndTypePair>& data_type_with_names,
- std::vector<segment_v2::InvertedIndexIterator*> iterators, uint32_t num_rows,
+ std::vector<segment_v2::IndexIterator*> iterators, uint32_t num_rows,
segment_v2::InvertedIndexResultBitmap& bitmap_result) const {
DCHECK(arguments.size() == 2);
std::shared_ptr<roaring::Roaring> roaring = std::make_shared<roaring::Roaring>();
@@ -75,11 +75,15 @@ Status FunctionMultiMatch::evaluate_inverted_index(
auto query_str = arguments[1].column->get_data_at(0);
auto param_type = arguments[1].type->get_type_as_type_descriptor().type;
if (!is_string_type(param_type)) {
- return Status::Error<ErrorCode::INVERTED_INDEX_INVALID_PARAMETERS>(
+ return Status::Error<ErrorCode::INDEX_INVALID_PARAMETERS>(
"arguments for multi_match must be string");
}
// search
+ InvertedIndexParam param;
+ param.query_value = &query_str;
+ param.query_type = query_type;
+ param.num_rows = num_rows;
for (size_t i = 0; i < data_type_with_names.size(); i++) {
auto column_name = data_type_with_names[i].first;
auto* iter = iterators[i];
@@ -89,10 +93,10 @@ Status FunctionMultiMatch::evaluate_inverted_index(
return Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>(error_msg);
}
- auto single_result = std::make_shared<roaring::Roaring>();
- RETURN_IF_ERROR(iter->read_from_inverted_index(column_name, &query_str, query_type,
- num_rows, single_result));
- *roaring |= *single_result;
+ param.column_name = column_name;
+ param.roaring = std::make_shared<roaring::Roaring>();
+ RETURN_IF_ERROR(iter->read_from_index(¶m));
+ *roaring |= *param.roaring;
}
segment_v2::InvertedIndexResultBitmap result(roaring, null_bitmap);
bitmap_result = result;
diff --git a/be/src/vec/functions/function_multi_match.h b/be/src/vec/functions/function_multi_match.h
index ba395450a30..45ad1d5dca8 100644
--- a/be/src/vec/functions/function_multi_match.h
+++ b/be/src/vec/functions/function_multi_match.h
@@ -61,7 +61,7 @@ public:
Status evaluate_inverted_index(
const ColumnsWithTypeAndName& arguments,
const std::vector<vectorized::IndexFieldNameAndTypePair>& data_type_with_names,
- std::vector<segment_v2::InvertedIndexIterator*> iterators, uint32_t num_rows,
+ std::vector<segment_v2::IndexIterator*> iterators, uint32_t num_rows,
segment_v2::InvertedIndexResultBitmap& bitmap_result) const override;
};
diff --git a/be/src/vec/functions/function_tokenize.cpp b/be/src/vec/functions/function_tokenize.cpp
index f0a7c3b68ae..f64a248cd9d 100644
--- a/be/src/vec/functions/function_tokenize.cpp
+++ b/be/src/vec/functions/function_tokenize.cpp
@@ -143,7 +143,7 @@ Status FunctionTokenize::execute_impl(FunctionContext* /*context*/, Block& block
inverted_index_ctx.parser_type = get_inverted_index_parser_type_from_string(
get_parser_string_from_properties(properties));
if (inverted_index_ctx.parser_type == InvertedIndexParserType::PARSER_UNKNOWN) {
- return Status::Error<doris::ErrorCode::INVERTED_INDEX_INVALID_PARAMETERS>(
+ return Status::Error<doris::ErrorCode::INDEX_INVALID_PARAMETERS>(
"unsupported parser type. currently, only 'english', 'chinese', and "
"'unicode' analyzers are supported.");
}
diff --git a/be/src/vec/functions/functions_comparison.h b/be/src/vec/functions/functions_comparison.h
index 2da767fa7ef..aeb5c326b38 100644
--- a/be/src/vec/functions/functions_comparison.h
+++ b/be/src/vec/functions/functions_comparison.h
@@ -533,7 +533,7 @@ public:
Status evaluate_inverted_index(
const ColumnsWithTypeAndName& arguments,
const std::vector<vectorized::IndexFieldNameAndTypePair>& data_type_with_names,
- std::vector<segment_v2::InvertedIndexIterator*> iterators, uint32_t num_rows,
+ std::vector<segment_v2::IndexIterator*> iterators, uint32_t num_rows,
segment_v2::InvertedIndexResultBitmap& bitmap_result) const override {
DCHECK(arguments.size() == 1);
DCHECK(data_type_with_names.size() == 1);
@@ -543,8 +543,7 @@ public:
if (iter == nullptr) {
return Status::OK();
}
- if (iter->get_inverted_index_reader_type() ==
- segment_v2::InvertedIndexReaderType::FULLTEXT) {
+ if (iter->get_reader()->is_fulltext_index()) {
//NOT support comparison predicate when parser is FULLTEXT for expr inverted index evaluate.
return Status::OK();
}
@@ -564,9 +563,7 @@ public:
return Status::InvalidArgument("invalid comparison op type {}", Name::name);
}
- if (segment_v2::is_range_query(query_type) &&
- iter->get_inverted_index_reader_type() ==
- segment_v2::InvertedIndexReaderType::STRING_TYPE) {
+ if (segment_v2::is_range_query(query_type) && iter->get_reader()->is_string_index()) {
// untokenized strings exceed ignore_above, they are written as null, causing range query errors
return Status::OK();
}
@@ -577,18 +574,23 @@ public:
std::unique_ptr<segment_v2::InvertedIndexQueryParamFactory> query_param = nullptr;
RETURN_IF_ERROR(segment_v2::InvertedIndexQueryParamFactory::create_query_value(
param_type, ¶m_value, query_param));
- std::shared_ptr<roaring::Roaring> roaring = std::make_shared<roaring::Roaring>();
RETURN_IF_ERROR(segment_v2::InvertedIndexQueryParamFactory::create_query_value(
param_type, ¶m_value, query_param));
- RETURN_IF_ERROR(iter->read_from_inverted_index(column_name, query_param->get_value(),
- query_type, num_rows, roaring));
+
+ segment_v2::InvertedIndexParam param;
+ param.column_name = column_name;
+ param.query_value = query_param->get_value();
+ param.query_type = query_type;
+ param.num_rows = num_rows;
+ param.roaring = std::make_shared<roaring::Roaring>();
+ RETURN_IF_ERROR(iter->read_from_index(¶m));
std::shared_ptr<roaring::Roaring> null_bitmap = std::make_shared<roaring::Roaring>();
if (iter->has_null()) {
segment_v2::InvertedIndexQueryCacheHandle null_bitmap_cache_handle;
RETURN_IF_ERROR(iter->read_null_bitmap(&null_bitmap_cache_handle));
null_bitmap = null_bitmap_cache_handle.get_bitmap();
}
- segment_v2::InvertedIndexResultBitmap result(roaring, null_bitmap);
+ segment_v2::InvertedIndexResultBitmap result(param.roaring, null_bitmap);
bitmap_result = result;
bitmap_result.mask_out_null();
diff --git a/be/src/vec/functions/in.h b/be/src/vec/functions/in.h
index 4a588353d58..6a1576c3288 100644
--- a/be/src/vec/functions/in.h
+++ b/be/src/vec/functions/in.h
@@ -137,7 +137,7 @@ public:
Status evaluate_inverted_index(
const ColumnsWithTypeAndName& arguments,
const std::vector<vectorized::IndexFieldNameAndTypePair>& data_type_with_names,
- std::vector<segment_v2::InvertedIndexIterator*> iterators, uint32_t num_rows,
+ std::vector<segment_v2::IndexIterator*> iterators, uint32_t num_rows,
segment_v2::InvertedIndexResultBitmap& bitmap_result) const override {
DCHECK(data_type_with_names.size() == 1);
DCHECK(iterators.size() == 1);
@@ -149,8 +149,7 @@ public:
if (iter == nullptr) {
return Status::OK();
}
- if (iter->get_inverted_index_reader_type() ==
- segment_v2::InvertedIndexReaderType::FULLTEXT) {
+ if (iter->get_reader()->is_fulltext_index()) {
//NOT support in list when parser is FULLTEXT for expr inverted index evaluate.
return Status::OK();
}
@@ -176,10 +175,15 @@ public:
RETURN_IF_ERROR(InvertedIndexQueryParamFactory::create_query_value(
param_type, ¶m_value, query_param));
InvertedIndexQueryType query_type = InvertedIndexQueryType::EQUAL_QUERY;
- std::shared_ptr<roaring::Roaring> index = std::make_shared<roaring::Roaring>();
- RETURN_IF_ERROR(iter->read_from_inverted_index(column_name, query_param->get_value(),
- query_type, num_rows, index));
- *roaring |= *index;
+ segment_v2::InvertedIndexParam param;
+ param.column_name = column_name;
+ param.query_value = query_param->get_value();
+ param.query_type = query_type;
+ param.num_rows = num_rows;
+ param.roaring = std::make_shared<roaring::Roaring>();
+ ;
+ RETURN_IF_ERROR(iter->read_from_index(¶m));
+ *roaring |= *param.roaring;
}
segment_v2::InvertedIndexResultBitmap result(roaring, null_bitmap);
bitmap_result = result;
diff --git a/be/src/vec/functions/match.cpp b/be/src/vec/functions/match.cpp
index 8ba4f2f8ebb..7d7f00cbee5 100644
--- a/be/src/vec/functions/match.cpp
+++ b/be/src/vec/functions/match.cpp
@@ -28,7 +28,7 @@ namespace doris::vectorized {
Status FunctionMatchBase::evaluate_inverted_index(
const ColumnsWithTypeAndName& arguments,
const std::vector<vectorized::IndexFieldNameAndTypePair>& data_type_with_names,
- std::vector<segment_v2::InvertedIndexIterator*> iterators, uint32_t num_rows,
+ std::vector<segment_v2::IndexIterator*> iterators, uint32_t num_rows,
segment_v2::InvertedIndexResultBitmap& bitmap_result) const {
DCHECK(arguments.size() == 1);
DCHECK(data_type_with_names.size() == 1);
@@ -42,14 +42,11 @@ Status FunctionMatchBase::evaluate_inverted_index(
if (function_name == MATCH_PHRASE_FUNCTION || function_name == MATCH_PHRASE_PREFIX_FUNCTION ||
function_name == MATCH_PHRASE_EDGE_FUNCTION) {
- if (iter->get_inverted_index_reader_type() == InvertedIndexReaderType::FULLTEXT &&
- get_parser_phrase_support_string_from_properties(iter->get_index_properties()) ==
- INVERTED_INDEX_PARSER_PHRASE_SUPPORT_NO) {
- return Status::Error<ErrorCode::INVERTED_INDEX_INVALID_PARAMETERS>(
+ if (iter->get_reader()->is_fulltext_index() && !iter->get_reader()->is_support_phrase()) {
+ return Status::Error<ErrorCode::INDEX_INVALID_PARAMETERS>(
"phrase queries require setting support_phrase = true");
}
}
- std::shared_ptr<roaring::Roaring> roaring = std::make_shared<roaring::Roaring>();
Field param_value;
arguments[0].column->get(0, param_value);
if (param_value.is_null()) {
@@ -58,19 +55,23 @@ Status FunctionMatchBase::evaluate_inverted_index(
}
auto param_type = arguments[0].type->get_type_as_type_descriptor().type;
if (!is_string_type(param_type)) {
- return Status::Error<ErrorCode::INVERTED_INDEX_INVALID_PARAMETERS>(
+ return Status::Error<ErrorCode::INDEX_INVALID_PARAMETERS>(
"arguments for match must be string");
}
std::unique_ptr<InvertedIndexQueryParamFactory> query_param = nullptr;
RETURN_IF_ERROR(InvertedIndexQueryParamFactory::create_query_value(param_type, ¶m_value,
query_param));
+
+ InvertedIndexParam param;
+ param.column_name = data_type_with_name.first;
+ param.query_value = query_param->get_value();
+ param.query_type = get_query_type_from_fn_name();
+ param.num_rows = num_rows;
+ param.roaring = std::make_shared<roaring::Roaring>();
if (is_string_type(param_type)) {
- auto inverted_index_query_type = get_query_type_from_fn_name();
- RETURN_IF_ERROR(
- iter->read_from_inverted_index(data_type_with_name.first, query_param->get_value(),
- inverted_index_query_type, num_rows, roaring));
+ RETURN_IF_ERROR(iter->read_from_index(¶m));
} else {
- return Status::Error<ErrorCode::INVERTED_INDEX_INVALID_PARAMETERS>(
+ return Status::Error<ErrorCode::INDEX_INVALID_PARAMETERS>(
"invalid params type for FunctionMatchBase::evaluate_inverted_index {}",
param_type);
}
@@ -80,7 +81,7 @@ Status FunctionMatchBase::evaluate_inverted_index(
RETURN_IF_ERROR(iter->read_null_bitmap(&null_bitmap_cache_handle));
null_bitmap = null_bitmap_cache_handle.get_bitmap();
}
- segment_v2::InvertedIndexResultBitmap result(roaring, null_bitmap);
+ segment_v2::InvertedIndexResultBitmap result(param.roaring, null_bitmap);
bitmap_result = result;
bitmap_result.mask_out_null();
@@ -463,13 +464,13 @@ Status FunctionMatchRegexp::execute_match(FunctionContext* context, const std::s
err_message.append(compile_err->message);
LOG(ERROR) << err_message;
hs_free_compile_error(compile_err);
- return Status::Error<ErrorCode::INVERTED_INDEX_INVALID_PARAMETERS>(err_message);
+ return Status::Error<ErrorCode::INDEX_INVALID_PARAMETERS>(err_message);
}
if (hs_alloc_scratch(database, &scratch) != HS_SUCCESS) {
LOG(ERROR) << "hyperscan could not allocate scratch space.";
hs_free_database(database);
- return Status::Error<ErrorCode::INVERTED_INDEX_INVALID_PARAMETERS>(
+ return Status::Error<ErrorCode::INDEX_INVALID_PARAMETERS>(
"hyperscan could not allocate scratch space.");
}
diff --git a/be/src/vec/functions/match.h b/be/src/vec/functions/match.h
index 48523e9748b..1f0cedca412 100644
--- a/be/src/vec/functions/match.h
+++ b/be/src/vec/functions/match.h
@@ -92,10 +92,11 @@ public:
int32_t& current_src_array_offset) const;
Status check(FunctionContext* context, const std::string& function_name) const;
+
Status evaluate_inverted_index(
const ColumnsWithTypeAndName& arguments,
const std::vector<vectorized::IndexFieldNameAndTypePair>& data_type_with_names,
- std::vector<segment_v2::InvertedIndexIterator*> iterators, uint32_t num_rows,
+ std::vector<segment_v2::IndexIterator*> iterators, uint32_t num_rows,
segment_v2::InvertedIndexResultBitmap& bitmap_result) const override;
};
diff --git a/be/src/vector/stream_wrapper.h b/be/src/vector/stream_wrapper.h
index 6c930ef08fe..5f31e92814a 100644
--- a/be/src/vector/stream_wrapper.h
+++ b/be/src/vector/stream_wrapper.h
@@ -1,3 +1,20 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
#pragma once
#include <CLucene.h>
diff --git a/be/test/olap/rowset/segment_v2/inverted_index/compaction/util/index_compaction_utils.cpp b/be/test/olap/rowset/segment_v2/inverted_index/compaction/util/index_compaction_utils.cpp
index 02353fc5441..3fcb791f457 100644
--- a/be/test/olap/rowset/segment_v2/inverted_index/compaction/util/index_compaction_utils.cpp
+++ b/be/test/olap/rowset/segment_v2/inverted_index/compaction/util/index_compaction_utils.cpp
@@ -34,7 +34,7 @@
#include "olap/rowset/beta_rowset_writer.h"
#include "olap/rowset/rowset_factory.h"
#include "olap/rowset/segment_v2/inverted_index/query/query_factory.h"
-#include "olap/rowset/segment_v2/inverted_index_file_reader.h"
+#include "olap/rowset/segment_v2/x_index_file_reader.h"
#include "olap/storage_engine.h"
namespace doris {
@@ -146,12 +146,12 @@ class IndexCompactionUtils {
}
static bool query_bkd(const TabletIndex* index,
- std::shared_ptr<InvertedIndexFileReader>& inverted_index_file_reader,
+ std::shared_ptr<XIndexFileReader>& index_file_reader,
const std::vector<int>& query_data,
const std::vector<int>& query_result) {
- const auto& idx_reader = BkdIndexReader::create_shared(index, inverted_index_file_reader);
+ const auto& idx_reader = BkdIndexReader::create_shared(index, index_file_reader);
const auto& index_searcher_builder = std::make_unique<BKDIndexSearcherBuilder>();
- auto dir = inverted_index_file_reader->open(index);
+ auto dir = index_file_reader->open(index);
EXPECT_TRUE(dir.has_value());
auto searcher_result = index_searcher_builder->get_index_searcher(dir.value().release());
EXPECT_TRUE(searcher_result.has_value());
@@ -179,14 +179,14 @@ class IndexCompactionUtils {
}
static bool query_string(const TabletIndex* index,
- std::shared_ptr<InvertedIndexFileReader>& inverted_index_file_reader,
+ std::shared_ptr<XIndexFileReader>& index_file_reader,
const std::string& column_name,
const std::vector<std::string>& query_data,
const std::vector<int>& query_result) {
const auto& idx_reader =
- StringTypeInvertedIndexReader::create_shared(index, inverted_index_file_reader);
+ StringTypeInvertedIndexReader::create_shared(index, index_file_reader);
const auto& index_searcher_builder = std::make_unique<FulltextIndexSearcherBuilder>();
- auto dir = inverted_index_file_reader->open(index);
+ auto dir = index_file_reader->open(index);
EXPECT_TRUE(dir.has_value());
auto searcher_result = index_searcher_builder->get_index_searcher(dir.value().release());
EXPECT_TRUE(searcher_result.has_value());
@@ -211,14 +211,13 @@ class IndexCompactionUtils {
}
static bool query_fulltext(const TabletIndex* index,
- std::shared_ptr<InvertedIndexFileReader>& inverted_index_file_reader,
+ std::shared_ptr<XIndexFileReader>& index_file_reader,
const std::string& column_name,
const std::vector<std::string>& query_data,
const std::vector<int>& query_result) {
- const auto& idx_reader =
- FullTextIndexReader::create_shared(index, inverted_index_file_reader);
+ const auto& idx_reader = FullTextIndexReader::create_shared(index, index_file_reader);
const auto& index_searcher_builder = std::make_unique<FulltextIndexSearcherBuilder>();
- auto dir = inverted_index_file_reader->open(index);
+ auto dir = index_file_reader->open(index);
EXPECT_TRUE(dir.has_value());
auto searcher_result = index_searcher_builder->get_index_searcher(dir.value().release());
EXPECT_TRUE(searcher_result.has_value());
@@ -564,11 +563,11 @@ class IndexCompactionUtils {
EXPECT_TRUE(seg_path.has_value());
const auto& index_file_path_prefix =
InvertedIndexDescriptor::get_index_file_path_prefix(seg_path.value());
- auto inverted_index_file_reader = std::make_shared<InvertedIndexFileReader>(
+ auto index_file_reader = std::make_shared<XIndexFileReader>(
fs, std::string(index_file_path_prefix),
tablet_schema->get_inverted_index_storage_format(), index_info);
- EXPECT_TRUE(inverted_index_file_reader->init().ok());
- const auto& dirs = inverted_index_file_reader->get_all_directories();
+ EXPECT_TRUE(index_file_reader->init().ok());
+ const auto& dirs = index_file_reader->get_all_directories();
EXPECT_TRUE(dirs.has_value());
EXPECT_EQ(dirs.value().size(), 4);
@@ -583,17 +582,15 @@ class IndexCompactionUtils {
for (const auto& data : query_data.first) {
query_data_int.push_back(std::stoi(data));
}
- EXPECT_TRUE(query_bkd(index, inverted_index_file_reader, query_data_int,
- query_data.second));
+ EXPECT_TRUE(query_bkd(index, index_file_reader, query_data_int, query_data.second));
} else if (col_uid == 1) {
// String index
- EXPECT_TRUE(query_string(index, inverted_index_file_reader, std::to_string(col_uid),
+ EXPECT_TRUE(query_string(index, index_file_reader, std::to_string(col_uid),
query_data.first, query_data.second));
} else if (col_uid == 2) {
// Fulltext index
- EXPECT_TRUE(query_fulltext(index, inverted_index_file_reader,
- std::to_string(col_uid), query_data.first,
- query_data.second));
+ EXPECT_TRUE(query_fulltext(index, index_file_reader, std::to_string(col_uid),
+ query_data.first, query_data.second));
}
}
}
@@ -710,12 +707,12 @@ class IndexCompactionUtils {
EXPECT_TRUE(seg_path.has_value());
const auto& index_file_path_prefix =
InvertedIndexDescriptor::get_index_file_path_prefix(seg_path.value());
- auto inverted_index_file_reader = std::make_shared<InvertedIndexFileReader>(
+ auto index_file_reader = std::make_shared<XIndexFileReader>(
fs, std::string(index_file_path_prefix),
schema->get_inverted_index_storage_format(), index_info);
- st = inverted_index_file_reader->init();
+ st = index_file_reader->init();
EXPECT_TRUE(st.ok()) << st.to_string();
- const auto& dirs = inverted_index_file_reader->get_all_directories();
+ const auto& dirs = index_file_reader->get_all_directories();
EXPECT_TRUE(dirs.has_value());
if (custom_check) {
custom_check(dirs.value().size());
@@ -724,18 +721,18 @@ class IndexCompactionUtils {
}
}
- static std::shared_ptr<InvertedIndexFileReader> init_index_file_reader(
+ static std::shared_ptr<XIndexFileReader> init_index_file_reader(
const RowsetSharedPtr& output_rowset, const std::string& seg_path,
const InvertedIndexStorageFormatPB& index_storage_format) {
const auto& index_file_path_prefix =
InvertedIndexDescriptor::get_index_file_path_prefix(seg_path);
- auto inverted_index_file_reader_index = std::make_shared<InvertedIndexFileReader>(
+ auto index_file_reader = std::make_shared<XIndexFileReader>(
output_rowset->_rowset_meta->fs(), std::string(index_file_path_prefix),
index_storage_format);
- auto st = inverted_index_file_reader_index->init();
+ auto st = index_file_reader->init();
EXPECT_TRUE(st.ok()) << st.to_string();
- return inverted_index_file_reader_index;
+ return index_file_reader;
}
};
diff --git a/be/test/olap/rowset/segment_v2/inverted_index_array_test.cpp b/be/test/olap/rowset/segment_v2/inverted_index_array_test.cpp
index 06e1559f091..74284fdef8d 100644
--- a/be/test/olap/rowset/segment_v2/inverted_index_array_test.cpp
+++ b/be/test/olap/rowset/segment_v2/inverted_index_array_test.cpp
@@ -33,10 +33,10 @@
#include "io/fs/local_file_system.h"
#include "olap/rowset/segment_v2/inverted_index_compound_reader.h"
#include "olap/rowset/segment_v2/inverted_index_desc.h"
-#include "olap/rowset/segment_v2/inverted_index_file_reader.h"
#include "olap/rowset/segment_v2/inverted_index_file_writer.h"
#include "olap/rowset/segment_v2/inverted_index_fs_directory.h"
#include "olap/rowset/segment_v2/inverted_index_writer.h"
+#include "olap/rowset/segment_v2/x_index_file_reader.h"
#include "olap/rowset/segment_v2/zone_map_index.h"
#include "olap/tablet_schema.h"
#include "olap/tablet_schema_helper.h"
@@ -76,7 +76,7 @@ public:
} else if (format == InvertedIndexStorageFormatPB::V2) {
file_str = InvertedIndexDescriptor::get_index_file_path_v2(index_prefix);
}
- std::unique_ptr<InvertedIndexFileReader> reader = std::make_unique<InvertedIndexFileReader>(
+ std::unique_ptr<XIndexFileReader> reader = std::make_unique<XIndexFileReader>(
io::global_local_filesystem(), index_prefix, format);
auto st = reader->init();
EXPECT_EQ(st, Status::OK());
@@ -902,7 +902,7 @@ public:
ExpectedDocMap expected = {{"123", {0}}, {"456", {0}}, {"789", {2}}, {"101112", {2}}};
std::vector<int> expected_null_bitmap = {1};
- std::unique_ptr<InvertedIndexFileReader> reader = std::make_unique<InvertedIndexFileReader>(
+ std::unique_ptr<XIndexFileReader> reader = std::make_unique<XIndexFileReader>(
io::global_local_filesystem(), index_path_prefix, InvertedIndexStorageFormatPB::V1);
auto sts = reader->init();
EXPECT_EQ(sts, Status::OK());
diff --git a/be/test/olap/rowset/segment_v2/inverted_index_compound_reader_test.cpp b/be/test/olap/rowset/segment_v2/inverted_index_compound_reader_test.cpp
index ce0e4560501..78f04111206 100644
--- a/be/test/olap/rowset/segment_v2/inverted_index_compound_reader_test.cpp
+++ b/be/test/olap/rowset/segment_v2/inverted_index_compound_reader_test.cpp
@@ -36,9 +36,9 @@
#include "gtest/gtest_pred_impl.h"
#include "io/fs/local_file_system.h"
#include "olap/rowset/segment_v2/inverted_index_desc.h"
-#include "olap/rowset/segment_v2/inverted_index_file_reader.h"
#include "olap/rowset/segment_v2/inverted_index_file_writer.h"
#include "olap/rowset/segment_v2/inverted_index_fs_directory.h"
+#include "olap/rowset/segment_v2/x_index_file_reader.h"
#include "olap/tablet_schema.h"
#include "olap/tablet_schema_helper.h"
#include "runtime/exec_env.h"
@@ -382,7 +382,7 @@ TEST_F(DorisCompoundReaderTest, IntegrationWithFileWriter) {
auto st = index_file_writer->close();
ASSERT_TRUE(st.ok()) << st;
- auto file_reader = std::make_unique<InvertedIndexFileReader>(
+ auto file_reader = std::make_unique<XIndexFileReader>(
io::global_local_filesystem(), index_path_prefix, InvertedIndexStorageFormatPB::V1);
st = file_reader->init();
@@ -429,7 +429,7 @@ TEST_F(DorisCompoundReaderTest, IntegrationWithFileWriter) {
st = index_file_writer->close();
ASSERT_TRUE(st.ok()) << st;
- auto file_reader = std::make_unique<InvertedIndexFileReader>(
+ auto file_reader = std::make_unique<XIndexFileReader>(
io::global_local_filesystem(), index_path_prefix, InvertedIndexStorageFormatPB::V2);
st = file_reader->init();
diff --git a/be/test/olap/rowset/segment_v2/inverted_index_writer_test.cpp b/be/test/olap/rowset/segment_v2/inverted_index_writer_test.cpp
index ae3551a60b7..d6e74bd47e2 100644
--- a/be/test/olap/rowset/segment_v2/inverted_index_writer_test.cpp
+++ b/be/test/olap/rowset/segment_v2/inverted_index_writer_test.cpp
@@ -35,10 +35,10 @@
#include "io/fs/local_file_system.h"
#include "olap/field.h"
#include "olap/rowset/segment_v2/inverted_index_desc.h"
-#include "olap/rowset/segment_v2/inverted_index_file_reader.h"
#include "olap/rowset/segment_v2/inverted_index_file_writer.h"
#include "olap/rowset/segment_v2/inverted_index_fs_directory.h"
#include "olap/rowset/segment_v2/inverted_index_reader.h"
+#include "olap/rowset/segment_v2/x_index_file_reader.h"
#include "olap/tablet_schema.h"
#include "olap/tablet_schema_helper.h"
#include "runtime/runtime_state.h"
@@ -73,7 +73,7 @@ public:
} else if (format == InvertedIndexStorageFormatPB::V2) {
file_str = InvertedIndexDescriptor::get_index_file_path_v2(index_prefix);
}
- std::unique_ptr<InvertedIndexFileReader> reader = std::make_unique<InvertedIndexFileReader>(
+ std::unique_ptr<XIndexFileReader> reader = std::make_unique<XIndexFileReader>(
io::global_local_filesystem(), index_prefix, format);
auto st = reader->init();
EXPECT_EQ(st, Status::OK());
@@ -160,7 +160,7 @@ public:
query_options.enable_inverted_index_searcher_cache = false;
runtime_state.set_query_options(query_options);
// Create a BkdIndexReader to verify the index
- auto reader = std::make_shared<InvertedIndexFileReader>(
+ auto reader = std::make_shared<XIndexFileReader>(
io::global_local_filesystem(), index_prefix, InvertedIndexStorageFormatPB::V2);
auto st = reader->init();
EXPECT_EQ(st, Status::OK());
@@ -581,7 +581,7 @@ public:
runtime_state.set_query_options(query_options);
// Create a reader to verify the index
- auto reader = std::make_shared<InvertedIndexFileReader>(
+ auto reader = std::make_shared<XIndexFileReader>(
io::global_local_filesystem(), index_path_prefix, InvertedIndexStorageFormatPB::V2);
status = reader->init();
EXPECT_EQ(status, Status::OK());
@@ -765,18 +765,18 @@ TEST_F(InvertedIndexWriterTest, CompareUnicodeStringWriteResults) {
io::IOContext io_ctx;
// Create readers for both indexes
- auto reader_enabled = std::make_shared<InvertedIndexFileReader>(
- io::global_local_filesystem(), index_path_prefix_enabled,
- InvertedIndexStorageFormatPB::V2);
+ auto reader_enabled = std::make_shared<XIndexFileReader>(io::global_local_filesystem(),
+ index_path_prefix_enabled,
+ InvertedIndexStorageFormatPB::V2);
status = reader_enabled->init();
EXPECT_EQ(status, Status::OK());
auto result_enabled = reader_enabled->open(&idx_meta);
EXPECT_TRUE(result_enabled.has_value())
<< "Failed to open compound reader" << result_enabled.error();
- auto reader_disabled = std::make_shared<InvertedIndexFileReader>(
- io::global_local_filesystem(), index_path_prefix_disabled,
- InvertedIndexStorageFormatPB::V2);
+ auto reader_disabled = std::make_shared<XIndexFileReader>(io::global_local_filesystem(),
+ index_path_prefix_disabled,
+ InvertedIndexStorageFormatPB::V2);
status = reader_disabled->init();
EXPECT_EQ(status, Status::OK());
auto result_disabled = reader_disabled->open(&idx_meta);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org