You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2025/10/30 06:33:42 UTC
(doris) branch branch-3.1 updated: branch-3.1: [opt](inverted index) add custom analyzer support with char_filter, basic and icu tokenizer #56243 #57055 (#57137)
This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 347775bc98e branch-3.1: [opt](inverted index) add custom analyzer support with char_filter, basic and icu tokenizer #56243 #57055 (#57137)
347775bc98e is described below
commit 347775bc98ef58e98399d9db2038753c375f66c8
Author: zzzxl <ya...@selectdb.com>
AuthorDate: Thu Oct 30 14:33:35 2025 +0800
branch-3.1: [opt](inverted index) add custom analyzer support with char_filter, basic and icu tokenizer #56243 #57055 (#57137)
picked from #56243 #57055
---
be/src/clucene | 2 +-
be/src/olap/inverted_index_parser.cpp | 16 +-
be/src/olap/inverted_index_parser.h | 2 +
.../inverted_index/analysis_factory_mgr.cpp | 61 +++-
.../inverted_index/analysis_factory_mgr.h | 8 +-
.../inverted_index/analyzer/analyzer.cpp | 23 +-
.../segment_v2/inverted_index/analyzer/analyzer.h | 5 +-
.../inverted_index/analyzer/basic/basic_analyzer.h | 47 ++-
.../inverted_index/analyzer/custom_analyzer.cpp | 51 ++--
.../inverted_index/analyzer/custom_analyzer.h | 28 +-
.../analyzer/custom_analyzer_config.cpp | 10 +
.../analyzer/custom_analyzer_config.h | 4 +
.../inverted_index/analyzer/icu/icu_analyzer.h | 49 ++-
.../tokenizer.h => char_filter/char_filter.h} | 41 ++-
.../char_filter/char_filter_factory.h | 24 +-
.../char_filter/char_replace_char_filter.cpp | 25 +-
.../char_filter/char_replace_char_filter.h | 16 +-
.../char_filter/char_replace_char_filter_factory.h | 75 +++++
.../char_filter/empty_char_filter_factory.h | 61 ++++
.../rowset/segment_v2/inverted_index/setting.h | 61 +++-
.../empty_token_filter_factory.h} | 43 ++-
.../inverted_index/token_filter/token_filter.h | 2 -
.../token_filter/word_delimiter_filter_factory.h | 2 +
.../segment_v2/inverted_index/token_stream.h | 40 ++-
.../basic/basic_tokenizer.cpp | 43 ++-
.../basic/basic_tokenizer.h | 26 +-
.../tokenizer/basic/basic_tokenizer_factory.h | 50 ++++
.../tokenizer/empty/empty_tokenizer_factory.h | 72 +++++
.../icu/break_iterator_wrapper.cpp | 4 +-
.../icu/break_iterator_wrapper.h | 4 +-
.../icu/composite_break_iterator.cpp | 4 +-
.../icu/composite_break_iterator.h | 4 +-
.../icu/default_icu_tokenizer_config.cpp | 4 +-
.../icu/default_icu_tokenizer_config.h | 4 +-
.../{analyzer => tokenizer}/icu/icu_common.h | 4 +-
.../{analyzer => tokenizer}/icu/icu_tokenizer.cpp | 15 +-
.../{analyzer => tokenizer}/icu/icu_tokenizer.h | 13 +-
.../icu/icu_tokenizer_config.h | 4 +-
.../icu/icu_tokenizer_factory.h} | 21 +-
.../icu/script_iterator.cpp | 4 +-
.../{analyzer => tokenizer}/icu/script_iterator.h | 4 +-
.../tokenizer/standard/standard_tokenizer_impl.h | 7 +-
.../inverted_index/tokenizer/tokenizer.h | 9 +-
.../{token_filter/token_filter.h => util/reader.h} | 17 +-
.../rowset/segment_v2/inverted_index_writer.cpp | 25 +-
be/src/olap/tablet_schema.h | 6 +-
be/src/runtime/index_policy/index_policy_mgr.cpp | 84 ++++--
be/src/runtime/index_policy/index_policy_mgr.h | 7 +
be/src/vec/functions/function_tokenize.cpp | 4 +-
be/src/vec/functions/match.cpp | 6 +-
.../inverted_index/analysis_factory_mgr_test.cpp | 249 ++++++++++++++++
.../inverted_index/analyzer/icu_analyzer_test.cpp | 59 +++-
.../analyzer/simple_analyzer_test.cpp | 8 +-
.../ananlyzer/custom_analyzer_test.cpp | 328 +++++++++++++--------
.../char_filter/char_filter_test.cpp | 76 +++++
.../char_replace_char_filter_factory_test.cpp | 193 ++++++++++++
.../segment_v2/inverted_index/setting_test.cpp | 95 +++++-
.../ascii_folding_filter_factory_test.cpp | 6 +-
.../lower_case_filter_factory_test.cpp | 6 +-
.../token_filter/word_delimiter_filter_test.cpp | 6 +-
.../tokenizer/basic_tokenizer_factory_test.cpp | 235 +++++++++++++++
.../char_group_tokenizer_factory_test.cpp | 6 +-
.../tokenizer/edge_ngram_tokenizer_test.cpp | 6 +-
.../tokenizer/icu_tokenizer_factory_test.cpp | 212 +++++++++++++
.../tokenizer/keyword_analyzer_test.cpp | 6 +-
.../tokenizer/ngram_tokenizer_test.cpp | 6 +-
.../tokenizer/standard_tokenizer_factory_test.cpp | 6 +-
.../segment_v2/inverted_index/util/reader_test.cpp | 92 ++++++
.../antlr4/org/apache/doris/nereids/DorisLexer.g4 | 1 +
.../antlr4/org/apache/doris/nereids/DorisParser.g4 | 4 +
.../java/org/apache/doris/analysis/IndexDef.java | 5 +-
.../apache/doris/analysis/InvertedIndexUtil.java | 38 ++-
.../main/java/org/apache/doris/catalog/Index.java | 16 +-
...Validator.java => BasicTokenizerValidator.java} | 23 +-
...or.java => CharReplaceCharFilterValidator.java} | 30 +-
...erValidator.java => ICUTokenizerValidator.java} | 9 +-
.../org/apache/doris/indexpolicy/IndexPolicy.java | 8 +-
.../apache/doris/indexpolicy/IndexPolicyMgr.java | 69 ++++-
.../doris/indexpolicy/IndexPolicyTypeEnum.java | 3 +-
...zerValidator.java => NoOperationValidator.java} | 10 +-
.../doris/nereids/parser/LogicalPlanBuilder.java | 28 ++
.../apache/doris/nereids/trees/plans/PlanType.java | 5 +-
.../commands/CreateIndexCharFilterCommand.java | 75 +++++
.../plans/commands/DropIndexCharFilterCommand.java | 62 ++++
.../plans/commands/ShowIndexCharFilterCommand.java | 53 ++++
.../trees/plans/commands/info/IndexDefinition.java | 13 +-
.../trees/plans/visitor/CommandVisitor.java | 18 ++
gensrc/thrift/AgentService.thrift | 3 +-
.../analyzer/test_custom_analyzer.out | 6 +
.../analyzer/test_custom_analyzer1.out | 6 +
.../analyzer/test_custom_analyzer2.out | 16 +
.../analyzer/test_custom_analyzer.groovy | 26 +-
.../analyzer/test_custom_analyzer1.groovy | 69 ++++-
.../analyzer/test_custom_analyzer2.groovy | 62 ++++
94 files changed, 2826 insertions(+), 568 deletions(-)
diff --git a/be/src/clucene b/be/src/clucene
index 15bafbb370d..ceffc1cf36f 160000
--- a/be/src/clucene
+++ b/be/src/clucene
@@ -1 +1 @@
-Subproject commit 15bafbb370dec3c4fb3964b4efd6875ed445857b
+Subproject commit ceffc1cf36f0d22ce47bb17dbff607fa6ac8d5f5
diff --git a/be/src/olap/inverted_index_parser.cpp b/be/src/olap/inverted_index_parser.cpp
index 1d892b9357d..3a7775d8130 100644
--- a/be/src/olap/inverted_index_parser.cpp
+++ b/be/src/olap/inverted_index_parser.cpp
@@ -17,7 +17,6 @@
#include "olap/inverted_index_parser.h"
-#include "olap/rowset/segment_v2/inverted_index/char_filter/char_filter_factory.h"
#include "util/string_util.h"
namespace doris {
@@ -70,11 +69,15 @@ InvertedIndexParserType get_inverted_index_parser_type_from_string(const std::st
std::string get_parser_string_from_properties(
const std::map<std::string, std::string>& properties) {
- if (properties.find(INVERTED_INDEX_PARSER_KEY) != properties.end()) {
- return properties.at(INVERTED_INDEX_PARSER_KEY);
- } else {
- return INVERTED_INDEX_PARSER_NONE;
+ auto it = properties.find(INVERTED_INDEX_PARSER_KEY);
+ if (it != properties.end()) {
+ return it->second;
}
+ it = properties.find(INVERTED_INDEX_PARSER_KEY_ALIAS);
+ if (it != properties.end()) {
+ return it->second;
+ }
+ return INVERTED_INDEX_PARSER_NONE;
}
std::string get_parser_mode_string_from_properties(
@@ -83,6 +86,9 @@ std::string get_parser_mode_string_from_properties(
return properties.at(INVERTED_INDEX_PARSER_MODE_KEY);
} else {
auto parser_it = properties.find(INVERTED_INDEX_PARSER_KEY);
+ if (parser_it == properties.end()) {
+ parser_it = properties.find(INVERTED_INDEX_PARSER_KEY_ALIAS);
+ }
if (parser_it != properties.end() && parser_it->second == INVERTED_INDEX_PARSER_IK) {
return INVERTED_INDEX_PARSER_SMART;
}
diff --git a/be/src/olap/inverted_index_parser.h b/be/src/olap/inverted_index_parser.h
index a276d7de4f3..075a4f8e543 100644
--- a/be/src/olap/inverted_index_parser.h
+++ b/be/src/olap/inverted_index_parser.h
@@ -68,6 +68,7 @@ const std::string INVERTED_INDEX_PARSER_MAX_WORD = "ik_max_word";
const std::string INVERTED_INDEX_PARSER_SMART = "ik_smart";
const std::string INVERTED_INDEX_PARSER_KEY = "parser";
+const std::string INVERTED_INDEX_PARSER_KEY_ALIAS = "built_in_analyzer";
const std::string INVERTED_INDEX_PARSER_UNKNOWN = "unknown";
const std::string INVERTED_INDEX_PARSER_NONE = "none";
const std::string INVERTED_INDEX_PARSER_STANDARD = "standard";
@@ -85,6 +86,7 @@ const std::string INVERTED_INDEX_PARSER_PHRASE_SUPPORT_NO = "false";
const std::string INVERTED_INDEX_PARSER_CHAR_FILTER_TYPE = "char_filter_type";
const std::string INVERTED_INDEX_PARSER_CHAR_FILTER_PATTERN = "char_filter_pattern";
const std::string INVERTED_INDEX_PARSER_CHAR_FILTER_REPLACEMENT = "char_filter_replacement";
+const std::string INVERTED_INDEX_CHAR_FILTER_CHAR_REPLACE = "char_replace";
const std::string INVERTED_INDEX_PARSER_IGNORE_ABOVE_KEY = "ignore_above";
const std::string INVERTED_INDEX_PARSER_IGNORE_ABOVE_VALUE = "256";
diff --git a/be/src/olap/rowset/segment_v2/inverted_index/analysis_factory_mgr.cpp b/be/src/olap/rowset/segment_v2/inverted_index/analysis_factory_mgr.cpp
index 51585e5580b..78511e6bd3c 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index/analysis_factory_mgr.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index/analysis_factory_mgr.cpp
@@ -17,10 +17,16 @@
#include "analysis_factory_mgr.h"
+#include "olap/rowset/segment_v2/inverted_index/char_filter/char_replace_char_filter_factory.h"
+#include "olap/rowset/segment_v2/inverted_index/char_filter/empty_char_filter_factory.h"
#include "olap/rowset/segment_v2/inverted_index/token_filter/ascii_folding_filter_factory.h"
+#include "olap/rowset/segment_v2/inverted_index/token_filter/empty_token_filter_factory.h"
#include "olap/rowset/segment_v2/inverted_index/token_filter/lower_case_filter_factory.h"
#include "olap/rowset/segment_v2/inverted_index/token_filter/word_delimiter_filter_factory.h"
+#include "olap/rowset/segment_v2/inverted_index/tokenizer/basic/basic_tokenizer_factory.h"
#include "olap/rowset/segment_v2/inverted_index/tokenizer/char/char_group_tokenizer_factory.h"
+#include "olap/rowset/segment_v2/inverted_index/tokenizer/empty/empty_tokenizer_factory.h"
+#include "olap/rowset/segment_v2/inverted_index/tokenizer/icu/icu_tokenizer_factory.h"
#include "olap/rowset/segment_v2/inverted_index/tokenizer/keyword/keyword_tokenizer_factory.h"
#include "olap/rowset/segment_v2/inverted_index/tokenizer/ngram/edge_ngram_tokenizer_factory.h"
#include "olap/rowset/segment_v2/inverted_index/tokenizer/standard/standard_tokenizer_factory.h"
@@ -30,26 +36,46 @@ namespace doris::segment_v2::inverted_index {
void AnalysisFactoryMgr::initialise() {
static std::once_flag once_flag;
std::call_once(once_flag, [this]() {
+ // char_filter
+ registerFactory<CharFilterFactory>(
+ "empty", []() { return std::make_shared<EmptyCharFilterFactory>(); });
+ registerFactory<CharFilterFactory>(
+ "char_replace", []() { return std::make_shared<CharReplaceCharFilterFactory>(); });
+
// tokenizer
- registerFactory("standard", []() { return std::make_shared<StandardTokenizerFactory>(); });
- registerFactory("keyword", []() { return std::make_shared<KeywordTokenizerFactory>(); });
- registerFactory("ngram", []() { return std::make_shared<NGramTokenizerFactory>(); });
- registerFactory("edge_ngram",
- []() { return std::make_shared<EdgeNGramTokenizerFactory>(); });
- registerFactory("char_group",
- []() { return std::make_shared<CharGroupTokenizerFactory>(); });
+ registerFactory<TokenizerFactory>(
+ "empty", []() { return std::make_shared<EmptyTokenizerFactory>(); });
+ registerFactory<TokenizerFactory>(
+ "standard", []() { return std::make_shared<StandardTokenizerFactory>(); });
+ registerFactory<TokenizerFactory>(
+ "keyword", []() { return std::make_shared<KeywordTokenizerFactory>(); });
+ registerFactory<TokenizerFactory>(
+ "ngram", []() { return std::make_shared<NGramTokenizerFactory>(); });
+ registerFactory<TokenizerFactory>(
+ "edge_ngram", []() { return std::make_shared<EdgeNGramTokenizerFactory>(); });
+ registerFactory<TokenizerFactory>(
+ "char_group", []() { return std::make_shared<CharGroupTokenizerFactory>(); });
+ registerFactory<TokenizerFactory>(
+ "basic", []() { return std::make_shared<BasicTokenizerFactory>(); });
+ registerFactory<TokenizerFactory>("icu",
+ []() { return std::make_shared<ICUTokenizerFactory>(); });
// token_filter
- registerFactory("lowercase", []() { return std::make_shared<LowerCaseFilterFactory>(); });
- registerFactory("asciifolding",
- []() { return std::make_shared<ASCIIFoldingFilterFactory>(); });
- registerFactory("word_delimiter",
- []() { return std::make_shared<WordDelimiterFilterFactory>(); });
+ registerFactory<TokenFilterFactory>(
+ "empty", []() { return std::make_shared<EmptyTokenFilterFactory>(); });
+ registerFactory<TokenFilterFactory>(
+ "lowercase", []() { return std::make_shared<LowerCaseFilterFactory>(); });
+ registerFactory<TokenFilterFactory>(
+ "asciifolding", []() { return std::make_shared<ASCIIFoldingFilterFactory>(); });
+ registerFactory<TokenFilterFactory>(
+ "word_delimiter", []() { return std::make_shared<WordDelimiterFilterFactory>(); });
});
}
+template <typename FactoryType>
void AnalysisFactoryMgr::registerFactory(const std::string& name, FactoryCreator creator) {
- registry_[name] = std::move(creator);
+ RegistryKey key = {std::type_index(typeid(FactoryType)), name};
+ registry_[key] = std::move(creator);
}
template <typename FactoryType>
@@ -59,9 +85,11 @@ std::shared_ptr<FactoryType> AnalysisFactoryMgr::create(const std::string& name,
initialise();
}
- auto it = registry_.find(name);
+ RegistryKey key = {std::type_index(typeid(FactoryType)), name};
+ auto it = registry_.find(key);
if (it == registry_.end()) {
- throw Exception(ErrorCode::INVALID_ARGUMENT, "Unknown factory name: {}", name);
+ throw Exception(ErrorCode::INVALID_ARGUMENT, "Unknown factory name: {} for type: {}", name,
+ typeid(FactoryType).name());
}
auto factory = std::static_pointer_cast<FactoryType>(it->second());
@@ -75,4 +103,7 @@ template std::shared_ptr<TokenizerFactory> AnalysisFactoryMgr::create<TokenizerF
template std::shared_ptr<TokenFilterFactory> AnalysisFactoryMgr::create<TokenFilterFactory>(
const std::string&, const Settings&);
+template std::shared_ptr<CharFilterFactory> AnalysisFactoryMgr::create<CharFilterFactory>(
+ const std::string&, const Settings&);
+
} // namespace doris::segment_v2::inverted_index
\ No newline at end of file
diff --git a/be/src/olap/rowset/segment_v2/inverted_index/analysis_factory_mgr.h b/be/src/olap/rowset/segment_v2/inverted_index/analysis_factory_mgr.h
index 71c2ce14425..3953b16b207 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index/analysis_factory_mgr.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index/analysis_factory_mgr.h
@@ -17,7 +17,8 @@
#pragma once
-#include "common/exception.h"
+#include <typeindex>
+
#include "olap/rowset/segment_v2/inverted_index/abstract_analysis_factory.h"
#include "olap/rowset/segment_v2/inverted_index/setting.h"
@@ -26,6 +27,7 @@ namespace doris::segment_v2::inverted_index {
class AnalysisFactoryMgr {
public:
using FactoryCreator = std::function<AbstractAnalysisFactoryPtr()>;
+ using RegistryKey = std::pair<std::type_index, std::string>;
AnalysisFactoryMgr(const AnalysisFactoryMgr&) = delete;
AnalysisFactoryMgr& operator=(const AnalysisFactoryMgr&) = delete;
@@ -36,6 +38,8 @@ public:
}
void initialise();
+
+ template <typename FactoryType>
void registerFactory(const std::string& name, FactoryCreator creator);
template <typename FactoryType>
@@ -45,7 +49,7 @@ private:
AnalysisFactoryMgr() = default;
~AnalysisFactoryMgr() = default;
- std::map<std::string, FactoryCreator> registry_;
+ std::map<RegistryKey, FactoryCreator> registry_;
};
} // namespace doris::segment_v2::inverted_index
\ No newline at end of file
diff --git a/be/src/olap/rowset/segment_v2/inverted_index/analyzer/analyzer.cpp b/be/src/olap/rowset/segment_v2/inverted_index/analyzer/analyzer.cpp
index eea8e20ab9c..bc81f0a7685 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index/analyzer/analyzer.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index/analyzer/analyzer.cpp
@@ -31,22 +31,21 @@
#include "olap/rowset/segment_v2/inverted_index/analyzer/basic/basic_analyzer.h"
#include "olap/rowset/segment_v2/inverted_index/analyzer/icu/icu_analyzer.h"
#include "olap/rowset/segment_v2/inverted_index/analyzer/ik/IKAnalyzer.h"
-#include "olap/rowset/segment_v2/inverted_index/char_filter/char_filter_factory.h"
+#include "olap/rowset/segment_v2/inverted_index/char_filter/char_replace_char_filter_factory.h"
#include "runtime/exec_env.h"
#include "runtime/index_policy/index_policy_mgr.h"
-#include "util/runtime_profile.h"
namespace doris::segment_v2::inverted_index {
-std::unique_ptr<lucene::util::Reader> InvertedIndexAnalyzer::create_reader(
- CharFilterMap& char_filter_map) {
- std::unique_ptr<lucene::util::Reader> reader =
- std::make_unique<lucene::util::SStringReader<char>>();
+ReaderPtr InvertedIndexAnalyzer::create_reader(CharFilterMap& char_filter_map) {
+ ReaderPtr reader = std::make_shared<lucene::util::SStringReader<char>>();
if (!char_filter_map.empty()) {
- reader = std::unique_ptr<lucene::util::Reader>(CharFilterFactory::create(
- char_filter_map[INVERTED_INDEX_PARSER_CHAR_FILTER_TYPE], reader.release(),
- char_filter_map[INVERTED_INDEX_PARSER_CHAR_FILTER_PATTERN],
- char_filter_map[INVERTED_INDEX_PARSER_CHAR_FILTER_REPLACEMENT]));
+ if (char_filter_map[INVERTED_INDEX_PARSER_CHAR_FILTER_TYPE] ==
+ INVERTED_INDEX_CHAR_FILTER_CHAR_REPLACE) {
+ reader = std::make_shared<CharReplaceCharFilter>(
+ reader, char_filter_map[INVERTED_INDEX_PARSER_CHAR_FILTER_PATTERN],
+ char_filter_map[INVERTED_INDEX_PARSER_CHAR_FILTER_REPLACEMENT]);
+ }
}
return reader;
}
@@ -113,7 +112,7 @@ std::shared_ptr<lucene::analysis::Analyzer> InvertedIndexAnalyzer::create_analyz
}
std::vector<TermInfo> InvertedIndexAnalyzer::get_analyse_result(
- lucene::util::Reader* reader, lucene::analysis::Analyzer* analyzer) {
+ ReaderPtr reader, lucene::analysis::Analyzer* analyzer) {
std::vector<TermInfo> analyse_result;
std::unique_ptr<lucene::analysis::TokenStream> token_stream(analyzer->tokenStream(L"", reader));
@@ -152,7 +151,7 @@ std::vector<TermInfo> InvertedIndexAnalyzer::get_analyse_result(
inverted_index_ctx->analyzer = analyzer.get();
auto reader = create_reader(inverted_index_ctx->char_filter_map);
reader->init(search_str.data(), static_cast<int32_t>(search_str.size()), true);
- return get_analyse_result(reader.get(), analyzer.get());
+ return get_analyse_result(reader, analyzer.get());
}
bool InvertedIndexAnalyzer::should_analyzer(const std::map<std::string, std::string>& properties) {
diff --git a/be/src/olap/rowset/segment_v2/inverted_index/analyzer/analyzer.h b/be/src/olap/rowset/segment_v2/inverted_index/analyzer/analyzer.h
index 756aa93aae3..df777e88889 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index/analyzer/analyzer.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index/analyzer/analyzer.h
@@ -23,6 +23,7 @@
#include "olap/inverted_index_parser.h"
#include "olap/olap_common.h"
#include "olap/rowset/segment_v2/inverted_index/query/query.h"
+#include "olap/rowset/segment_v2/inverted_index/util/reader.h"
#include "olap/rowset/segment_v2/inverted_index_query_type.h"
namespace lucene {
@@ -37,12 +38,12 @@ class Analyzer;
namespace doris::segment_v2::inverted_index {
class InvertedIndexAnalyzer {
public:
- static std::unique_ptr<lucene::util::Reader> create_reader(CharFilterMap& char_filter_map);
+ static ReaderPtr create_reader(CharFilterMap& char_filter_map);
static std::shared_ptr<lucene::analysis::Analyzer> create_analyzer(
const InvertedIndexCtx* inverted_index_ctx);
- static std::vector<TermInfo> get_analyse_result(lucene::util::Reader* reader,
+ static std::vector<TermInfo> get_analyse_result(ReaderPtr reader,
lucene::analysis::Analyzer* analyzer);
static std::vector<TermInfo> get_analyse_result(
diff --git a/be/src/olap/rowset/segment_v2/inverted_index/analyzer/basic/basic_analyzer.h b/be/src/olap/rowset/segment_v2/inverted_index/analyzer/basic/basic_analyzer.h
index b9f4f963666..8c24cc2d999 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index/analyzer/basic/basic_analyzer.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index/analyzer/basic/basic_analyzer.h
@@ -17,9 +17,9 @@
#pragma once
-#include <memory>
-
-#include "basic_tokenizer.h"
+#include "olap/rowset/segment_v2/inverted_index/token_filter/lower_case_filter.h"
+#include "olap/rowset/segment_v2/inverted_index/token_stream.h"
+#include "olap/rowset/segment_v2/inverted_index/tokenizer/basic/basic_tokenizer.h"
namespace doris::segment_v2 {
@@ -35,22 +35,47 @@ public:
bool isSDocOpt() override { return true; }
TokenStream* tokenStream(const TCHAR* fieldName, lucene::util::Reader* reader) override {
- auto* tokenizer = _CLNEW BasicTokenizer(_lowercase, _ownReader);
- tokenizer->reset(reader);
- return (TokenStream*)tokenizer;
+ throw Exception(ErrorCode::INVERTED_INDEX_NOT_SUPPORTED,
+ "BasicAnalyzer::tokenStream not supported");
}
TokenStream* reusableTokenStream(const TCHAR* fieldName,
lucene::util::Reader* reader) override {
- if (_tokenizer == nullptr) {
- _tokenizer = std::make_unique<BasicTokenizer>(_lowercase, _ownReader);
+ throw Exception(ErrorCode::INVERTED_INDEX_NOT_SUPPORTED,
+ "BasicAnalyzer::reusableTokenStream not supported");
+ }
+
+ TokenStream* tokenStream(const TCHAR* fieldName,
+ const inverted_index::ReaderPtr& reader) override {
+ auto token_stream = create_components();
+ token_stream->set_reader(reader);
+ token_stream->get_token_stream()->reset();
+ return new inverted_index::TokenStreamWrapper(token_stream->get_token_stream());
+ }
+
+ TokenStream* reusableTokenStream(const TCHAR* fieldName,
+ const inverted_index::ReaderPtr& reader) override {
+ if (_reuse_token_stream == nullptr) {
+ _reuse_token_stream = create_components();
}
- _tokenizer->reset(reader);
- return (TokenStream*)_tokenizer.get();
+ _reuse_token_stream->set_reader(reader);
+ return _reuse_token_stream->get_token_stream().get();
};
private:
- std::unique_ptr<BasicTokenizer> _tokenizer;
+ inverted_index::TokenStreamComponentsPtr create_components() {
+ auto tk = std::make_shared<inverted_index::BasicTokenizer>();
+ tk->initialize();
+ inverted_index::TokenStreamPtr ts = tk;
+ if (_lowercase) {
+ auto lower_case_filter = std::make_shared<inverted_index::LowerCaseFilter>(tk);
+ lower_case_filter->initialize();
+ ts = lower_case_filter;
+ }
+ return std::make_shared<inverted_index::TokenStreamComponents>(tk, ts);
+ }
+
+ inverted_index::TokenStreamComponentsPtr _reuse_token_stream;
};
} // namespace doris::segment_v2
\ No newline at end of file
diff --git a/be/src/olap/rowset/segment_v2/inverted_index/analyzer/custom_analyzer.cpp b/be/src/olap/rowset/segment_v2/inverted_index/analyzer/custom_analyzer.cpp
index c90b71a2e36..312abd52337 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index/analyzer/custom_analyzer.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index/analyzer/custom_analyzer.cpp
@@ -17,44 +17,54 @@
#include "custom_analyzer.h"
+#include "common/status.h"
#include "olap/rowset/segment_v2/inverted_index/analysis_factory_mgr.h"
+#include "olap/rowset/segment_v2/inverted_index/token_stream.h"
#include "runtime/exec_env.h"
namespace doris::segment_v2::inverted_index {
CustomAnalyzer::CustomAnalyzer(Builder* builder) {
_tokenizer = builder->_tokenizer;
+ _char_filters = builder->_char_filters;
_token_filters = builder->_token_filters;
}
TokenStream* CustomAnalyzer::tokenStream(const TCHAR* fieldName, lucene::util::Reader* reader) {
- class TokenStreamWrapper : public TokenStream {
- public:
- explicit TokenStreamWrapper(std::shared_ptr<TokenStream> ts) : _impl(std::move(ts)) {}
- ~TokenStreamWrapper() override = default;
-
- Token* next(Token* token) override { return _impl->next(token); }
- void close() override { _impl->close(); }
- void reset() override { _impl->reset(); }
-
- private:
- std::shared_ptr<TokenStream> _impl;
- };
+ throw Exception(ErrorCode::INVERTED_INDEX_NOT_SUPPORTED,
+ "CustomAnalyzer::tokenStream not supported");
+}
+
+TokenStream* CustomAnalyzer::reusableTokenStream(const TCHAR* fieldName,
+ lucene::util::Reader* reader) {
+ throw Exception(ErrorCode::INVERTED_INDEX_NOT_SUPPORTED,
+ "CustomAnalyzer::reusableTokenStream not supported");
+}
+
+TokenStream* CustomAnalyzer::tokenStream(const TCHAR* fieldName, const ReaderPtr& reader) {
+ auto r = init_reader(reader);
auto token_stream = create_components();
- token_stream->set_reader(reader);
+ token_stream->set_reader(r);
token_stream->get_token_stream()->reset();
return new TokenStreamWrapper(token_stream->get_token_stream());
}
-TokenStream* CustomAnalyzer::reusableTokenStream(const TCHAR* fieldName,
- lucene::util::Reader* reader) {
+TokenStream* CustomAnalyzer::reusableTokenStream(const TCHAR* fieldName, const ReaderPtr& reader) {
+ auto r = init_reader(reader);
if (_reuse_token_stream == nullptr) {
_reuse_token_stream = create_components();
}
- _reuse_token_stream->set_reader(reader);
+ _reuse_token_stream->set_reader(r);
return _reuse_token_stream->get_token_stream().get();
}
+ReaderPtr CustomAnalyzer::init_reader(ReaderPtr reader) {
+ for (const auto& filter : _char_filters) {
+ reader = filter->create(reader);
+ }
+ return reader;
+}
+
TokenStreamComponentsPtr CustomAnalyzer::create_components() {
auto tk = _tokenizer->create();
TokenStreamPtr ts = tk;
@@ -69,6 +79,9 @@ CustomAnalyzerPtr CustomAnalyzer::build_custom_analyzer(const CustomAnalyzerConf
throw Exception(ErrorCode::ILLEGAL_STATE, "Null configuration detected.");
}
CustomAnalyzer::Builder builder;
+ for (const auto& filter_config : config->get_char_filter_configs()) {
+ builder.add_char_filter(filter_config->get_name(), filter_config->get_params());
+ }
builder.with_tokenizer(config->get_tokenizer_config()->get_name(),
config->get_tokenizer_config()->get_params());
for (const auto& filter_config : config->get_token_filter_configs()) {
@@ -81,6 +94,10 @@ void CustomAnalyzer::Builder::with_tokenizer(const std::string& name, const Sett
_tokenizer = AnalysisFactoryMgr::instance().create<TokenizerFactory>(name, params);
}
+void CustomAnalyzer::Builder::add_char_filter(const std::string& name, const Settings& params) {
+ _char_filters.push_back(AnalysisFactoryMgr::instance().create<CharFilterFactory>(name, params));
+}
+
void CustomAnalyzer::Builder::add_token_filter(const std::string& name, const Settings& params) {
_token_filters.push_back(
AnalysisFactoryMgr::instance().create<TokenFilterFactory>(name, params));
@@ -93,7 +110,7 @@ CustomAnalyzerPtr CustomAnalyzer::Builder::build() {
return std::make_shared<CustomAnalyzer>(this);
}
-void TokenStreamComponents::set_reader(CL_NS(util)::Reader* reader) {
+void TokenStreamComponents::set_reader(const ReaderPtr& reader) {
_source->set_reader(reader);
}
diff --git a/be/src/olap/rowset/segment_v2/inverted_index/analyzer/custom_analyzer.h b/be/src/olap/rowset/segment_v2/inverted_index/analyzer/custom_analyzer.h
index 7b3bc0444ae..cb7294e7e64 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index/analyzer/custom_analyzer.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index/analyzer/custom_analyzer.h
@@ -17,18 +17,14 @@
#pragma once
-#include "common/exception.h"
-#include "olap/rowset/segment_v2/inverted_index/analysis_factory_mgr.h"
#include "olap/rowset/segment_v2/inverted_index/analyzer/custom_analyzer_config.h"
+#include "olap/rowset/segment_v2/inverted_index/char_filter/char_filter_factory.h"
#include "olap/rowset/segment_v2/inverted_index/setting.h"
#include "olap/rowset/segment_v2/inverted_index/token_filter/token_filter_factory.h"
#include "olap/rowset/segment_v2/inverted_index/tokenizer/tokenizer_factory.h"
namespace doris::segment_v2::inverted_index {
-class TokenStreamComponents;
-using TokenStreamComponentsPtr = std::shared_ptr<TokenStreamComponents>;
-
class CustomAnalyzer;
using CustomAnalyzerPtr = std::shared_ptr<CustomAnalyzer>;
@@ -40,11 +36,14 @@ public:
~Builder() = default;
void with_tokenizer(const std::string& name, const Settings& params);
+ void add_char_filter(const std::string& name, const Settings& params);
void add_token_filter(const std::string& name, const Settings& params);
+
CustomAnalyzerPtr build();
private:
TokenizerFactoryPtr _tokenizer;
+ std::vector<CharFilterFactoryPtr> _char_filters;
std::vector<TokenFilterFactoryPtr> _token_filters;
friend class CustomAnalyzer;
@@ -58,29 +57,20 @@ public:
TokenStream* tokenStream(const TCHAR* fieldName, lucene::util::Reader* reader) override;
TokenStream* reusableTokenStream(const TCHAR* fieldName, lucene::util::Reader* reader) override;
+ TokenStream* tokenStream(const TCHAR* fieldName, const ReaderPtr& reader) override;
+ TokenStream* reusableTokenStream(const TCHAR* fieldName, const ReaderPtr& reader) override;
+
static CustomAnalyzerPtr build_custom_analyzer(const CustomAnalyzerConfigPtr& config);
private:
+ ReaderPtr init_reader(ReaderPtr reader);
TokenStreamComponentsPtr create_components();
TokenizerFactoryPtr _tokenizer;
+ std::vector<CharFilterFactoryPtr> _char_filters;
std::vector<TokenFilterFactoryPtr> _token_filters;
TokenStreamComponentsPtr _reuse_token_stream;
};
-class TokenStreamComponents {
-public:
- TokenStreamComponents(TokenizerPtr tokenizer, TokenStreamPtr result)
- : _source(std::move(tokenizer)), _sink(std::move(result)) {}
-
- void set_reader(CL_NS(util)::Reader* reader);
- TokenStreamPtr get_token_stream();
- TokenizerPtr get_source();
-
-private:
- TokenizerPtr _source;
- TokenStreamPtr _sink;
-};
-
} // namespace doris::segment_v2::inverted_index
\ No newline at end of file
diff --git a/be/src/olap/rowset/segment_v2/inverted_index/analyzer/custom_analyzer_config.cpp b/be/src/olap/rowset/segment_v2/inverted_index/analyzer/custom_analyzer_config.cpp
index f1d593ecfa5..161d267efb6 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index/analyzer/custom_analyzer_config.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index/analyzer/custom_analyzer_config.cpp
@@ -23,6 +23,7 @@ namespace doris::segment_v2::inverted_index {
CustomAnalyzerConfig::CustomAnalyzerConfig(Builder* builder) {
_tokenizer_config = builder->_tokenizer_config;
+ _char_filters = builder->_char_filters;
_token_filters = builder->_token_filters;
}
@@ -30,6 +31,10 @@ ComponentConfigPtr CustomAnalyzerConfig::get_tokenizer_config() {
return _tokenizer_config;
}
+std::vector<ComponentConfigPtr> CustomAnalyzerConfig::get_char_filter_configs() {
+ return _char_filters;
+}
+
std::vector<ComponentConfigPtr> CustomAnalyzerConfig::get_token_filter_configs() {
return _token_filters;
}
@@ -39,6 +44,11 @@ void CustomAnalyzerConfig::Builder::with_tokenizer_config(const std::string& nam
_tokenizer_config = std::make_shared<ComponentConfig>(name, params);
}
+void CustomAnalyzerConfig::Builder::add_char_filter_config(const std::string& name,
+ const Settings& params) {
+ _char_filters.emplace_back(std::make_shared<ComponentConfig>(name, params));
+}
+
void CustomAnalyzerConfig::Builder::add_token_filter_config(const std::string& name,
const Settings& params) {
_token_filters.emplace_back(std::make_shared<ComponentConfig>(name, params));
diff --git a/be/src/olap/rowset/segment_v2/inverted_index/analyzer/custom_analyzer_config.h b/be/src/olap/rowset/segment_v2/inverted_index/analyzer/custom_analyzer_config.h
index 0df8507e26a..134d4ee0d45 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index/analyzer/custom_analyzer_config.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index/analyzer/custom_analyzer_config.h
@@ -39,11 +39,13 @@ public:
~Builder() = default;
void with_tokenizer_config(const std::string& name, const Settings& params);
+ void add_char_filter_config(const std::string& name, const Settings& params);
void add_token_filter_config(const std::string& name, const Settings& params);
CustomAnalyzerConfigPtr build();
private:
ComponentConfigPtr _tokenizer_config;
+ std::vector<ComponentConfigPtr> _char_filters;
std::vector<ComponentConfigPtr> _token_filters;
friend class CustomAnalyzerConfig;
@@ -53,10 +55,12 @@ public:
~CustomAnalyzerConfig() = default;
ComponentConfigPtr get_tokenizer_config();
+ std::vector<ComponentConfigPtr> get_char_filter_configs();
std::vector<ComponentConfigPtr> get_token_filter_configs();
private:
ComponentConfigPtr _tokenizer_config;
+ std::vector<ComponentConfigPtr> _char_filters;
std::vector<ComponentConfigPtr> _token_filters;
};
diff --git a/be/src/olap/rowset/segment_v2/inverted_index/analyzer/icu/icu_analyzer.h b/be/src/olap/rowset/segment_v2/inverted_index/analyzer/icu/icu_analyzer.h
index 072cf85bc7d..ccf27dfc8cb 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index/analyzer/icu/icu_analyzer.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index/analyzer/icu/icu_analyzer.h
@@ -17,9 +17,9 @@
#pragma once
-#include <memory>
-
-#include "icu_tokenizer.h"
+#include "olap/rowset/segment_v2/inverted_index/token_filter/lower_case_filter.h"
+#include "olap/rowset/segment_v2/inverted_index/token_stream.h"
+#include "olap/rowset/segment_v2/inverted_index/tokenizer/icu/icu_tokenizer.h"
namespace doris::segment_v2 {
@@ -37,25 +37,48 @@ public:
void initDict(const std::string& dictPath) override { dictPath_ = dictPath; }
TokenStream* tokenStream(const TCHAR* fieldName, lucene::util::Reader* reader) override {
- auto* tokenizer = _CLNEW ICUTokenizer(_lowercase, _ownReader);
- tokenizer->initialize(dictPath_);
- tokenizer->reset(reader);
- return (TokenStream*)tokenizer;
+ throw Exception(ErrorCode::INVERTED_INDEX_NOT_SUPPORTED,
+ "ICUAnalyzer::tokenStream not supported");
}
TokenStream* reusableTokenStream(const TCHAR* fieldName,
lucene::util::Reader* reader) override {
- if (tokenizer_ == nullptr) {
- tokenizer_ = std::make_unique<ICUTokenizer>(_lowercase, _ownReader);
- tokenizer_->initialize(dictPath_);
+ throw Exception(ErrorCode::INVERTED_INDEX_NOT_SUPPORTED,
+ "ICUAnalyzer::reusableTokenStream not supported");
+ }
+
+ TokenStream* tokenStream(const TCHAR* fieldName,
+ const inverted_index::ReaderPtr& reader) override {
+ auto token_stream = create_components();
+ token_stream->set_reader(reader);
+ token_stream->get_token_stream()->reset();
+ return new inverted_index::TokenStreamWrapper(token_stream->get_token_stream());
+ }
+
+ TokenStream* reusableTokenStream(const TCHAR* fieldName,
+ const inverted_index::ReaderPtr& reader) override {
+ if (_reuse_token_stream == nullptr) {
+ _reuse_token_stream = create_components();
}
- tokenizer_->reset(reader);
- return (TokenStream*)tokenizer_.get();
+ _reuse_token_stream->set_reader(reader);
+ return _reuse_token_stream->get_token_stream().get();
};
private:
+ inverted_index::TokenStreamComponentsPtr create_components() {
+ auto tk = std::make_shared<inverted_index::ICUTokenizer>();
+ tk->initialize(dictPath_);
+ inverted_index::TokenStreamPtr ts = tk;
+ if (_lowercase) {
+ auto lower_case_filter = std::make_shared<inverted_index::LowerCaseFilter>(tk);
+ lower_case_filter->initialize();
+ ts = lower_case_filter;
+ }
+ return std::make_shared<inverted_index::TokenStreamComponents>(tk, ts);
+ }
+
std::string dictPath_;
- std::unique_ptr<ICUTokenizer> tokenizer_;
+ inverted_index::TokenStreamComponentsPtr _reuse_token_stream;
};
} // namespace doris::segment_v2
\ No newline at end of file
diff --git a/be/src/olap/rowset/segment_v2/inverted_index/tokenizer/tokenizer.h b/be/src/olap/rowset/segment_v2/inverted_index/char_filter/char_filter.h
similarity index 52%
copy from be/src/olap/rowset/segment_v2/inverted_index/tokenizer/tokenizer.h
copy to be/src/olap/rowset/segment_v2/inverted_index/char_filter/char_filter.h
index 6a7119b5fc5..7e4c3a849f3 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index/tokenizer/tokenizer.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index/char_filter/char_filter.h
@@ -17,36 +17,35 @@
#pragma once
-#include <unicode/utf8.h>
+#include <memory>
-#include <string_view>
-
-#include "olap/rowset/segment_v2/inverted_index/token_stream.h"
-
-using TokenStreamPtr = std::shared_ptr<TokenStream>;
+#include "common/exception.h"
+#include "olap/rowset/segment_v2/inverted_index/util/reader.h"
namespace doris::segment_v2::inverted_index {
-class DorisTokenizer : public Tokenizer, public DorisTokenStream {
+class DorisCharFilter : public lucene::util::Reader {
public:
- DorisTokenizer() = default;
- ~DorisTokenizer() override = default;
-
- void set_reader(CL_NS(util)::Reader* in) {
- if (in == nullptr) {
- throw Exception(ErrorCode::INVALID_ARGUMENT, "reader must not be null");
- }
- _in_pending = in;
+ DorisCharFilter(ReaderPtr reader) : _reader(std::move(reader)) {}
+ ~DorisCharFilter() override = default;
+
+ virtual void initialize() = 0;
+
+ int64_t position() override {
+ throw Exception(ErrorCode::INVERTED_INDEX_NOT_SUPPORTED, "CharFilter::position");
}
- using Tokenizer::reset;
- // Only use the parameterless reset method
- void reset() override { _in = _in_pending; };
+ int64_t skip(int64_t ntoskip) override {
+ throw Exception(ErrorCode::INVERTED_INDEX_NOT_SUPPORTED, "CharFilter::skip");
+ }
+
+ size_t size() override {
+ throw Exception(ErrorCode::INVERTED_INDEX_NOT_SUPPORTED, "CharFilter::size");
+ }
protected:
- CL_NS(util)::Reader* _in = nullptr;
- CL_NS(util)::Reader* _in_pending = nullptr;
+ ReaderPtr _reader;
};
-using TokenizerPtr = std::shared_ptr<DorisTokenizer>;
+using CharFilterPtr = std::shared_ptr<DorisCharFilter>;
} // namespace doris::segment_v2::inverted_index
\ No newline at end of file
diff --git a/be/src/olap/rowset/segment_v2/inverted_index/char_filter/char_filter_factory.h b/be/src/olap/rowset/segment_v2/inverted_index/char_filter/char_filter_factory.h
index bebbea58f72..925f9adaf6e 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index/char_filter/char_filter_factory.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index/char_filter/char_filter_factory.h
@@ -17,22 +17,18 @@
#pragma once
-#include "olap/rowset/segment_v2/inverted_index/char_filter/char_replace_char_filter.h"
+#include "olap/rowset/segment_v2/inverted_index/abstract_analysis_factory.h"
+#include "olap/rowset/segment_v2/inverted_index/util/reader.h"
-namespace doris {
+namespace doris::segment_v2::inverted_index {
-static const std::string INVERTED_INDEX_CHAR_FILTER_CHAR_REPLACE = "char_replace";
-
-class CharFilterFactory {
+class CharFilterFactory : public AbstractAnalysisFactory {
public:
- template <typename... Args>
- static lucene::analysis::CharFilter* create(const std::string& name, Args&&... args) {
- DBUG_EXECUTE_IF("CharFilterFactory::create_return_nullptr", { return nullptr; })
- if (name == INVERTED_INDEX_CHAR_FILTER_CHAR_REPLACE) {
- return new CharReplaceCharFilter(std::forward<Args>(args)...);
- }
- return nullptr;
- }
+ CharFilterFactory() = default;
+ ~CharFilterFactory() override = default;
+
+ virtual ReaderPtr create(const ReaderPtr& in) = 0;
};
+using CharFilterFactoryPtr = std::shared_ptr<CharFilterFactory>;
-} // namespace doris
\ No newline at end of file
+} // namespace doris::segment_v2::inverted_index
\ No newline at end of file
diff --git a/be/src/olap/rowset/segment_v2/inverted_index/char_filter/char_replace_char_filter.cpp b/be/src/olap/rowset/segment_v2/inverted_index/char_filter/char_replace_char_filter.cpp
index c0545b9cf97..95c7bc23afa 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index/char_filter/char_replace_char_filter.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index/char_filter/char_replace_char_filter.cpp
@@ -19,16 +19,23 @@
#include <boost/algorithm/string/replace.hpp>
-namespace doris {
+namespace doris::segment_v2::inverted_index {
-CharReplaceCharFilter::CharReplaceCharFilter(lucene::util::Reader* in, const std::string& pattern,
- const std::string& replacement)
- : CharFilter(in), _replacement(replacement) {
- std::for_each(pattern.begin(), pattern.end(), [this](uint8_t c) { _patterns.set(c); });
+CharReplaceCharFilter::CharReplaceCharFilter(ReaderPtr reader, const std::string& pattern,
+ std::string replacement)
+ : DorisCharFilter(std::move(reader)), _replacement(std::move(replacement)) {
+ std::ranges::for_each(pattern, [this](uint8_t c) { _patterns.set(c); });
+}
+
+void CharReplaceCharFilter::initialize() {
+ if (_transformed_input.size() != 0) {
+ return;
+ }
+ fill();
}
void CharReplaceCharFilter::init(const void* _value, int32_t _length, bool copyData) {
- input_->init(_value, _length, copyData);
+ _reader->init(_value, _length, copyData);
fill();
}
@@ -41,8 +48,8 @@ int32_t CharReplaceCharFilter::readCopy(void* start, int32_t off, int32_t len) {
}
void CharReplaceCharFilter::fill() {
- _buf.resize(input_->size());
- input_->readCopy(_buf.data(), 0, _buf.size());
+ _buf.resize(_reader->size());
+ _reader->readCopy(_buf.data(), 0, static_cast<int32_t>(_buf.size()));
process_pattern(_buf);
_transformed_input.init(_buf.data(), _buf.size(), false);
}
@@ -56,4 +63,4 @@ void CharReplaceCharFilter::process_pattern(std::string& buf) {
}
}
-} // namespace doris
\ No newline at end of file
+} // namespace doris::segment_v2::inverted_index
diff --git a/be/src/olap/rowset/segment_v2/inverted_index/char_filter/char_replace_char_filter.h b/be/src/olap/rowset/segment_v2/inverted_index/char_filter/char_replace_char_filter.h
index 1e5e6f5d5ce..082c80ffc52 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index/char_filter/char_replace_char_filter.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index/char_filter/char_replace_char_filter.h
@@ -17,19 +17,19 @@
#pragma once
-#include <CLucene.h> // IWYU pragma: keep
-#include <CLucene/analysis/CharFilter.h>
-
#include <bitset>
-namespace doris {
+#include "olap/rowset/segment_v2/inverted_index/char_filter/char_filter.h"
+
+namespace doris::segment_v2::inverted_index {
-class CharReplaceCharFilter : public lucene::analysis::CharFilter {
+class CharReplaceCharFilter : public DorisCharFilter {
public:
- CharReplaceCharFilter(lucene::util::Reader* in, const std::string& pattern,
- const std::string& replacement);
+ CharReplaceCharFilter(ReaderPtr in, const std::string& pattern, std::string replacement);
~CharReplaceCharFilter() override = default;
+ void initialize() override;
+
void init(const void* _value, int32_t _length, bool copyData) override;
int32_t read(const void** start, int32_t min, int32_t max) override;
int32_t readCopy(void* start, int32_t off, int32_t len) override;
@@ -47,4 +47,4 @@ private:
lucene::util::SStringReader<char> _transformed_input;
};
-} // namespace doris
\ No newline at end of file
+} // namespace doris::segment_v2::inverted_index
\ No newline at end of file
diff --git a/be/src/olap/rowset/segment_v2/inverted_index/char_filter/char_replace_char_filter_factory.h b/be/src/olap/rowset/segment_v2/inverted_index/char_filter/char_replace_char_filter_factory.h
new file mode 100644
index 00000000000..debdb591071
--- /dev/null
+++ b/be/src/olap/rowset/segment_v2/inverted_index/char_filter/char_replace_char_filter_factory.h
@@ -0,0 +1,75 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include "olap/rowset/segment_v2/inverted_index/char_filter/char_filter_factory.h"
+#include "olap/rowset/segment_v2/inverted_index/char_filter/char_replace_char_filter.h"
+
+namespace doris::segment_v2::inverted_index {
+
+static const std::string CHAR_REPLACE_PATTERN = "pattern";
+static const std::string CHAR_REPLACE_REPLACEMENT = "replacement";
+
+static const std::string CHAR_REPLACE_DEFAULT_PATTERN = ",._";
+
+class CharReplaceCharFilterFactory : public CharFilterFactory {
+public:
+ CharReplaceCharFilterFactory() = default;
+ ~CharReplaceCharFilterFactory() override = default;
+
+ void initialize(const Settings& settings) override {
+ _pattern = settings.get_string(CHAR_REPLACE_PATTERN, CHAR_REPLACE_DEFAULT_PATTERN);
+ if (_pattern.empty()) {
+ throw Exception(ErrorCode::INVALID_ARGUMENT,
+ "Missing '${CHAR_REPLACE_PATTERN}' for char_replace filter type");
+ }
+ for (char ch : _pattern) {
+ unsigned int uc = static_cast<unsigned char>(ch);
+ if (uc > 255) {
+ throw Exception(ErrorCode::INVALID_ARGUMENT,
+ "Invalid '${CHAR_REPLACE_PATTERN}' for char_replace "
+ "filter type: each char must "
+ "be in [0,255]");
+ }
+ }
+ _replacement = settings.get_string(CHAR_REPLACE_REPLACEMENT, " ");
+ if (_replacement.size() != 1) {
+ throw Exception(ErrorCode::INVALID_ARGUMENT,
+ "Invalid '${CHAR_REPLACE_REPLACEMENT}' for char_replace "
+ "filter type: must be exactly 1 byte");
+ }
+ unsigned int rep = static_cast<unsigned char>(_replacement[0]);
+ if (rep > 255) {
+ throw Exception(ErrorCode::INVALID_ARGUMENT,
+ "Invalid '${CHAR_REPLACE_REPLACEMENT}' for char_replace "
+ "filter type: must be in [0,255]");
+ }
+ }
+
+ ReaderPtr create(const ReaderPtr& reader) override {
+ auto r = std::make_shared<CharReplaceCharFilter>(reader, _pattern, _replacement);
+ r->initialize();
+ return r;
+ }
+
+private:
+ std::string _pattern;
+ std::string _replacement;
+};
+
+} // namespace doris::segment_v2::inverted_index
\ No newline at end of file
diff --git a/be/src/olap/rowset/segment_v2/inverted_index/char_filter/empty_char_filter_factory.h b/be/src/olap/rowset/segment_v2/inverted_index/char_filter/empty_char_filter_factory.h
new file mode 100644
index 00000000000..ac945f4a708
--- /dev/null
+++ b/be/src/olap/rowset/segment_v2/inverted_index/char_filter/empty_char_filter_factory.h
@@ -0,0 +1,61 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include "olap/rowset/segment_v2/inverted_index/char_filter/char_filter.h"
+#include "olap/rowset/segment_v2/inverted_index/char_filter/char_filter_factory.h"
+
+namespace doris::segment_v2::inverted_index {
+
+class EmptyCharFilter : public DorisCharFilter {
+public:
+ EmptyCharFilter(ReaderPtr in) : DorisCharFilter(std::move(in)) {}
+ ~EmptyCharFilter() override = default;
+
+ void initialize() override {}
+
+ void init(const void* _value, int32_t _length, bool copyData) override {
+ _reader->init(_value, _length, copyData);
+ }
+
+ int32_t read(const void** start, int32_t min, int32_t max) override {
+ return _reader->read(start, min, max);
+ }
+
+ int32_t readCopy(void* start, int32_t off, int32_t len) override {
+ return _reader->readCopy(start, off, len);
+ }
+
+ size_t size() override { return _reader->size(); }
+};
+
+class EmptyCharFilterFactory : public CharFilterFactory {
+public:
+ EmptyCharFilterFactory() = default;
+ ~EmptyCharFilterFactory() override = default;
+
+ void initialize(const Settings& settings) override {}
+
+ ReaderPtr create(const ReaderPtr& reader) override {
+ auto filter = std::make_shared<EmptyCharFilter>(reader);
+ filter->initialize();
+ return filter;
+ }
+};
+
+} // namespace doris::segment_v2::inverted_index
\ No newline at end of file
diff --git a/be/src/olap/rowset/segment_v2/inverted_index/setting.h b/be/src/olap/rowset/segment_v2/inverted_index/setting.h
index e06deca5e4f..51782ab0b2d 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index/setting.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index/setting.h
@@ -22,8 +22,7 @@
#include <boost/algorithm/string.hpp>
#include <boost/algorithm/string/split.hpp>
#include <boost/algorithm/string/trim.hpp>
-#include <memory>
-#include <regex>
+#include <boost/regex.hpp>
#include <unordered_map>
#include <utility>
@@ -77,25 +76,65 @@ public:
return default_value;
}
- std::string get_string(const std::string& key) const {
+ std::string get_string(const std::string& key, const std::string& default_value = "") const {
auto it = _args.find(key);
if (it != _args.end()) {
return it->second;
}
- return "";
+ return default_value;
}
std::vector<std::string> get_entry_list(const std::string& key) const {
+ static const boost::regex sep(R"((?<=\])\s*,\s*(?=\[))");
std::vector<std::string> lists;
auto it = _args.find(key);
if (it != _args.end()) {
- static std::regex pattern(R"(\[([^\]]+)\])");
- std::smatch match;
- std::sregex_iterator iter(it->second.begin(), it->second.end(), pattern);
- std::sregex_iterator end;
- for (; iter != end; ++iter) {
- if (iter->size() > 1) {
- lists.emplace_back((*iter)[1].str());
+ std::string trimmed_input = boost::algorithm::trim_copy(it->second);
+ if (trimmed_input.empty()) {
+ return lists;
+ }
+
+ auto validate_single = [&](const std::string& item, const std::string& prefix) {
+ if (item.size() < 2 || item.front() != '[' || item.back() != ']') {
+ throw Exception(ErrorCode::INVALID_ARGUMENT,
+ prefix + key + " must be enclosed in []");
+ }
+ int depth = 0;
+ for (size_t i = 0; i + 1 < item.size(); ++i) {
+ char c = item[i];
+ if (c == '[') {
+ ++depth;
+ } else if (c == ']') {
+ --depth;
+ if (depth == 0) {
+ throw Exception(ErrorCode::INVALID_ARGUMENT,
+ prefix + key + " must be enclosed in []");
+ }
+ }
+ }
+ };
+
+ if (boost::regex_search(trimmed_input, sep)) {
+ boost::sregex_token_iterator regex_it(trimmed_input.begin(), trimmed_input.end(),
+ sep, -1);
+ boost::sregex_token_iterator end;
+ for (; regex_it != end; ++regex_it) {
+ std::string item = boost::algorithm::trim_copy(regex_it->str());
+ validate_single(item, "Each item in ");
+ std::string content = item.substr(1, item.size() - 2);
+ if (!content.empty()) {
+ lists.emplace_back(content);
+ }
+ }
+ } else {
+ if (trimmed_input.size() < 2 || trimmed_input.front() != '[' ||
+ trimmed_input.back() != ']') {
+ throw Exception(ErrorCode::INVALID_ARGUMENT,
+ "Item in " + key + " must be enclosed in []");
+ }
+ std::string content = trimmed_input.substr(1, trimmed_input.size() - 2);
+ if (!content.empty()) {
+ lists.emplace_back(content);
}
}
}
diff --git a/be/src/olap/rowset/segment_v2/inverted_index/tokenizer/tokenizer.h b/be/src/olap/rowset/segment_v2/inverted_index/token_filter/empty_token_filter_factory.h
similarity index 51%
copy from be/src/olap/rowset/segment_v2/inverted_index/tokenizer/tokenizer.h
copy to be/src/olap/rowset/segment_v2/inverted_index/token_filter/empty_token_filter_factory.h
index 6a7119b5fc5..b2776adf996 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index/tokenizer/tokenizer.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index/token_filter/empty_token_filter_factory.h
@@ -17,36 +17,35 @@
#pragma once
-#include <unicode/utf8.h>
+#include "token_filter.h"
+#include "token_filter_factory.h"
-#include <string_view>
+namespace doris::segment_v2::inverted_index {
-#include "olap/rowset/segment_v2/inverted_index/token_stream.h"
+class EmptyTokenFilter : public DorisTokenFilter {
+public:
+ EmptyTokenFilter(const TokenStreamPtr& in) : DorisTokenFilter(in) {}
+ ~EmptyTokenFilter() override = default;
-using TokenStreamPtr = std::shared_ptr<TokenStream>;
+ void initialize() {}
-namespace doris::segment_v2::inverted_index {
+ Token* next(Token* t) override { return _in->next(t); }
-class DorisTokenizer : public Tokenizer, public DorisTokenStream {
+ void reset() override { DorisTokenFilter::reset(); }
+};
+
+class EmptyTokenFilterFactory : public TokenFilterFactory {
public:
- DorisTokenizer() = default;
- ~DorisTokenizer() override = default;
-
- void set_reader(CL_NS(util)::Reader* in) {
- if (in == nullptr) {
- throw Exception(ErrorCode::INVALID_ARGUMENT, "reader must not be null");
- }
- _in_pending = in;
- }
+ EmptyTokenFilterFactory() = default;
+ ~EmptyTokenFilterFactory() override = default;
- using Tokenizer::reset;
- // Only use the parameterless reset method
- void reset() override { _in = _in_pending; };
+ void initialize(const Settings& settings) override {}
-protected:
- CL_NS(util)::Reader* _in = nullptr;
- CL_NS(util)::Reader* _in_pending = nullptr;
+ TokenFilterPtr create(const TokenStreamPtr& in) override {
+ auto token_filter = std::make_shared<EmptyTokenFilter>(in);
+ token_filter->initialize();
+ return token_filter;
+ }
};
-using TokenizerPtr = std::shared_ptr<DorisTokenizer>;
} // namespace doris::segment_v2::inverted_index
\ No newline at end of file
diff --git a/be/src/olap/rowset/segment_v2/inverted_index/token_filter/token_filter.h b/be/src/olap/rowset/segment_v2/inverted_index/token_filter/token_filter.h
index 41631ae6e44..0271db5a392 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index/token_filter/token_filter.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index/token_filter/token_filter.h
@@ -19,8 +19,6 @@
#include "olap/rowset/segment_v2/inverted_index/token_stream.h"
-using TokenStreamPtr = std::shared_ptr<TokenStream>;
-
namespace doris::segment_v2::inverted_index {
class DorisTokenFilter : public TokenFilter, public DorisTokenStream {
diff --git a/be/src/olap/rowset/segment_v2/inverted_index/token_filter/word_delimiter_filter_factory.h b/be/src/olap/rowset/segment_v2/inverted_index/token_filter/word_delimiter_filter_factory.h
index a125150bcad..9e0e60b73a0 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index/token_filter/word_delimiter_filter_factory.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index/token_filter/word_delimiter_filter_factory.h
@@ -17,6 +17,8 @@
#pragma once
+#include <regex>
+
#include "olap/rowset/segment_v2/inverted_index/setting.h"
#include "token_filter_factory.h"
#include "word_delimiter_filter.h"
diff --git a/be/src/olap/rowset/segment_v2/inverted_index/token_stream.h b/be/src/olap/rowset/segment_v2/inverted_index/token_stream.h
index 6435a4cbfbd..b0d055a69aa 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index/token_stream.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index/token_stream.h
@@ -21,17 +21,21 @@
#include <memory>
#include <string_view>
-#include <unordered_set>
#include "CLucene.h"
#include "CLucene/analysis/AnalysisHeader.h"
-#include "common/exception.h"
-#include "common/logging.h"
+#include "common/cast_set.h"
+#include "olap/rowset/segment_v2/inverted_index/util/reader.h"
using namespace lucene::analysis;
namespace doris::segment_v2::inverted_index {
+class DorisTokenizer;
+using TokenizerPtr = std::shared_ptr<DorisTokenizer>;
+
+using TokenStreamPtr = std::shared_ptr<TokenStream>;
+
/**
* All custom tokenizers and token_filters must use the following functions
* to set token information. Using these unified set methods helps avoid
@@ -57,4 +61,32 @@ public:
void set_position_increment(Token* t, int32_t pos) { t->setPositionIncrement(pos); }
};
-}; // namespace doris::segment_v2::inverted_index
\ No newline at end of file
+class TokenStreamWrapper : public TokenStream {
+public:
+ explicit TokenStreamWrapper(std::shared_ptr<TokenStream> ts) : _impl(std::move(ts)) {}
+ ~TokenStreamWrapper() override = default;
+
+ Token* next(Token* token) override { return _impl->next(token); }
+ void close() override { _impl->close(); }
+ void reset() override { _impl->reset(); }
+
+private:
+ std::shared_ptr<TokenStream> _impl;
+};
+
+class TokenStreamComponents {
+public:
+ TokenStreamComponents(TokenizerPtr tokenizer, TokenStreamPtr result)
+ : _source(std::move(tokenizer)), _sink(std::move(result)) {}
+
+ void set_reader(const ReaderPtr& reader);
+ TokenStreamPtr get_token_stream();
+ TokenizerPtr get_source();
+
+private:
+ TokenizerPtr _source;
+ TokenStreamPtr _sink;
+};
+using TokenStreamComponentsPtr = std::shared_ptr<TokenStreamComponents>;
+
+}; // namespace doris::segment_v2::inverted_index
diff --git a/be/src/olap/rowset/segment_v2/inverted_index/analyzer/basic/basic_tokenizer.cpp b/be/src/olap/rowset/segment_v2/inverted_index/tokenizer/basic/basic_tokenizer.cpp
similarity index 75%
rename from be/src/olap/rowset/segment_v2/inverted_index/analyzer/basic/basic_tokenizer.cpp
rename to be/src/olap/rowset/segment_v2/inverted_index/tokenizer/basic/basic_tokenizer.cpp
index 0679fdbdd26..2697b4d433b 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index/analyzer/basic/basic_tokenizer.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index/tokenizer/basic/basic_tokenizer.cpp
@@ -19,7 +19,7 @@
#include <unicode/unistr.h>
-namespace doris::segment_v2 {
+namespace doris::segment_v2::inverted_index {
#include "common/compile_check_begin.h"
#define IS_IN_RANGE(c, start, end) ((uint32_t)((c) - (start)) <= ((end) - (start)))
@@ -29,14 +29,19 @@ namespace doris::segment_v2 {
IS_IN_RANGE(c, 0x20000, 0x2A6DF) || IS_IN_RANGE(c, 0x2A700, 0x2EBEF) || \
IS_IN_RANGE(c, 0x30000, 0x3134A))
-BasicTokenizer::BasicTokenizer() {
- this->lowercase = false;
- this->ownReader = false;
+BasicTokenizer::BasicTokenizer(bool own_reader) {
+ this->ownReader = own_reader;
}
-BasicTokenizer::BasicTokenizer(bool lower_case, bool own_reader) : BasicTokenizer() {
- this->lowercase = lower_case;
- this->ownReader = own_reader;
+void BasicTokenizer::initialize(const std::string& extra_chars) {
+ _extra_char_set.fill(false);
+ _has_extra_chars = !extra_chars.empty();
+
+ for (uint8_t c : extra_chars) {
+ if (c < 128) {
+ _extra_char_set[c] = true;
+ }
+ }
}
Token* BasicTokenizer::next(Token* token) {
@@ -50,21 +55,28 @@ Token* BasicTokenizer::next(Token* token) {
return token;
}
-void BasicTokenizer::reset(lucene::util::Reader* reader) {
+void BasicTokenizer::reset() {
+ DorisTokenizer::reset();
+
_buffer_index = 0;
_data_len = 0;
_tokens_text.clear();
- _buffer.resize(reader->size());
- size_t numRead = reader->readCopy(_buffer.data(), 0, static_cast<int32_t>(_buffer.size()));
+ _buffer.resize(_in->size());
+ size_t numRead = _in->readCopy(_buffer.data(), 0, static_cast<int32_t>(_buffer.size()));
(void)numRead;
assert(_buffer.size() == numRead);
- cut();
+ if (_has_extra_chars) {
+ cut<true>();
+ } else {
+ cut<false>();
+ }
_data_len = static_cast<int32_t>(_tokens_text.size());
}
+template <bool HasExtraChars>
void BasicTokenizer::cut() {
auto* s = (uint8_t*)_buffer.data();
auto length = static_cast<int32_t>(_buffer.size());
@@ -101,9 +113,16 @@ void BasicTokenizer::cut() {
const int32_t len = i - prev_i;
_tokens_text.emplace_back(reinterpret_cast<const char*>(s + prev_i), len);
}
+
+ if constexpr (HasExtraChars) {
+ if (c < 128 && _extra_char_set[c]) {
+ const int32_t len = i - prev_i;
+ _tokens_text.emplace_back(reinterpret_cast<const char*>(s + prev_i), len);
+ }
+ }
}
}
}
#include "common/compile_check_end.h"
-} // namespace doris::segment_v2
\ No newline at end of file
+} // namespace doris::segment_v2::inverted_index
\ No newline at end of file
diff --git a/be/src/olap/rowset/segment_v2/inverted_index/analyzer/basic/basic_tokenizer.h b/be/src/olap/rowset/segment_v2/inverted_index/tokenizer/basic/basic_tokenizer.h
similarity index 69%
rename from be/src/olap/rowset/segment_v2/inverted_index/analyzer/basic/basic_tokenizer.h
rename to be/src/olap/rowset/segment_v2/inverted_index/tokenizer/basic/basic_tokenizer.h
index 211301351d8..27f2ec7ca7f 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index/analyzer/basic/basic_tokenizer.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index/tokenizer/basic/basic_tokenizer.h
@@ -17,32 +17,34 @@
#pragma once
-#include <unicode/utext.h>
-
-#include "CLucene.h"
-#include "CLucene/analysis/AnalysisHeader.h"
-#include "olap/rowset/segment_v2/inverted_index/analyzer/icu/icu_common.h"
+#include "olap/rowset/segment_v2/inverted_index/tokenizer/tokenizer.h"
using namespace lucene::analysis;
-namespace doris::segment_v2 {
+namespace doris::segment_v2::inverted_index {
-class BasicTokenizer : public Tokenizer {
+class BasicTokenizer : public DorisTokenizer {
public:
- BasicTokenizer();
- BasicTokenizer(bool lowercase, bool ownReader);
+ BasicTokenizer() = default;
+ BasicTokenizer(bool own_reader);
~BasicTokenizer() override = default;
+ void initialize(const std::string& extra_chars = "");
+
Token* next(Token* token) override;
- void reset(lucene::util::Reader* reader) override;
+ void reset() override;
+private:
+ template <bool HasExtraChars>
void cut();
-private:
int32_t _buffer_index = 0;
int32_t _data_len = 0;
std::string _buffer;
std::vector<std::string_view> _tokens_text;
+
+ std::array<bool, 128> _extra_char_set {};
+ bool _has_extra_chars = false;
};
-} // namespace doris::segment_v2
\ No newline at end of file
+} // namespace doris::segment_v2::inverted_index
\ No newline at end of file
diff --git a/be/src/olap/rowset/segment_v2/inverted_index/tokenizer/basic/basic_tokenizer_factory.h b/be/src/olap/rowset/segment_v2/inverted_index/tokenizer/basic/basic_tokenizer_factory.h
new file mode 100644
index 00000000000..fe148b7e89f
--- /dev/null
+++ b/be/src/olap/rowset/segment_v2/inverted_index/tokenizer/basic/basic_tokenizer_factory.h
@@ -0,0 +1,50 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include "basic_tokenizer.h"
+#include "olap/rowset/segment_v2/inverted_index/tokenizer/tokenizer_factory.h"
+
+namespace doris::segment_v2::inverted_index {
+class BasicTokenizerFactory : public TokenizerFactory {
+public:
+ BasicTokenizerFactory() = default;
+ ~BasicTokenizerFactory() override = default;
+
+ void initialize(const Settings& settings) override {
+ _extra_chars = settings.get_string("extra_chars", "");
+ for (uint8_t c : _extra_chars) {
+ if (c >= 128) {
+ throw Exception(
+ ErrorCode::INVALID_ARGUMENT,
+ "Invalid extra_chars for basic tokenizer: contains non-ASCII character");
+ }
+ }
+ }
+
+ TokenizerPtr create() override {
+ auto tokenzier = std::make_shared<BasicTokenizer>();
+ tokenzier->initialize(_extra_chars);
+ return tokenzier;
+ }
+
+private:
+ std::string _extra_chars;
+};
+
+} // namespace doris::segment_v2::inverted_index
\ No newline at end of file
diff --git a/be/src/olap/rowset/segment_v2/inverted_index/tokenizer/empty/empty_tokenizer_factory.h b/be/src/olap/rowset/segment_v2/inverted_index/tokenizer/empty/empty_tokenizer_factory.h
new file mode 100644
index 00000000000..37e51c4b5f4
--- /dev/null
+++ b/be/src/olap/rowset/segment_v2/inverted_index/tokenizer/empty/empty_tokenizer_factory.h
@@ -0,0 +1,72 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include "olap/rowset/segment_v2/inverted_index/tokenizer/tokenizer.h"
+#include "olap/rowset/segment_v2/inverted_index/tokenizer/tokenizer_factory.h"
+
+namespace doris::segment_v2::inverted_index {
+
+class EmptyTokenizer : public DorisTokenizer {
+public:
+ EmptyTokenizer() = default;
+ ~EmptyTokenizer() override = default;
+
+ void initialize() {}
+
+ Token* next(Token* token) override {
+ if (!_done) {
+ _done = true;
+ if (_char_buffer == nullptr) {
+ return nullptr;
+ }
+ std::string_view term(_char_buffer, _char_length);
+ set(token, term);
+ return token;
+ }
+ return nullptr;
+ }
+
+ void reset() override {
+ DorisTokenizer::reset();
+ _done = false;
+ _char_buffer = nullptr;
+ _char_length = _in->read((const void**)&_char_buffer, 0, static_cast<int32_t>(_in->size()));
+ }
+
+private:
+ bool _done = false;
+ const char* _char_buffer = nullptr;
+ int32_t _char_length = 0;
+};
+
+class EmptyTokenizerFactory : public TokenizerFactory {
+public:
+ EmptyTokenizerFactory() = default;
+ ~EmptyTokenizerFactory() override = default;
+
+ void initialize(const Settings& settings) override {}
+
+ TokenizerPtr create() override {
+ auto tokenizer = std::make_shared<EmptyTokenizer>();
+ tokenizer->initialize();
+ return tokenizer;
+ }
+};
+
+} // namespace doris::segment_v2::inverted_index
\ No newline at end of file
diff --git a/be/src/olap/rowset/segment_v2/inverted_index/analyzer/icu/break_iterator_wrapper.cpp b/be/src/olap/rowset/segment_v2/inverted_index/tokenizer/icu/break_iterator_wrapper.cpp
similarity index 97%
rename from be/src/olap/rowset/segment_v2/inverted_index/analyzer/icu/break_iterator_wrapper.cpp
rename to be/src/olap/rowset/segment_v2/inverted_index/tokenizer/icu/break_iterator_wrapper.cpp
index 50094e54f7b..0ed63c28a15 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index/analyzer/icu/break_iterator_wrapper.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index/tokenizer/icu/break_iterator_wrapper.cpp
@@ -25,7 +25,7 @@
#include "icu_common.h"
#include "icu_tokenizer_config.h"
-namespace doris::segment_v2 {
+namespace doris::segment_v2::inverted_index {
icu::UnicodeSet BreakIteratorWrapper::EMOJI_RK;
icu::UnicodeSet BreakIteratorWrapper::EMOJI;
@@ -104,4 +104,4 @@ void BreakIteratorWrapper::set_text(const UChar* text, int32_t start, int32_t le
status_ = UBRK_WORD_NONE;
}
-} // namespace doris::segment_v2
\ No newline at end of file
+} // namespace doris::segment_v2::inverted_index
\ No newline at end of file
diff --git a/be/src/olap/rowset/segment_v2/inverted_index/analyzer/icu/break_iterator_wrapper.h b/be/src/olap/rowset/segment_v2/inverted_index/tokenizer/icu/break_iterator_wrapper.h
similarity index 94%
rename from be/src/olap/rowset/segment_v2/inverted_index/analyzer/icu/break_iterator_wrapper.h
rename to be/src/olap/rowset/segment_v2/inverted_index/tokenizer/icu/break_iterator_wrapper.h
index dea60d1d1f7..554b02c9eaf 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index/analyzer/icu/break_iterator_wrapper.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index/tokenizer/icu/break_iterator_wrapper.h
@@ -25,7 +25,7 @@
#include "icu_common.h"
-namespace doris::segment_v2 {
+namespace doris::segment_v2::inverted_index {
class BreakIteratorWrapper {
public:
@@ -51,4 +51,4 @@ private:
};
using BreakIteratorWrapperPtr = std::unique_ptr<BreakIteratorWrapper>;
-} // namespace doris::segment_v2
\ No newline at end of file
+} // namespace doris::segment_v2::inverted_index
\ No newline at end of file
diff --git a/be/src/olap/rowset/segment_v2/inverted_index/analyzer/icu/composite_break_iterator.cpp b/be/src/olap/rowset/segment_v2/inverted_index/tokenizer/icu/composite_break_iterator.cpp
similarity index 97%
rename from be/src/olap/rowset/segment_v2/inverted_index/analyzer/icu/composite_break_iterator.cpp
rename to be/src/olap/rowset/segment_v2/inverted_index/tokenizer/icu/composite_break_iterator.cpp
index e178ad35c13..5a4d56d11e6 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index/analyzer/icu/composite_break_iterator.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index/tokenizer/icu/composite_break_iterator.cpp
@@ -21,7 +21,7 @@
#include <memory>
-namespace doris::segment_v2 {
+namespace doris::segment_v2::inverted_index {
CompositeBreakIterator::CompositeBreakIterator(const ICUTokenizerConfigPtr& config)
: config_(config) {
@@ -80,4 +80,4 @@ BreakIteratorWrapper* CompositeBreakIterator::get_break_iterator(int32_t scriptC
return word_breakers_[scriptCode].get();
}
-} // namespace doris::segment_v2
\ No newline at end of file
+} // namespace doris::segment_v2::inverted_index
\ No newline at end of file
diff --git a/be/src/olap/rowset/segment_v2/inverted_index/analyzer/icu/composite_break_iterator.h b/be/src/olap/rowset/segment_v2/inverted_index/tokenizer/icu/composite_break_iterator.h
similarity index 94%
rename from be/src/olap/rowset/segment_v2/inverted_index/analyzer/icu/composite_break_iterator.h
rename to be/src/olap/rowset/segment_v2/inverted_index/tokenizer/icu/composite_break_iterator.h
index 8599be88dc2..4d5e2a6b4d6 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index/analyzer/icu/composite_break_iterator.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index/tokenizer/icu/composite_break_iterator.h
@@ -29,7 +29,7 @@
#include "icu_tokenizer_config.h"
#include "script_iterator.h"
-namespace doris::segment_v2 {
+namespace doris::segment_v2::inverted_index {
class CompositeBreakIterator {
public:
@@ -55,4 +55,4 @@ private:
};
using CompositeBreakIteratorPtr = std::unique_ptr<CompositeBreakIterator>;
-} // namespace doris::segment_v2
\ No newline at end of file
+} // namespace doris::segment_v2::inverted_index
\ No newline at end of file
diff --git a/be/src/olap/rowset/segment_v2/inverted_index/analyzer/icu/default_icu_tokenizer_config.cpp b/be/src/olap/rowset/segment_v2/inverted_index/tokenizer/icu/default_icu_tokenizer_config.cpp
similarity index 98%
rename from be/src/olap/rowset/segment_v2/inverted_index/analyzer/icu/default_icu_tokenizer_config.cpp
rename to be/src/olap/rowset/segment_v2/inverted_index/tokenizer/icu/default_icu_tokenizer_config.cpp
index dfbcf2dcdf6..4d9d8d9b2a7 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index/analyzer/icu/default_icu_tokenizer_config.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index/tokenizer/icu/default_icu_tokenizer_config.cpp
@@ -24,7 +24,7 @@
#include <sstream>
#include <string>
-namespace doris::segment_v2 {
+namespace doris::segment_v2::inverted_index {
BreakIteratorPtr DefaultICUTokenizerConfig::cjk_break_iterator_;
BreakIteratorPtr DefaultICUTokenizerConfig::default_break_iterator_;
@@ -125,4 +125,4 @@ void DefaultICUTokenizerConfig::read_break_iterator(BreakIteratorPtr& rbbi,
}
}
-} // namespace doris::segment_v2
\ No newline at end of file
+} // namespace doris::segment_v2::inverted_index
\ No newline at end of file
diff --git a/be/src/olap/rowset/segment_v2/inverted_index/analyzer/icu/default_icu_tokenizer_config.h b/be/src/olap/rowset/segment_v2/inverted_index/tokenizer/icu/default_icu_tokenizer_config.h
similarity index 94%
rename from be/src/olap/rowset/segment_v2/inverted_index/analyzer/icu/default_icu_tokenizer_config.h
rename to be/src/olap/rowset/segment_v2/inverted_index/tokenizer/icu/default_icu_tokenizer_config.h
index 6500cf230eb..21e9359e5a3 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index/analyzer/icu/default_icu_tokenizer_config.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index/tokenizer/icu/default_icu_tokenizer_config.h
@@ -19,7 +19,7 @@
#include "icu_tokenizer_config.h"
-namespace doris::segment_v2 {
+namespace doris::segment_v2::inverted_index {
class DefaultICUTokenizerConfig : public ICUTokenizerConfig {
public:
@@ -41,4 +41,4 @@ private:
bool myanmar_as_words_ = false;
};
-} // namespace doris::segment_v2
\ No newline at end of file
+} // namespace doris::segment_v2::inverted_index
\ No newline at end of file
diff --git a/be/src/olap/rowset/segment_v2/inverted_index/analyzer/icu/icu_common.h b/be/src/olap/rowset/segment_v2/inverted_index/tokenizer/icu/icu_common.h
similarity index 93%
rename from be/src/olap/rowset/segment_v2/inverted_index/analyzer/icu/icu_common.h
rename to be/src/olap/rowset/segment_v2/inverted_index/tokenizer/icu/icu_common.h
index 1cdffab48d3..0066e26ec9b 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index/analyzer/icu/icu_common.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index/tokenizer/icu/icu_common.h
@@ -31,7 +31,7 @@
#include "unicode/utext.h"
#include "unicode/utf8.h"
-namespace doris::segment_v2 {
+namespace doris::segment_v2::inverted_index {
using BreakIteratorPtr = std::unique_ptr<icu::BreakIterator>;
@@ -45,4 +45,4 @@ struct UTextDeleter {
using UTextPtr = std::unique_ptr<UText, UTextDeleter>;
-} // namespace doris::segment_v2
\ No newline at end of file
+} // namespace doris::segment_v2::inverted_index
\ No newline at end of file
diff --git a/be/src/olap/rowset/segment_v2/inverted_index/analyzer/icu/icu_tokenizer.cpp b/be/src/olap/rowset/segment_v2/inverted_index/tokenizer/icu/icu_tokenizer.cpp
similarity index 86%
rename from be/src/olap/rowset/segment_v2/inverted_index/analyzer/icu/icu_tokenizer.cpp
rename to be/src/olap/rowset/segment_v2/inverted_index/tokenizer/icu/icu_tokenizer.cpp
index e10b197d517..1aa1031093e 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index/analyzer/icu/icu_tokenizer.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index/tokenizer/icu/icu_tokenizer.cpp
@@ -22,18 +22,14 @@
#include <memory>
#include <string>
-namespace doris::segment_v2 {
+namespace doris::segment_v2::inverted_index {
ICUTokenizer::ICUTokenizer() {
- this->lowercase = false;
- this->ownReader = false;
-
config_ = std::make_shared<DefaultICUTokenizerConfig>(true, true);
breaker_ = std::make_unique<CompositeBreakIterator>(config_);
}
-ICUTokenizer::ICUTokenizer(bool lower_case, bool own_reader) : ICUTokenizer() {
- this->lowercase = lower_case;
+ICUTokenizer::ICUTokenizer(bool own_reader) : ICUTokenizer() {
this->ownReader = own_reader;
}
@@ -69,9 +65,10 @@ Token* ICUTokenizer::next(Token* token) {
return token;
}
-void ICUTokenizer::reset(lucene::util::Reader* reader) {
+void ICUTokenizer::reset() {
+ DorisTokenizer::reset();
const char* buf = nullptr;
- int32_t len = reader->read((const void**)&buf, 0, reader->size());
+ int32_t len = _in->read((const void**)&buf, 0, static_cast<int32_t>(_in->size()));
buffer_ = icu::UnicodeString::fromUTF8(icu::StringPiece(buf, len));
if (!buffer_.isEmpty() && buffer_.isBogus()) {
_CLTHROWT(CL_ERR_Runtime, "Failed to convert UTF-8 string to UnicodeString.");
@@ -79,4 +76,4 @@ void ICUTokenizer::reset(lucene::util::Reader* reader) {
breaker_->set_text(buffer_.getBuffer(), 0, buffer_.length());
}
-} // namespace doris::segment_v2
\ No newline at end of file
+} // namespace doris::segment_v2::inverted_index
diff --git a/be/src/olap/rowset/segment_v2/inverted_index/analyzer/icu/icu_tokenizer.h b/be/src/olap/rowset/segment_v2/inverted_index/tokenizer/icu/icu_tokenizer.h
similarity index 82%
rename from be/src/olap/rowset/segment_v2/inverted_index/analyzer/icu/icu_tokenizer.h
rename to be/src/olap/rowset/segment_v2/inverted_index/tokenizer/icu/icu_tokenizer.h
index d11d0c67ed6..bad250ea6e8 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index/analyzer/icu/icu_tokenizer.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index/tokenizer/icu/icu_tokenizer.h
@@ -19,25 +19,24 @@
#include <unicode/utext.h>
-#include "CLucene.h"
-#include "CLucene/analysis/AnalysisHeader.h"
#include "composite_break_iterator.h"
#include "default_icu_tokenizer_config.h"
#include "icu_common.h"
+#include "olap/rowset/segment_v2/inverted_index/tokenizer/tokenizer.h"
using namespace lucene::analysis;
-namespace doris::segment_v2 {
+namespace doris::segment_v2::inverted_index {
-class ICUTokenizer : public Tokenizer {
+class ICUTokenizer : public DorisTokenizer {
public:
ICUTokenizer();
- ICUTokenizer(bool lowercase, bool ownReader);
+ ICUTokenizer(bool ownReader);
~ICUTokenizer() override = default;
void initialize(const std::string& dictPath);
Token* next(Token* token) override;
- void reset(lucene::util::Reader* reader) override;
+ void reset() override;
private:
std::string utf8Str_;
@@ -47,4 +46,4 @@ private:
CompositeBreakIteratorPtr breaker_;
};
-} // namespace doris::segment_v2
\ No newline at end of file
+} // namespace doris::segment_v2::inverted_index
\ No newline at end of file
diff --git a/be/src/olap/rowset/segment_v2/inverted_index/analyzer/icu/icu_tokenizer_config.h b/be/src/olap/rowset/segment_v2/inverted_index/tokenizer/icu/icu_tokenizer_config.h
similarity index 93%
rename from be/src/olap/rowset/segment_v2/inverted_index/analyzer/icu/icu_tokenizer_config.h
rename to be/src/olap/rowset/segment_v2/inverted_index/tokenizer/icu/icu_tokenizer_config.h
index dd7b743e74b..af3a1f3bee3 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index/analyzer/icu/icu_tokenizer_config.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index/tokenizer/icu/icu_tokenizer_config.h
@@ -19,7 +19,7 @@
#include "icu_common.h"
-namespace doris::segment_v2 {
+namespace doris::segment_v2::inverted_index {
class ICUTokenizerConfig {
public:
@@ -34,4 +34,4 @@ public:
};
using ICUTokenizerConfigPtr = std::shared_ptr<ICUTokenizerConfig>;
-} // namespace doris::segment_v2
\ No newline at end of file
+} // namespace doris::segment_v2::inverted_index
\ No newline at end of file
diff --git a/be/src/olap/rowset/segment_v2/inverted_index/token_filter/token_filter.h b/be/src/olap/rowset/segment_v2/inverted_index/tokenizer/icu/icu_tokenizer_factory.h
similarity index 61%
copy from be/src/olap/rowset/segment_v2/inverted_index/token_filter/token_filter.h
copy to be/src/olap/rowset/segment_v2/inverted_index/tokenizer/icu/icu_tokenizer_factory.h
index 41631ae6e44..f750d652ad7 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index/token_filter/token_filter.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index/tokenizer/icu/icu_tokenizer_factory.h
@@ -17,22 +17,23 @@
#pragma once
-#include "olap/rowset/segment_v2/inverted_index/token_stream.h"
-
-using TokenStreamPtr = std::shared_ptr<TokenStream>;
+#include "icu_tokenizer.h"
+#include "olap/rowset/segment_v2/inverted_index/tokenizer/tokenizer_factory.h"
namespace doris::segment_v2::inverted_index {
-class DorisTokenFilter : public TokenFilter, public DorisTokenStream {
+class ICUTokenizerFactory : public TokenizerFactory {
public:
- DorisTokenFilter(TokenStreamPtr in) : TokenFilter(nullptr), _in(std::move(in)) {}
- ~DorisTokenFilter() override = default;
+ ICUTokenizerFactory() = default;
+ ~ICUTokenizerFactory() override = default;
- void reset() override { _in->reset(); }
+ void initialize(const Settings& settings) override {}
-protected:
- TokenStreamPtr _in;
+ TokenizerPtr create() override {
+ auto tokenizer = std::make_shared<ICUTokenizer>();
+ tokenizer->initialize(config::inverted_index_dict_path + "/icu");
+ return tokenizer;
+ }
};
-using TokenFilterPtr = std::shared_ptr<DorisTokenFilter>;
} // namespace doris::segment_v2::inverted_index
\ No newline at end of file
diff --git a/be/src/olap/rowset/segment_v2/inverted_index/analyzer/icu/script_iterator.cpp b/be/src/olap/rowset/segment_v2/inverted_index/tokenizer/icu/script_iterator.cpp
similarity index 97%
rename from be/src/olap/rowset/segment_v2/inverted_index/analyzer/icu/script_iterator.cpp
rename to be/src/olap/rowset/segment_v2/inverted_index/tokenizer/icu/script_iterator.cpp
index 7fee3055d3b..c742991d1f0 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index/analyzer/icu/script_iterator.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index/tokenizer/icu/script_iterator.cpp
@@ -22,7 +22,7 @@
#include <mutex>
#include <string>
-namespace doris::segment_v2 {
+namespace doris::segment_v2::inverted_index {
std::vector<int32_t> ScriptIterator::k_basic_latin(128);
@@ -118,4 +118,4 @@ bool ScriptIterator::is_combining_mark(UChar32 codepoint) {
type == U_ENCLOSING_MARK);
}
-} // namespace doris::segment_v2
\ No newline at end of file
+} // namespace doris::segment_v2::inverted_index
\ No newline at end of file
diff --git a/be/src/olap/rowset/segment_v2/inverted_index/analyzer/icu/script_iterator.h b/be/src/olap/rowset/segment_v2/inverted_index/tokenizer/icu/script_iterator.h
similarity index 95%
rename from be/src/olap/rowset/segment_v2/inverted_index/analyzer/icu/script_iterator.h
rename to be/src/olap/rowset/segment_v2/inverted_index/tokenizer/icu/script_iterator.h
index bc93eea8670..3db78d25c71 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index/analyzer/icu/script_iterator.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index/tokenizer/icu/script_iterator.h
@@ -25,7 +25,7 @@
#include "icu_common.h"
-namespace doris::segment_v2 {
+namespace doris::segment_v2::inverted_index {
class ScriptIterator {
public:
@@ -61,4 +61,4 @@ private:
};
using ScriptIteratorPtr = std::unique_ptr<ScriptIterator>;
-} // namespace doris::segment_v2
\ No newline at end of file
+} // namespace doris::segment_v2::inverted_index
\ No newline at end of file
diff --git a/be/src/olap/rowset/segment_v2/inverted_index/tokenizer/standard/standard_tokenizer_impl.h b/be/src/olap/rowset/segment_v2/inverted_index/tokenizer/standard/standard_tokenizer_impl.h
index b707e9add33..2b27f6b4756 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index/tokenizer/standard/standard_tokenizer_impl.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index/tokenizer/standard/standard_tokenizer_impl.h
@@ -24,8 +24,7 @@
#include <string_view>
#include <vector>
-#include "CLucene.h"
-#include "CLucene/analysis/AnalysisHeader.h"
+#include "olap/rowset/segment_v2/inverted_index/util/reader.h"
namespace doris::segment_v2::inverted_index {
#include "common/compile_check_begin.h"
@@ -168,7 +167,7 @@ public:
return {_zz_buffer.data() + _zz_start_read, (size_t)(_zz_marked_pos - _zz_start_read)};
}
- inline void yyreset(CL_NS(util)::Reader* reader) {
+ inline void yyreset(const ReaderPtr& reader) {
_zz_reader = reader;
_zz_at_eof = false;
_zz_current_pos = 0;
@@ -283,7 +282,7 @@ private:
static const int32_t ZZ_BUFFERSIZE = 255;
- CL_NS(util)::Reader* _zz_reader = nullptr;
+ ReaderPtr _zz_reader;
std::string _zz_buffer;
int32_t _zz_state = 0;
diff --git a/be/src/olap/rowset/segment_v2/inverted_index/tokenizer/tokenizer.h b/be/src/olap/rowset/segment_v2/inverted_index/tokenizer/tokenizer.h
index 6a7119b5fc5..8b7898f8337 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index/tokenizer/tokenizer.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index/tokenizer/tokenizer.h
@@ -22,8 +22,7 @@
#include <string_view>
#include "olap/rowset/segment_v2/inverted_index/token_stream.h"
-
-using TokenStreamPtr = std::shared_ptr<TokenStream>;
+#include "olap/rowset/segment_v2/inverted_index/util/reader.h"
namespace doris::segment_v2::inverted_index {
@@ -32,7 +31,7 @@ public:
DorisTokenizer() = default;
~DorisTokenizer() override = default;
- void set_reader(CL_NS(util)::Reader* in) {
+ void set_reader(const ReaderPtr& in) {
if (in == nullptr) {
throw Exception(ErrorCode::INVALID_ARGUMENT, "reader must not be null");
}
@@ -44,8 +43,8 @@ public:
void reset() override { _in = _in_pending; };
protected:
- CL_NS(util)::Reader* _in = nullptr;
- CL_NS(util)::Reader* _in_pending = nullptr;
+ ReaderPtr _in;
+ ReaderPtr _in_pending;
};
using TokenizerPtr = std::shared_ptr<DorisTokenizer>;
diff --git a/be/src/olap/rowset/segment_v2/inverted_index/token_filter/token_filter.h b/be/src/olap/rowset/segment_v2/inverted_index/util/reader.h
similarity index 62%
copy from be/src/olap/rowset/segment_v2/inverted_index/token_filter/token_filter.h
copy to be/src/olap/rowset/segment_v2/inverted_index/util/reader.h
index 41631ae6e44..27428f491b4 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index/token_filter/token_filter.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index/util/reader.h
@@ -17,22 +17,13 @@
#pragma once
-#include "olap/rowset/segment_v2/inverted_index/token_stream.h"
+#include <memory>
-using TokenStreamPtr = std::shared_ptr<TokenStream>;
+#include "CLucene.h"
+#include "CLucene/util/CLStreams.h"
namespace doris::segment_v2::inverted_index {
-class DorisTokenFilter : public TokenFilter, public DorisTokenStream {
-public:
- DorisTokenFilter(TokenStreamPtr in) : TokenFilter(nullptr), _in(std::move(in)) {}
- ~DorisTokenFilter() override = default;
-
- void reset() override { _in->reset(); }
-
-protected:
- TokenStreamPtr _in;
-};
-using TokenFilterPtr = std::shared_ptr<DorisTokenFilter>;
+using ReaderPtr = std::shared_ptr<lucene::util::Reader>;
} // namespace doris::segment_v2::inverted_index
\ No newline at end of file
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp b/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp
index 2a66d6419d3..f13b9af3065 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp
@@ -52,12 +52,12 @@
#include "olap/rowset/segment_v2/common.h"
#include "olap/rowset/segment_v2/inverted_index/analyzer/analyzer.h"
#include "olap/rowset/segment_v2/inverted_index/char_filter/char_filter_factory.h"
+#include "olap/rowset/segment_v2/inverted_index/util/reader.h"
#include "olap/rowset/segment_v2/inverted_index_common.h"
#include "olap/rowset/segment_v2/inverted_index_desc.h"
#include "olap/rowset/segment_v2/inverted_index_file_writer.h"
#include "olap/rowset/segment_v2/inverted_index_fs_directory.h"
#include "olap/tablet_schema.h"
-#include "olap/types.h"
#include "runtime/collection_value.h"
#include "runtime/exec_env.h"
#include "util/debug_points.h"
@@ -66,6 +66,9 @@
#include "util/string_util.h"
namespace doris::segment_v2 {
+
+using namespace doris::segment_v2::inverted_index;
+
const int32_t MAX_FIELD_LEN = 0x7FFFFFFFL;
const int32_t MERGE_FACTOR = 100000000;
const int32_t MAX_LEAF_COUNT = 1024;
@@ -166,8 +169,7 @@ public:
return open_index_directory();
}
- Result<std::unique_ptr<lucene::util::Reader>> create_char_string_reader(
- CharFilterMap& char_filter_map) {
+ Result<ReaderPtr> create_char_string_reader(CharFilterMap& char_filter_map) {
try {
return inverted_index::InvertedIndexAnalyzer::create_reader(char_filter_map);
} catch (CLuceneError& e) {
@@ -379,7 +381,7 @@ public:
_CLTHROWA(CL_ERR_UnsupportedOperation,
"UnsupportedOperationException: CLStream::init");
})
- auto* stream = _analyzer->reusableTokenStream(field->name(), _char_string_reader.get());
+ auto* stream = _analyzer->reusableTokenStream(field->name(), _char_string_reader);
field->setValue(stream);
}
@@ -441,6 +443,7 @@ public:
return Status::InternalError("index writer is null in inverted index writer");
}
size_t start_off = 0;
+ std::vector<ReaderPtr> keep_readers;
for (int i = 0; i < count; ++i) {
// nullmap & value ptr-array may not from offsets[i] because olap_convertor make offsets accumulate from _base_offset which may not is 0, but nullmap & value in this segment is from 0, we only need
// every single array row element size to go through the nullmap & value ptr-array, and also can go through the every row in array to keep with _rid++
@@ -480,15 +483,12 @@ public:
// in this case stream need to delete after add_document, because the
// stream can not reuse for different field
bool own_token_stream = true;
- bool own_reader = true;
- std::unique_ptr<lucene::util::Reader> char_string_reader =
- DORIS_TRY(create_char_string_reader(
- _inverted_index_ctx->char_filter_map));
+ ReaderPtr char_string_reader = DORIS_TRY(create_char_string_reader(
+ _inverted_index_ctx->char_filter_map));
char_string_reader->init(v->get_data(), v->get_size(), false);
- _analyzer->set_ownReader(own_reader);
- ts = _analyzer->tokenStream(new_field->name(),
- char_string_reader.release());
+ ts = _analyzer->tokenStream(new_field->name(), char_string_reader);
new_field->setValue(ts, own_token_stream);
+ keep_readers.emplace_back(std::move(char_string_reader));
} else {
new_field_char_value(v->get_data(), v->get_size(), new_field.get());
}
@@ -541,6 +541,7 @@ public:
_doc->clear();
}
_rid++;
+ keep_readers.clear();
}
} else if constexpr (field_is_numeric_type(field_type)) {
size_t start_off = 0;
@@ -751,7 +752,7 @@ private:
std::shared_ptr<DorisFSDirectory> _dir = nullptr;
std::unique_ptr<lucene::index::IndexWriter> _index_writer = nullptr;
std::shared_ptr<lucene::analysis::Analyzer> _analyzer = nullptr;
- std::unique_ptr<lucene::util::Reader> _char_string_reader = nullptr;
+ ReaderPtr _char_string_reader = nullptr;
std::shared_ptr<lucene::util::bkd::bkd_writer> _bkd_writer = nullptr;
InvertedIndexCtxSPtr _inverted_index_ctx = nullptr;
const KeyCoder* _value_key_coder;
diff --git a/be/src/olap/tablet_schema.h b/be/src/olap/tablet_schema.h
index ad4897adc9b..8d12b4bdcb5 100644
--- a/be/src/olap/tablet_schema.h
+++ b/be/src/olap/tablet_schema.h
@@ -338,7 +338,11 @@ public:
bool is_inverted_index() const { return _index_type == IndexType::INVERTED; }
- void remove_parser_and_analyzer() { _properties.erase(INVERTED_INDEX_PARSER_KEY); }
+ void remove_parser_and_analyzer() {
+ _properties.erase(INVERTED_INDEX_PARSER_KEY);
+ _properties.erase(INVERTED_INDEX_PARSER_KEY_ALIAS);
+ _properties.erase(INVERTED_INDEX_CUSTOM_ANALYZER_KEY);
+ }
std::string field_pattern() const {
if (_properties.contains("field_pattern")) {
diff --git a/be/src/runtime/index_policy/index_policy_mgr.cpp b/be/src/runtime/index_policy/index_policy_mgr.cpp
index f032c976999..035dd19f35e 100644
--- a/be/src/runtime/index_policy/index_policy_mgr.cpp
+++ b/be/src/runtime/index_policy/index_policy_mgr.cpp
@@ -130,44 +130,66 @@ segment_v2::inverted_index::CustomAnalyzerPtr IndexPolicyMgr::get_policy_by_name
builder.with_tokenizer_config(tokenzier_name, {});
}
+ // Process char filters
+ process_filter_configs(index_policy_analyzer, PROP_CHAR_FILTER, "char filter",
+ [&builder](const std::string& name,
+ const segment_v2::inverted_index::Settings& settings) {
+ builder.add_char_filter_config(name, settings);
+ });
+
// Process token filters
- auto token_filter_it = index_policy_analyzer.properties.find(PROP_TOKEN_FILTER);
- if (token_filter_it != index_policy_analyzer.properties.end()) {
- std::vector<std::string> token_filter_strs;
- boost::split(token_filter_strs, token_filter_it->second, boost::is_any_of(","));
-
- for (auto& filter_name : token_filter_strs) {
- boost::trim(filter_name);
- if (filter_name.empty()) {
- continue;
- }
+ process_filter_configs(index_policy_analyzer, PROP_TOKEN_FILTER, "token filter",
+ [&builder](const std::string& name,
+ const segment_v2::inverted_index::Settings& settings) {
+ builder.add_token_filter_config(name, settings);
+ });
- if (_name_to_id.contains(filter_name)) {
- // Nested token filter policy
- const auto& filter_policy = _policys[_name_to_id[filter_name]];
- auto type_it = filter_policy.properties.find(PROP_TYPE);
- if (type_it == filter_policy.properties.end()) {
- throw Exception(ErrorCode::INVALID_ARGUMENT,
- "Invalid token filter configuration in policy: " + filter_name);
- }
+ auto custom_analyzer_config = builder.build();
+ return segment_v2::inverted_index::CustomAnalyzer::build_custom_analyzer(
+ custom_analyzer_config);
+}
+
+void IndexPolicyMgr::process_filter_configs(
+ const TIndexPolicy& index_policy_analyzer, const std::string& prop_name,
+ const std::string& error_prefix,
+ std::function<void(const std::string&, const segment_v2::inverted_index::Settings&)>
+ add_config_func) {
+ auto filter_it = index_policy_analyzer.properties.find(prop_name);
+ if (filter_it == index_policy_analyzer.properties.end()) {
+ return;
+ }
+
+ std::vector<std::string> filter_strs;
+ boost::split(filter_strs, filter_it->second, boost::is_any_of(","));
- segment_v2::inverted_index::Settings settings;
- for (const auto& prop : filter_policy.properties) {
- if (prop.first != PROP_TYPE) {
- settings.set(prop.first, prop.second);
- }
+ for (auto& filter_name : filter_strs) {
+ boost::trim(filter_name);
+ if (filter_name.empty()) {
+ continue;
+ }
+
+ if (_name_to_id.contains(filter_name)) {
+ // Nested filter policy
+ const auto& filter_policy = _policys[_name_to_id[filter_name]];
+ auto type_it = filter_policy.properties.find(PROP_TYPE);
+ if (type_it == filter_policy.properties.end()) {
+ throw Exception(
+ ErrorCode::INVALID_ARGUMENT,
+ "Invalid " + error_prefix + " configuration in policy: " + filter_name);
+ }
+
+ segment_v2::inverted_index::Settings settings;
+ for (const auto& prop : filter_policy.properties) {
+ if (prop.first != PROP_TYPE) {
+ settings.set(prop.first, prop.second);
}
- builder.add_token_filter_config(type_it->second, settings);
- } else {
- // Simple token filter
- builder.add_token_filter_config(filter_name, {});
}
+ add_config_func(type_it->second, settings);
+ } else {
+ // Simple filter
+ add_config_func(filter_name, {});
}
}
-
- auto custom_analyzer_config = builder.build();
- return segment_v2::inverted_index::CustomAnalyzer::build_custom_analyzer(
- custom_analyzer_config);
}
} // namespace doris
\ No newline at end of file
diff --git a/be/src/runtime/index_policy/index_policy_mgr.h b/be/src/runtime/index_policy/index_policy_mgr.h
index aa0b25a0448..707270930fe 100644
--- a/be/src/runtime/index_policy/index_policy_mgr.h
+++ b/be/src/runtime/index_policy/index_policy_mgr.h
@@ -40,9 +40,16 @@ public:
private:
constexpr static auto PROP_TOKENIZER = "tokenizer";
+ constexpr static auto PROP_CHAR_FILTER = "char_filter";
constexpr static auto PROP_TOKEN_FILTER = "token_filter";
constexpr static auto PROP_TYPE = "type";
+ void process_filter_configs(
+ const TIndexPolicy& index_policy_analyzer, const std::string& prop_name,
+ const std::string& error_prefix,
+ std::function<void(const std::string&, const segment_v2::inverted_index::Settings&)>
+ add_config_func);
+
std::shared_mutex _mutex;
Policys _policys;
diff --git a/be/src/vec/functions/function_tokenize.cpp b/be/src/vec/functions/function_tokenize.cpp
index c30cc041661..30c67f7c170 100644
--- a/be/src/vec/functions/function_tokenize.cpp
+++ b/be/src/vec/functions/function_tokenize.cpp
@@ -105,8 +105,8 @@ void FunctionTokenize::_do_tokenize(const ColumnString& src_column_string,
auto reader = InvertedIndexAnalyzer::create_reader(inverted_index_ctx.char_filter_map);
reader->init(tokenize_str.data, tokenize_str.size, true);
- auto analyzer_tokens = InvertedIndexAnalyzer::get_analyse_result(
- reader.get(), inverted_index_ctx.analyzer);
+ auto analyzer_tokens =
+ InvertedIndexAnalyzer::get_analyse_result(reader, inverted_index_ctx.analyzer);
rapidjson::Document doc;
doc.SetArray();
diff --git a/be/src/vec/functions/match.cpp b/be/src/vec/functions/match.cpp
index 166109775eb..ef94d534933 100644
--- a/be/src/vec/functions/match.cpp
+++ b/be/src/vec/functions/match.cpp
@@ -185,7 +185,7 @@ std::vector<TermInfo> FunctionMatchBase::analyse_query_str_token(
inverted_index_ctx->char_filter_map);
reader->init(match_query_str.data(), match_query_str.size(), true);
query_tokens = doris::segment_v2::inverted_index::InvertedIndexAnalyzer::get_analyse_result(
- reader.get(), inverted_index_ctx->analyzer);
+ reader, inverted_index_ctx->analyzer);
return query_tokens;
}
@@ -208,7 +208,7 @@ inline std::vector<TermInfo> FunctionMatchBase::analyse_data_token(
data_tokens =
doris::segment_v2::inverted_index::InvertedIndexAnalyzer::get_analyse_result(
- reader.get(), inverted_index_ctx->analyzer);
+ reader, inverted_index_ctx->analyzer);
}
} else {
const auto& str_ref = string_col->get_data_at(current_block_row_idx);
@@ -221,7 +221,7 @@ inline std::vector<TermInfo> FunctionMatchBase::analyse_data_token(
reader->init(str_ref.data, str_ref.size, true);
data_tokens =
doris::segment_v2::inverted_index::InvertedIndexAnalyzer::get_analyse_result(
- reader.get(), inverted_index_ctx->analyzer);
+ reader, inverted_index_ctx->analyzer);
}
}
return data_tokens;
diff --git a/be/test/olap/rowset/segment_v2/inverted_index/analysis_factory_mgr_test.cpp b/be/test/olap/rowset/segment_v2/inverted_index/analysis_factory_mgr_test.cpp
new file mode 100644
index 00000000000..7f061fbce7a
--- /dev/null
+++ b/be/test/olap/rowset/segment_v2/inverted_index/analysis_factory_mgr_test.cpp
@@ -0,0 +1,249 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "olap/rowset/segment_v2/inverted_index/analysis_factory_mgr.h"
+
+#include <gtest/gtest.h>
+
+#include "olap/rowset/segment_v2/inverted_index/char_filter/char_filter_factory.h"
+#include "olap/rowset/segment_v2/inverted_index/char_filter/empty_char_filter_factory.h"
+#include "olap/rowset/segment_v2/inverted_index/token_filter/empty_token_filter_factory.h"
+#include "olap/rowset/segment_v2/inverted_index/token_filter/token_filter_factory.h"
+#include "olap/rowset/segment_v2/inverted_index/tokenizer/empty/empty_tokenizer_factory.h"
+#include "olap/rowset/segment_v2/inverted_index/tokenizer/tokenizer_factory.h"
+
+namespace doris::segment_v2::inverted_index {
+
+class AnalysisFactoryMgrTest : public ::testing::Test {
+protected:
+ void SetUp() override { AnalysisFactoryMgr::instance().initialise(); }
+
+ void TearDown() override {}
+};
+
+TEST_F(AnalysisFactoryMgrTest, CreateBuiltInFactories) {
+ Settings empty_settings;
+
+ auto tokenizer_factory =
+ AnalysisFactoryMgr::instance().create<TokenizerFactory>("standard", empty_settings);
+ ASSERT_NE(tokenizer_factory, nullptr);
+ auto tokenizer = tokenizer_factory->create();
+ ASSERT_NE(tokenizer, nullptr);
+
+ auto char_filter_factory = AnalysisFactoryMgr::instance().create<CharFilterFactory>(
+ "char_replace", empty_settings);
+ ASSERT_NE(char_filter_factory, nullptr);
+
+ auto token_filter_factory =
+ AnalysisFactoryMgr::instance().create<TokenFilterFactory>("lowercase", empty_settings);
+ ASSERT_NE(token_filter_factory, nullptr);
+}
+
+TEST_F(AnalysisFactoryMgrTest, SameNameDifferentTypes) {
+ Settings empty_settings;
+
+ auto char_filter_factory =
+ AnalysisFactoryMgr::instance().create<CharFilterFactory>("empty", empty_settings);
+ ASSERT_NE(char_filter_factory, nullptr);
+
+ auto tokenizer_factory =
+ AnalysisFactoryMgr::instance().create<TokenizerFactory>("empty", empty_settings);
+ ASSERT_NE(tokenizer_factory, nullptr);
+
+ auto token_filter_factory =
+ AnalysisFactoryMgr::instance().create<TokenFilterFactory>("empty", empty_settings);
+ ASSERT_NE(token_filter_factory, nullptr);
+
+ EXPECT_NE(dynamic_cast<EmptyCharFilterFactory*>(char_filter_factory.get()), nullptr);
+ EXPECT_NE(dynamic_cast<EmptyTokenizerFactory*>(tokenizer_factory.get()), nullptr);
+ EXPECT_NE(dynamic_cast<EmptyTokenFilterFactory*>(token_filter_factory.get()), nullptr);
+}
+
+TEST_F(AnalysisFactoryMgrTest, EmptyCharFilterPassesThrough) {
+ Settings empty_settings;
+
+ auto factory =
+ AnalysisFactoryMgr::instance().create<CharFilterFactory>("empty", empty_settings);
+ ASSERT_NE(factory, nullptr);
+
+ std::string test_data = "Hello World!";
+ ReaderPtr reader = std::make_shared<lucene::util::SStringReader<char>>();
+ reader->init(test_data.data(), test_data.size(), false);
+
+ ReaderPtr filtered = factory->create(reader);
+ ASSERT_NE(filtered, nullptr);
+
+ const char* buffer = nullptr;
+ int32_t len = filtered->read((const void**)&buffer, 0, test_data.size());
+ EXPECT_EQ(len, test_data.size());
+ std::string result(buffer, len);
+ EXPECT_EQ(result, test_data);
+}
+
+TEST_F(AnalysisFactoryMgrTest, EmptyTokenizerReturnsWholeInput) {
+ Settings empty_settings;
+
+ auto factory = AnalysisFactoryMgr::instance().create<TokenizerFactory>("empty", empty_settings);
+ ASSERT_NE(factory, nullptr);
+
+ auto tokenizer = factory->create();
+ ASSERT_NE(tokenizer, nullptr);
+
+ std::string test_data = "Hello World Test";
+ ReaderPtr reader = std::make_shared<lucene::util::SStringReader<char>>();
+ reader->init(test_data.data(), test_data.size(), false);
+
+ tokenizer->set_reader(reader);
+ tokenizer->reset();
+
+ Token token;
+ EXPECT_TRUE(tokenizer->next(&token));
+ std::string result(token.termBuffer<char>(), token.termLength<char>());
+ EXPECT_EQ(result, test_data);
+
+ EXPECT_FALSE(tokenizer->next(&token));
+}
+
+TEST_F(AnalysisFactoryMgrTest, EmptyTokenFilterPassesThrough) {
+ Settings empty_settings;
+
+ auto tokenizer_factory =
+ AnalysisFactoryMgr::instance().create<TokenizerFactory>("keyword", empty_settings);
+ auto tokenizer = tokenizer_factory->create();
+
+ std::string test_data = "TestToken";
+ ReaderPtr reader = std::make_shared<lucene::util::SStringReader<char>>();
+ reader->init(test_data.data(), test_data.size(), false);
+ tokenizer->set_reader(reader);
+ tokenizer->reset();
+
+ auto filter_factory =
+ AnalysisFactoryMgr::instance().create<TokenFilterFactory>("empty", empty_settings);
+ auto filter = filter_factory->create(tokenizer);
+ ASSERT_NE(filter, nullptr);
+
+ Token token;
+ EXPECT_TRUE(filter->next(&token));
+ std::string result(token.termBuffer<char>(), token.termLength<char>());
+ EXPECT_EQ(result, test_data);
+
+ EXPECT_FALSE(filter->next(&token));
+}
+
+TEST_F(AnalysisFactoryMgrTest, UnknownFactoryNameThrowsException) {
+ Settings empty_settings;
+
+ EXPECT_THROW(
+ AnalysisFactoryMgr::instance().create<TokenizerFactory>("nonexistent", empty_settings),
+ Exception);
+
+ EXPECT_THROW(AnalysisFactoryMgr::instance().create<CharFilterFactory>("unknown_char_filter",
+ empty_settings),
+ Exception);
+
+ EXPECT_THROW(AnalysisFactoryMgr::instance().create<TokenFilterFactory>("unknown_filter",
+ empty_settings),
+ Exception);
+}
+
+TEST_F(AnalysisFactoryMgrTest, FactoryWithSettings) {
+ Settings settings;
+ settings.set("buffer_size", "512");
+
+ auto factory = AnalysisFactoryMgr::instance().create<TokenizerFactory>("keyword", settings);
+ ASSERT_NE(factory, nullptr);
+
+ auto tokenizer = factory->create();
+ ASSERT_NE(tokenizer, nullptr);
+}
+
+TEST_F(AnalysisFactoryMgrTest, SingletonBehavior) {
+ auto& instance1 = AnalysisFactoryMgr::instance();
+ auto& instance2 = AnalysisFactoryMgr::instance();
+
+ EXPECT_EQ(&instance1, &instance2);
+}
+
+TEST_F(AnalysisFactoryMgrTest, MultipleCreationsFromSameName) {
+ Settings empty_settings;
+
+ auto factory1 =
+ AnalysisFactoryMgr::instance().create<TokenizerFactory>("standard", empty_settings);
+ auto factory2 =
+ AnalysisFactoryMgr::instance().create<TokenizerFactory>("standard", empty_settings);
+
+ ASSERT_NE(factory1, nullptr);
+ ASSERT_NE(factory2, nullptr);
+
+ EXPECT_NE(factory1.get(), factory2.get());
+}
+
+TEST_F(AnalysisFactoryMgrTest, AllBuiltInTokenizersRegistered) {
+ Settings empty_settings;
+
+ std::vector<std::string> tokenizer_names = {"standard", "keyword", "ngram", "edge_ngram",
+ "char_group", "basic", "icu", "empty"};
+
+ for (const auto& name : tokenizer_names) {
+ auto factory =
+ AnalysisFactoryMgr::instance().create<TokenizerFactory>(name, empty_settings);
+ EXPECT_NE(factory, nullptr) << "Tokenizer '" << name << "' should be registered";
+ }
+}
+
+TEST_F(AnalysisFactoryMgrTest, AllBuiltInCharFiltersRegistered) {
+ Settings empty_settings;
+
+ std::vector<std::string> char_filter_names = {"char_replace", "empty"};
+
+ for (const auto& name : char_filter_names) {
+ auto factory =
+ AnalysisFactoryMgr::instance().create<CharFilterFactory>(name, empty_settings);
+ EXPECT_NE(factory, nullptr) << "Char filter '" << name << "' should be registered";
+ }
+}
+
+TEST_F(AnalysisFactoryMgrTest, AllBuiltInTokenFiltersRegistered) {
+ Settings empty_settings;
+
+ std::vector<std::string> token_filter_names = {"lowercase", "asciifolding", "word_delimiter",
+ "empty"};
+
+ for (const auto& name : token_filter_names) {
+ auto factory =
+ AnalysisFactoryMgr::instance().create<TokenFilterFactory>(name, empty_settings);
+ EXPECT_NE(factory, nullptr) << "Token filter '" << name << "' should be registered";
+ }
+}
+
+TEST_F(AnalysisFactoryMgrTest, TypeSafetyForDifferentFactoryTypes) {
+ Settings empty_settings;
+
+ EXPECT_THROW(
+ AnalysisFactoryMgr::instance().create<TokenizerFactory>("lowercase", empty_settings),
+ Exception);
+
+ EXPECT_THROW(
+ AnalysisFactoryMgr::instance().create<CharFilterFactory>("standard", empty_settings),
+ Exception);
+
+ EXPECT_THROW(AnalysisFactoryMgr::instance().create<TokenFilterFactory>("char_replace",
+ empty_settings),
+ Exception);
+}
+
+} // namespace doris::segment_v2::inverted_index
\ No newline at end of file
diff --git a/be/test/olap/rowset/segment_v2/inverted_index/analyzer/icu_analyzer_test.cpp b/be/test/olap/rowset/segment_v2/inverted_index/analyzer/icu_analyzer_test.cpp
index 4cd6d180a2e..eae5e59ffb2 100644
--- a/be/test/olap/rowset/segment_v2/inverted_index/analyzer/icu_analyzer_test.cpp
+++ b/be/test/olap/rowset/segment_v2/inverted_index/analyzer/icu_analyzer_test.cpp
@@ -35,11 +35,11 @@ protected:
analyzer.initDict("./be/dict/icu");
analyzer.set_lowercase(false);
- lucene::util::SStringReader<char> reader;
- reader.init(s.data(), s.size(), false);
+ auto reader = std::make_shared<lucene::util::SStringReader<char>>();
+ reader->init(s.data(), s.size(), false);
- std::unique_ptr<ICUTokenizer> tokenizer;
- tokenizer.reset((ICUTokenizer*)analyzer.tokenStream(L"", &reader));
+ std::unique_ptr<inverted_index::ICUTokenizer> tokenizer;
+ tokenizer.reset((inverted_index::ICUTokenizer*)analyzer.tokenStream(L"", reader));
Token t;
while (tokenizer->next(&t)) {
@@ -572,4 +572,55 @@ TEST_F(ICUTokenizerTest, TestICUScriptExtensions) {
}
}
+TEST_F(ICUTokenizerTest, TestICUAnalyzerCreateComponentsWithLowercase) {
+ std::vector<std::string> datas;
+
+ ICUAnalyzer analyzer;
+ analyzer.initDict("./be/dict/icu");
+ analyzer.set_lowercase(true);
+
+ std::string text = "Mixed Case TEXT with Numbers 123.";
+ auto reader = std::make_shared<lucene::util::SStringReader<char>>();
+ reader->init(text.data(), text.size(), false);
+
+ auto token_stream = analyzer.tokenStream(L"", reader);
+ ASSERT_NE(token_stream, nullptr);
+
+ Token t;
+ while (token_stream->next(&t)) {
+ std::string term(t.termBuffer<char>(), t.termLength<char>());
+ datas.emplace_back(term);
+ }
+
+ std::vector<std::string> expected = {"mixed", "case", "text", "with", "numbers", "123"};
+ ASSERT_EQ(datas.size(), expected.size());
+ for (size_t i = 0; i < datas.size(); i++) {
+ ASSERT_EQ(datas[i], expected[i]);
+ }
+
+ delete token_stream;
+}
+
+TEST_F(ICUTokenizerTest, TestICUAnalyzerTokenStreamThrowsException) {
+ ICUAnalyzer analyzer;
+ analyzer.initDict("./be/dict/icu");
+
+ std::string text = "Hello World!";
+ auto reader = std::make_shared<lucene::util::SStringReader<char>>();
+ reader->init(text.data(), text.size(), false);
+
+ EXPECT_THROW({ analyzer.tokenStream(L"", reader.get()); }, Exception);
+}
+
+TEST_F(ICUTokenizerTest, TestICUAnalyzerReusableTokenStreamThrowsException) {
+ ICUAnalyzer analyzer;
+ analyzer.initDict("./be/dict/icu");
+
+ std::string text = "Hello World!";
+ auto reader = std::make_shared<lucene::util::SStringReader<char>>();
+ reader->init(text.data(), text.size(), false);
+
+ EXPECT_THROW({ analyzer.reusableTokenStream(L"", reader.get()); }, Exception);
+}
+
} // namespace doris::segment_v2
diff --git a/be/test/olap/rowset/segment_v2/inverted_index/analyzer/simple_analyzer_test.cpp b/be/test/olap/rowset/segment_v2/inverted_index/analyzer/simple_analyzer_test.cpp
index 6dba8233a2e..6b36abcd56d 100644
--- a/be/test/olap/rowset/segment_v2/inverted_index/analyzer/simple_analyzer_test.cpp
+++ b/be/test/olap/rowset/segment_v2/inverted_index/analyzer/simple_analyzer_test.cpp
@@ -31,11 +31,11 @@ std::vector<std::string> tokenize(const std::string& s, bool lowercase = false)
BasicAnalyzer analyzer;
analyzer.set_lowercase(lowercase);
- lucene::util::SStringReader<char> reader;
- reader.init(s.data(), s.size(), false);
+ auto reader = std::make_shared<lucene::util::SStringReader<char>>();
+ reader->init(s.data(), s.size(), false);
- std::unique_ptr<BasicTokenizer> tokenizer;
- tokenizer.reset((BasicTokenizer*)analyzer.tokenStream(L"", &reader));
+ std::unique_ptr<inverted_index::BasicTokenizer> tokenizer;
+ tokenizer.reset((inverted_index::BasicTokenizer*)analyzer.tokenStream(L"", reader));
Token t;
while (tokenizer->next(&t)) {
diff --git a/be/test/olap/rowset/segment_v2/inverted_index/ananlyzer/custom_analyzer_test.cpp b/be/test/olap/rowset/segment_v2/inverted_index/ananlyzer/custom_analyzer_test.cpp
index f71a48ee67c..0f18b4d5e60 100644
--- a/be/test/olap/rowset/segment_v2/inverted_index/ananlyzer/custom_analyzer_test.cpp
+++ b/be/test/olap/rowset/segment_v2/inverted_index/ananlyzer/custom_analyzer_test.cpp
@@ -94,12 +94,12 @@ protected:
};
int32_t tokenize(const CustomAnalyzerPtr& custom_analyzer, const std::vector<std::string>& lines) {
- lucene::util::SStringReader<char> reader;
+ auto reader = std::make_shared<lucene::util::SStringReader<char>>();
size_t total_count = 0;
Token t;
for (size_t i = 0; i < lines.size(); ++i) {
- reader.init(lines[i].data(), lines[i].size(), false);
- auto* token_stream = custom_analyzer->reusableTokenStream(L"", &reader);
+ reader->init(lines[i].data(), lines[i].size(), false);
+ auto* token_stream = custom_analyzer->reusableTokenStream(L"", reader);
token_stream->reset();
while (token_stream->next(&t)) {
total_count++;
@@ -120,9 +120,9 @@ struct ExpectedToken {
std::vector<ExpectedToken> tokenize1(const CustomAnalyzerPtr& custom_analyzer,
const std::string line) {
std::vector<ExpectedToken> results;
- lucene::util::SStringReader<char> reader;
- reader.init(line.data(), line.size(), false);
- auto* token_stream = custom_analyzer->reusableTokenStream(L"", &reader);
+ auto reader = std::make_shared<lucene::util::SStringReader<char>>();
+ reader->init(line.data(), line.size(), false);
+ auto* token_stream = custom_analyzer->reusableTokenStream(L"", reader);
token_stream->reset();
Token t;
while (token_stream->next(&t)) {
@@ -196,9 +196,69 @@ TEST_F(CustomAnalyzerTest, CustomNgramAnalyzer) {
}
}
+TEST_F(CustomAnalyzerTest, TokenStreamNotSupported) {
+ CustomAnalyzerConfig::Builder builder;
+ builder.with_tokenizer_config("standard", {});
+ auto custom_analyzer_config = builder.build();
+ auto custom_analyzer = CustomAnalyzer::build_custom_analyzer(custom_analyzer_config);
+
+ auto reader = std::make_shared<lucene::util::SStringReader<char>>();
+ reader->init("test content", 12, false);
+
+ EXPECT_THROW({ custom_analyzer->tokenStream(L"field", reader.get()); }, Exception);
+
+ EXPECT_THROW({ custom_analyzer->reusableTokenStream(L"field", reader.get()); }, Exception);
+}
+
+TEST_F(CustomAnalyzerTest, ReusableTokenStreamNotSupported) {
+ CustomAnalyzerConfig::Builder builder;
+ builder.with_tokenizer_config("standard", {});
+ auto custom_analyzer_config = builder.build();
+ auto custom_analyzer = CustomAnalyzer::build_custom_analyzer(custom_analyzer_config);
+
+ auto reader = std::make_shared<lucene::util::SStringReader<char>>();
+ reader->init("test content", 12, false);
+
+ EXPECT_THROW({ custom_analyzer->reusableTokenStream(L"field", reader.get()); }, Exception);
+
+ try {
+ custom_analyzer->reusableTokenStream(L"field", reader.get());
+ FAIL() << "Expected Exception to be thrown";
+ } catch (const Exception& e) {
+ EXPECT_EQ(e.code(), ErrorCode::INVERTED_INDEX_NOT_SUPPORTED);
+ EXPECT_STREQ(e.what(), "[E-6001] CustomAnalyzer::reusableTokenStream not supported");
+ }
+}
+
+TEST_F(CustomAnalyzerTest, TokenStreamWithReaderPtr) {
+ CustomAnalyzerConfig::Builder builder;
+ builder.with_tokenizer_config("standard", {});
+ builder.add_token_filter_config("lowercase", {});
+ auto custom_analyzer_config = builder.build();
+ auto custom_analyzer = CustomAnalyzer::build_custom_analyzer(custom_analyzer_config);
+
+ auto reader = std::make_shared<lucene::util::SStringReader<char>>();
+ reader->init("Hello World Test", 16, false);
+
+ auto* token_stream = custom_analyzer->tokenStream(L"field", reader);
+ EXPECT_NE(token_stream, nullptr);
+
+ Token t;
+ std::vector<std::string> tokens;
+ token_stream->reset();
+ while (token_stream->next(&t)) {
+ tokens.emplace_back(std::string(t.termBuffer<char>(), t.termLength<char>()));
+ }
+
+ std::vector<std::string> expected = {"hello", "world", "test"};
+ EXPECT_EQ(tokens, expected);
+
+ delete token_stream;
+}
+
// TEST_F(CustomAnalyzerTest, test) {
// std::string name = "name";
-// std::string path = "/mnt/disk2/yangsiyu/clucene/index";
+// std::string path = "/mnt/disk3/yangsiyu/clucene";
// std::vector<std::string> lines;
@@ -213,145 +273,157 @@ TEST_F(CustomAnalyzerTest, CustomNgramAnalyzer) {
// std::cout << "lines size: " << lines.size() << std::endl;
+// Settings char_replace_params;
+// char_replace_params.set("char_filter_pattern", "_");
+// char_replace_params.set("char_filter_replacement", " ");
+
// Settings word_delimiter_params;
// word_delimiter_params.set("preserve_original", "true");
// CustomAnalyzerConfig::Builder builder;
// builder.with_tokenizer_config("standard", {});
-// builder.add_token_filter_config("word_delimiter", word_delimiter_params);
+// builder.add_char_filter_config("char_replace", char_replace_params);
+// // builder.add_token_filter_config("word_delimiter", word_delimiter_params);
// // builder.add_token_filter_config("asciifolding", {});
-// // builder.add_token_filter_config("lowercase", {});
+// builder.add_token_filter_config("lowercase", {});
// auto custom_analyzer_config = builder.build();
// auto custom_analyzer = CustomAnalyzer::build_custom_analyzer(custom_analyzer_config);
-// {
-// TimeGuard t("load time");
+// auto result = tokenize1(custom_analyzer, lines[0]);
+// for (const auto& token : result) {
+// std::cout << token.term << " " << token.pos << std::endl;
+// }
-// lucene::index::IndexWriter indexwriter(path.c_str(), custom_analyzer.get(), true);
-// indexwriter.setRAMBufferSizeMB(512);
-// indexwriter.setMaxFieldLength(0x7FFFFFFFL);
-// indexwriter.setMergeFactor(1000000000);
-// indexwriter.setUseCompoundFile(false);
+// // {
+// // TimeGuard t("load time");
-// lucene::util::SStringReader<char> reader;
+// // lucene::index::IndexWriter indexwriter(path.c_str(), custom_analyzer.get(), true);
+// // indexwriter.setRAMBufferSizeMB(512);
+// // indexwriter.setMaxFieldLength(0x7FFFFFFFL);
+// // indexwriter.setMergeFactor(1000000000);
+// // indexwriter.setUseCompoundFile(false);
-// lucene::document::Document doc;
-// int32_t field_config = lucene::document::Field::STORE_NO;
-// field_config |= lucene::document::Field::INDEX_NONORMS;
-// field_config |= lucene::document::Field::INDEX_TOKENIZED;
-// auto field_name = std::wstring(name.begin(), name.end());
-// auto* field = _CLNEW lucene::document::Field(field_name.c_str(), field_config);
-// field->setOmitTermFreqAndPositions(false);
-// doc.add(*field);
+// // auto reader = std::make_shared<lucene::util::SStringReader<char>>();
-// for (int32_t j = 0; j < 1; j++) {
-// for (size_t k = 0; k < lines.size(); k++) {
-// reader.init(lines[k].data(), lines[k].size(), false);
-// auto* stream = custom_analyzer->reusableTokenStream(field->name(), &reader);
-// field->setValue(stream);
+// // lucene::document::Document doc;
+// // int32_t field_config = lucene::document::Field::STORE_NO;
+// // field_config |= lucene::document::Field::INDEX_NONORMS;
+// // field_config |= lucene::document::Field::INDEX_TOKENIZED;
+// // auto field_name = std::wstring(name.begin(), name.end());
+// // auto* field = _CLNEW lucene::document::Field(field_name.c_str(), field_config);
+// // field->setOmitTermFreqAndPositions(false);
+// // doc.add(*field);
-// indexwriter.addDocument(&doc);
-// }
-// }
+// // for (int32_t j = 0; j < 1; j++) {
+// // for (size_t k = 0; k < lines.size(); k++) {
+// // reader->init(lines[k].data(), lines[k].size(), false);
+// // auto* stream = custom_analyzer->reusableTokenStream(field->name(), reader);
+// // field->setValue(stream);
-// std::cout << "---------------------" << std::endl;
+// // indexwriter.addDocument(&doc);
+// // }
+// // }
-// indexwriter.close();
-// }
+// // std::cout << "---------------------" << std::endl;
-// std::cout << "-----------" << std::endl;
-
-// try {
-// {
-// auto* dir = FSDirectory::getDirectory(path.c_str());
-// auto* reader = IndexReader::open(dir, 1024 * 1024, true);
-// auto searcher = std::make_shared<IndexSearcher>(reader);
-
-// // std::cout << "macDoc: " << reader->maxDoc() << std::endl;
-
-// {
-// TimeGuard time("query time");
-
-// {
-// TQueryOptions query_options;
-// doris::segment_v2::PhraseQuery query(searcher, query_options, nullptr);
-
-// InvertedIndexQueryInfo query_info;
-// query_info.field_name = L"name";
-// {
-// doris::segment_v2::TermInfo t;
-// t.term = "Super_Duper";
-// t.position = 1;
-// query_info.term_infos.emplace_back(std::move(t));
-// }
-// {
-// doris::segment_v2::TermInfo t;
-// t.term = "Super";
-// t.position = 1;
-// query_info.term_infos.emplace_back(std::move(t));
-// }
-// {
-// doris::segment_v2::TermInfo t;
-// t.term = "Duper";
-// t.position = 2;
-// query_info.term_infos.emplace_back(std::move(t));
-// }
-// {
-// doris::segment_v2::TermInfo t;
-// t.term = "c";
-// t.position = 3;
-// query_info.term_infos.emplace_back(std::move(t));
-// }
-// query_info.slop = 1;
-// query_info.ordered = true;
-// query.add(query_info);
-
-// roaring::Roaring result;
-// query.search(result);
-
-// std::cout << "phrase_query count: " << result.cardinality() << std::endl;
-// }
-// // {
-// // TQueryOptions query_options;
-// // doris::segment_v2::PhrasePrefixQuery query(searcher, query_options, nullptr);
-
-// // InvertedIndexQueryInfo query_info;
-// // query_info.field_name = L"name";
-// // {
-// // doris::segment_v2::TermInfo t;
-// // t.term = "Super_Duper";
-// // t.position = 1;
-// // query_info.term_infos.emplace_back(std::move(t));
-// // }
-// // {
-// // doris::segment_v2::TermInfo t;
-// // t.term = "Super";
-// // t.position = 1;
-// // query_info.term_infos.emplace_back(std::move(t));
-// // }
-// // {
-// // doris::segment_v2::TermInfo t;
-// // t.term = "Dup";
-// // t.position = 2;
-// // query_info.term_infos.emplace_back(std::move(t));
-// // }
-// // query.add(query_info);
-
-// // roaring::Roaring result;
-// // query.search(result);
-
-// // std::cout << "phrase_prefix_query count: " << result.cardinality() << std::endl;
-// // }
-// }
-
-// reader->close();
-// _CLLDELETE(reader);
-// _CLDECDELETE(dir);
-// }
-// } catch (const CLuceneError& e) {
-// std::cout << e.number() << ": " << e.what() << std::endl;
-// }
+// // indexwriter.close();
+// // }
+
+// // std::cout << "-----------" << std::endl;
+
+// // try {
+// // {
+// // auto* dir = FSDirectory::getDirectory(path.c_str());
+// // auto* reader = IndexReader::open(dir, 1024 * 1024, true);
+// // auto searcher = std::make_shared<IndexSearcher>(reader);
+
+// // // std::cout << "macDoc: " << reader->maxDoc() << std::endl;
+
+// // {
+// // TimeGuard time("query time");
+
+// // {
+// // IndexQueryContextPtr context = std::make_shared<IndexQueryContext>();
+
+// // TQueryOptions query_options;
+// // doris::segment_v2::PhraseQuery query(searcher, context);
+
+// // InvertedIndexQueryInfo query_info;
+// // query_info.field_name = L"name";
+// // {
+// // doris::segment_v2::TermInfo t;
+// // t.term = "Super_Duper";
+// // t.position = 1;
+// // query_info.term_infos.emplace_back(std::move(t));
+// // }
+// // {
+// // doris::segment_v2::TermInfo t;
+// // t.term = "Super";
+// // t.position = 1;
+// // query_info.term_infos.emplace_back(std::move(t));
+// // }
+// // {
+// // doris::segment_v2::TermInfo t;
+// // t.term = "Duper";
+// // t.position = 2;
+// // query_info.term_infos.emplace_back(std::move(t));
+// // }
+// // {
+// // doris::segment_v2::TermInfo t;
+// // t.term = "c";
+// // t.position = 3;
+// // query_info.term_infos.emplace_back(std::move(t));
+// // }
+// // query_info.slop = 1;
+// // query_info.ordered = true;
+// // query.add(query_info);
+
+// // roaring::Roaring result;
+// // query.search(result);
+
+// // std::cout << "phrase_query count: " << result.cardinality() << std::endl;
+// // }
+// // // {
+// // // TQueryOptions query_options;
+// // // doris::segment_v2::PhrasePrefixQuery query(searcher, query_options, nullptr);
+
+// // // InvertedIndexQueryInfo query_info;
+// // // query_info.field_name = L"name";
+// // // {
+// // // doris::segment_v2::TermInfo t;
+// // // t.term = "Super_Duper";
+// // // t.position = 1;
+// // // query_info.term_infos.emplace_back(std::move(t));
+// // // }
+// // // {
+// // // doris::segment_v2::TermInfo t;
+// // // t.term = "Super";
+// // // t.position = 1;
+// // // query_info.term_infos.emplace_back(std::move(t));
+// // // }
+// // // {
+// // // doris::segment_v2::TermInfo t;
+// // // t.term = "Dup";
+// // // t.position = 2;
+// // // query_info.term_infos.emplace_back(std::move(t));
+// // // }
+// // // query.add(query_info);
+
+// // // roaring::Roaring result;
+// // // query.search(result);
+
+// // // std::cout << "phrase_prefix_query count: " << result.cardinality() << std::endl;
+// // // }
+// // }
+
+// // reader->close();
+// // _CLLDELETE(reader);
+// // _CLDECDELETE(dir);
+// // }
+// // } catch (const CLuceneError& e) {
+// // std::cout << e.number() << ": " << e.what() << std::endl;
+// // }
// }
} // namespace doris::segment_v2::inverted_index
\ No newline at end of file
diff --git a/be/test/olap/rowset/segment_v2/inverted_index/char_filter/char_filter_test.cpp b/be/test/olap/rowset/segment_v2/inverted_index/char_filter/char_filter_test.cpp
new file mode 100644
index 00000000000..020f0db8d11
--- /dev/null
+++ b/be/test/olap/rowset/segment_v2/inverted_index/char_filter/char_filter_test.cpp
@@ -0,0 +1,76 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "olap/rowset/segment_v2/inverted_index/char_filter/char_filter.h"
+
+#include <gtest/gtest.h>
+
+#include <memory>
+
+#include "olap/rowset/segment_v2/inverted_index/util/reader.h"
+
+using namespace lucene::analysis;
+
+namespace doris::segment_v2::inverted_index {
+
+class MockDorisCharFilter : public DorisCharFilter {
+public:
+ MockDorisCharFilter(ReaderPtr reader) : DorisCharFilter(std::move(reader)) {}
+ ~MockDorisCharFilter() override = default;
+
+ void initialize() override {}
+
+ // 实现必需的虚函数
+ void init(const void* _value, int32_t _length, bool copyData) override {
+ if (_reader) {
+ _reader->init(_value, _length, copyData);
+ }
+ }
+
+ int32_t read(const void** start, int32_t min, int32_t max) override {
+ if (_reader) {
+ return _reader->read(start, min, max);
+ }
+ return -1;
+ }
+
+ int32_t readCopy(void* start, int32_t off, int32_t len) override {
+ if (_reader) {
+ return _reader->readCopy(start, off, len);
+ }
+ return -1;
+ }
+};
+
+class DorisCharFilterTest : public ::testing::Test {
+protected:
+ void SetUp() override { _mock_reader = std::make_shared<lucene::util::SStringReader<char>>(); }
+
+ ReaderPtr _mock_reader;
+};
+
+TEST_F(DorisCharFilterTest, ExceptionThrowing) {
+ auto filter = std::make_shared<MockDorisCharFilter>(_mock_reader);
+
+ EXPECT_THROW(filter->position(), doris::Exception);
+
+ EXPECT_THROW(filter->skip(10), doris::Exception);
+
+ EXPECT_THROW(filter->size(), doris::Exception);
+}
+
+} // namespace doris::segment_v2::inverted_index
\ No newline at end of file
diff --git a/be/test/olap/rowset/segment_v2/inverted_index/char_filter/char_replace_char_filter_factory_test.cpp b/be/test/olap/rowset/segment_v2/inverted_index/char_filter/char_replace_char_filter_factory_test.cpp
new file mode 100644
index 00000000000..28bcc33f9ca
--- /dev/null
+++ b/be/test/olap/rowset/segment_v2/inverted_index/char_filter/char_replace_char_filter_factory_test.cpp
@@ -0,0 +1,193 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "olap/rowset/segment_v2/inverted_index/char_filter/char_replace_char_filter_factory.h"
+
+#include <gtest/gtest.h>
+
+namespace doris::segment_v2::inverted_index {
+
+ReaderPtr create_char_replace_filter(const std::string& text, const std::string& pattern,
+ const std::string& replacement = " ") {
+ ReaderPtr reader = std::make_shared<lucene::util::SStringReader<char>>();
+ reader->init(text.data(), text.size(), false);
+
+ Settings settings;
+ settings.set(CHAR_REPLACE_PATTERN, pattern);
+ settings.set(CHAR_REPLACE_REPLACEMENT, replacement);
+
+ CharReplaceCharFilterFactory factory;
+ factory.initialize(settings);
+ auto char_filter = factory.create(reader);
+ return char_filter;
+}
+
+struct ExpectedOutput {
+ std::string text;
+ std::string expected;
+};
+
+class CharReplaceCharFilterFactoryTest : public ::testing::Test {
+protected:
+ void assert_char_filter_output(const std::string& input_text, const std::string& pattern,
+ const std::string& expected_output,
+ const std::string& replacement = " ") {
+ auto char_filter = create_char_replace_filter(input_text, pattern, replacement);
+
+ const void* data = nullptr;
+ int32_t read_len = char_filter->read(&data, 0, char_filter->size());
+ ASSERT_GT(read_len, 0) << "Failed to read from char filter";
+
+ std::string result(static_cast<const char*>(data), read_len);
+ EXPECT_EQ(result, expected_output) << "Char filter output mismatch";
+ }
+};
+
+TEST_F(CharReplaceCharFilterFactoryTest, BasicReplacement) {
+ assert_char_filter_output("hello,world", ",", "hello world");
+}
+
+TEST_F(CharReplaceCharFilterFactoryTest, MultipleReplacements) {
+ assert_char_filter_output("a,b,c,d", ",", "a b c d");
+}
+
+TEST_F(CharReplaceCharFilterFactoryTest, CustomReplacement) {
+ assert_char_filter_output("hello,world", ",", "hello_world", "_");
+}
+
+TEST_F(CharReplaceCharFilterFactoryTest, MultiplePatternChars) {
+ assert_char_filter_output("a,b;c:d", ",;:", "a b c d");
+}
+
+TEST_F(CharReplaceCharFilterFactoryTest, NoMatch) {
+ assert_char_filter_output("hello world", "x", "hello world");
+}
+
+TEST_F(CharReplaceCharFilterFactoryTest, EmptyInput) {
+ auto char_filter = create_char_replace_filter("", ",");
+
+ const void* data = nullptr;
+ int32_t read_len = char_filter->read(&data, 0, char_filter->size());
+
+ // For empty input, read should return -1 (EOF)
+ EXPECT_EQ(read_len, -1) << "Empty input should return EOF";
+ EXPECT_TRUE(data == nullptr || char_filter->size() == 0)
+ << "No data should be available for empty input";
+}
+
+TEST_F(CharReplaceCharFilterFactoryTest, EmptyPattern) {
+ try {
+ assert_char_filter_output("hello,world", "", "hello,world");
+ } catch (const Exception& e) {
+ EXPECT_EQ(e.code(), ErrorCode::INVALID_ARGUMENT);
+ }
+}
+
+TEST_F(CharReplaceCharFilterFactoryTest, AllCharsMatch) {
+ assert_char_filter_output("abc", "abc", " ");
+}
+
+TEST_F(CharReplaceCharFilterFactoryTest, ChineseCharacters) {
+ assert_char_filter_output("你好,世界", ",", "你好 世界");
+}
+
+TEST_F(CharReplaceCharFilterFactoryTest, SpecialCharacters) {
+ assert_char_filter_output("test@example.com", "@.", "test example com");
+}
+
+TEST_F(CharReplaceCharFilterFactoryTest, FactoryInitialization) {
+ Settings settings;
+ settings.set(CHAR_REPLACE_PATTERN, ",");
+ settings.set(CHAR_REPLACE_REPLACEMENT, " ");
+
+ CharReplaceCharFilterFactory factory;
+ EXPECT_NO_THROW(factory.initialize(settings));
+}
+
+TEST_F(CharReplaceCharFilterFactoryTest, FactoryInitializationMissingPattern) {
+ Settings settings;
+ // Missing pattern - should throw exception
+ settings.set(CHAR_REPLACE_REPLACEMENT, " ");
+
+ CharReplaceCharFilterFactory factory;
+ EXPECT_NO_THROW(factory.initialize(settings));
+}
+
+TEST_F(CharReplaceCharFilterFactoryTest, FactoryInitializationEmptyPattern) {
+ Settings settings;
+ settings.set(CHAR_REPLACE_PATTERN, "");
+ settings.set(CHAR_REPLACE_REPLACEMENT, " ");
+
+ CharReplaceCharFilterFactory factory;
+ EXPECT_THROW(factory.initialize(settings), Exception);
+}
+
+TEST_F(CharReplaceCharFilterFactoryTest, FactoryCreateFilter) {
+ Settings settings;
+ settings.set(CHAR_REPLACE_PATTERN, ",");
+ settings.set(CHAR_REPLACE_REPLACEMENT, " ");
+
+ CharReplaceCharFilterFactory factory;
+ factory.initialize(settings);
+
+ ReaderPtr input_reader = std::make_shared<lucene::util::SStringReader<char>>();
+ input_reader->init("test,data", 9, false);
+
+ auto char_filter = factory.create(input_reader);
+ ASSERT_NE(char_filter, nullptr);
+
+ const void* data = nullptr;
+ int32_t read_len = char_filter->read(&data, 0, char_filter->size());
+ ASSERT_GT(read_len, 0);
+
+ std::string result(static_cast<const char*>(data), read_len);
+ EXPECT_EQ(result, "test data");
+}
+
+TEST_F(CharReplaceCharFilterFactoryTest, DefaultReplacement) {
+ Settings settings;
+ settings.set(CHAR_REPLACE_PATTERN, ",");
+ // No replacement specified - should use default " "
+
+ CharReplaceCharFilterFactory factory;
+ factory.initialize(settings);
+
+ ReaderPtr input_reader = std::make_shared<lucene::util::SStringReader<char>>();
+ input_reader->init("a,b,c", 5, false);
+
+ auto char_filter = factory.create(input_reader);
+
+ const void* data = nullptr;
+ int32_t read_len = char_filter->read(&data, 0, char_filter->size());
+ ASSERT_GT(read_len, 0);
+
+ std::string result(static_cast<const char*>(data), read_len);
+ EXPECT_EQ(result, "a b c");
+}
+
+TEST_F(CharReplaceCharFilterFactoryTest, EdgeCases) {
+ // Test with whitespace only
+ assert_char_filter_output(" ", ",", " ");
+
+ // Test with only pattern characters
+ assert_char_filter_output(",,,", ",", " ");
+
+ // Test with mixed content
+ assert_char_filter_output("a,,b,", ",", "a b ");
+}
+
+} // namespace doris::segment_v2::inverted_index
\ No newline at end of file
diff --git a/be/test/olap/rowset/segment_v2/inverted_index/setting_test.cpp b/be/test/olap/rowset/segment_v2/inverted_index/setting_test.cpp
index 84dd8d11a42..e616ae03620 100644
--- a/be/test/olap/rowset/segment_v2/inverted_index/setting_test.cpp
+++ b/be/test/olap/rowset/segment_v2/inverted_index/setting_test.cpp
@@ -111,21 +111,19 @@ TEST_F(SettingsTest, GetStringReturnsCorrectValues) {
TEST_F(SettingsTest, GetEntryListReturnsCorrectValues) {
Settings settings(testMap);
- auto emptyList = settings.get_entry_list("list_empty");
- EXPECT_TRUE(emptyList.empty());
+ // auto emptyList = settings.get_entry_list("list_empty");
+ // EXPECT_TRUE(emptyList.empty());
- auto singleList = settings.get_entry_list("list_single");
- ASSERT_EQ(singleList.size(), 1);
- EXPECT_EQ(singleList[0], "item1");
+ // auto singleList = settings.get_entry_list("list_single");
+ // ASSERT_EQ(singleList.size(), 1);
+ // EXPECT_EQ(singleList[0], "item1");
auto multiList = settings.get_entry_list("list_multiple");
- ASSERT_EQ(multiList.size(), 3);
- EXPECT_EQ(multiList[0], "item1");
- EXPECT_EQ(multiList[1], "item2");
- EXPECT_EQ(multiList[2], "item3");
+ ASSERT_EQ(multiList.size(), 1);
+ EXPECT_EQ(multiList[0], "item1][item2][item3");
- auto nonExistent = settings.get_entry_list("non_existent");
- EXPECT_TRUE(nonExistent.empty());
+ // auto nonExistent = settings.get_entry_list("non_existent");
+ // EXPECT_TRUE(nonExistent.empty());
}
TEST_F(SettingsTest, GetWordSetReturnsCorrectValues) {
@@ -181,4 +179,79 @@ TEST_F(SettingsTest, MoveConstructorWorks) {
EXPECT_TRUE(settings1.empty());
}
+TEST_F(SettingsTest, GetEntryListWithBracketsInside) {
+ Settings settings;
+
+ settings.set("list_with_brackets_inside", "[item[with]brackets]");
+ auto singleWithBrackets = settings.get_entry_list("list_with_brackets_inside");
+ ASSERT_EQ(singleWithBrackets.size(), 1);
+ EXPECT_EQ(singleWithBrackets[0], "item[with]brackets");
+
+ settings.set("list_multiple_with_brackets",
+ "[item1[with]brackets][item2[also]has[brackets]][item3]");
+ auto multiWithBrackets = settings.get_entry_list("list_multiple_with_brackets");
+ ASSERT_EQ(multiWithBrackets.size(), 1);
+ EXPECT_EQ(multiWithBrackets[0], "item1[with]brackets][item2[also]has[brackets]][item3");
+
+ settings.set("list_nested_brackets", "[[[nested]]][[double]][single]");
+ auto nestedBrackets = settings.get_entry_list("list_nested_brackets");
+ ASSERT_EQ(nestedBrackets.size(), 1);
+ EXPECT_EQ(nestedBrackets[0], "[[nested]]][[double]][single");
+
+ settings.set("list_empty_brackets_inside", "[item[]with][empty][]brackets");
+ EXPECT_THROW(settings.get_entry_list("list_empty_brackets_inside"), Exception);
+}
+
+TEST_F(SettingsTest, GetEntryListWithCommaSeparators) {
+ Settings settings;
+
+ settings.set("list_comma_separated", "[item1],[item2],[item3]");
+ auto commaList = settings.get_entry_list("list_comma_separated");
+ ASSERT_EQ(commaList.size(), 3);
+ EXPECT_EQ(commaList[0], "item1");
+ EXPECT_EQ(commaList[1], "item2");
+ EXPECT_EQ(commaList[2], "item3");
+
+ settings.set("list_with_empty_items", "[item1],[],[item3]");
+ auto listWithEmpty = settings.get_entry_list("list_with_empty_items");
+ ASSERT_EQ(listWithEmpty.size(), 2);
+ EXPECT_EQ(listWithEmpty[0], "item1");
+ EXPECT_EQ(listWithEmpty[1], "item3");
+
+ settings.set("list_comma_separated1", "[item1], [item2], [item3]");
+ auto commaList1 = settings.get_entry_list("list_comma_separated1");
+ ASSERT_EQ(commaList1.size(), 3);
+ EXPECT_EQ(commaList1[0], "item1");
+ EXPECT_EQ(commaList1[1], "item2");
+ EXPECT_EQ(commaList1[2], "item3");
+
+ settings.set("list_with_empty_items1", "");
+ auto listWithEmpty1 = settings.get_entry_list("list_with_empty_items1");
+ ASSERT_EQ(listWithEmpty1.size(), 0);
+}
+
+TEST_F(SettingsTest, GetEntryListSpecExamples) {
+ Settings settings;
+
+ settings.set("ex_empty", "[]");
+ auto v0 = settings.get_entry_list("ex_empty");
+ EXPECT_TRUE(v0.empty());
+
+ settings.set("ex_nested_balanced", "[[123]]");
+ auto v1 = settings.get_entry_list("ex_nested_balanced");
+ ASSERT_EQ(v1.size(), 1);
+ EXPECT_EQ(v1[0], "[123]");
+
+ settings.set("ex_nested_unbalanced_inside", "[[123[]");
+ auto v2 = settings.get_entry_list("ex_nested_unbalanced_inside");
+ ASSERT_EQ(v2.size(), 1);
+ EXPECT_EQ(v2[0], "[123[");
+
+ settings.set("ex_no_comma_multiple", "[123][123");
+ EXPECT_THROW(settings.get_entry_list("ex_no_comma_multiple"), Exception);
+
+ settings.set("ex_comma_missing_closing", "[123],[123");
+ EXPECT_THROW(settings.get_entry_list("ex_comma_missing_closing"), Exception);
+}
+
} // namespace doris::segment_v2::inverted_index
diff --git a/be/test/olap/rowset/segment_v2/inverted_index/token_filter/ascii_folding_filter_factory_test.cpp b/be/test/olap/rowset/segment_v2/inverted_index/token_filter/ascii_folding_filter_factory_test.cpp
index 8187a8bf64c..ce659971367 100644
--- a/be/test/olap/rowset/segment_v2/inverted_index/token_filter/ascii_folding_filter_factory_test.cpp
+++ b/be/test/olap/rowset/segment_v2/inverted_index/token_filter/ascii_folding_filter_factory_test.cpp
@@ -24,14 +24,14 @@
namespace doris::segment_v2::inverted_index {
TokenStreamPtr create_filter(const std::string& text, Settings token_filter_settings) {
- static lucene::util::SStringReader<char> reader;
- reader.init(text.data(), text.size(), false);
+ ReaderPtr reader = std::make_shared<lucene::util::SStringReader<char>>();
+ reader->init(text.data(), text.size(), false);
Settings settings;
KeywordTokenizerFactory tokenizer_factory;
tokenizer_factory.initialize(settings);
auto tokenizer = tokenizer_factory.create();
- tokenizer->set_reader(&reader);
+ tokenizer->set_reader(reader);
ASCIIFoldingFilterFactory token_filter_factory;
token_filter_factory.initialize(token_filter_settings);
diff --git a/be/test/olap/rowset/segment_v2/inverted_index/token_filter/lower_case_filter_factory_test.cpp b/be/test/olap/rowset/segment_v2/inverted_index/token_filter/lower_case_filter_factory_test.cpp
index 588dc0381a1..010c3b053b4 100644
--- a/be/test/olap/rowset/segment_v2/inverted_index/token_filter/lower_case_filter_factory_test.cpp
+++ b/be/test/olap/rowset/segment_v2/inverted_index/token_filter/lower_case_filter_factory_test.cpp
@@ -25,13 +25,13 @@
namespace doris::segment_v2::inverted_index {
TokenStreamPtr create_lowercase_filter(const std::string& text, Settings settings = Settings()) {
- static lucene::util::SStringReader<char> reader;
- reader.init(text.data(), text.size(), false);
+ ReaderPtr reader = std::make_shared<lucene::util::SStringReader<char>>();
+ reader->init(text.data(), text.size(), false);
KeywordTokenizerFactory tokenizer_factory;
tokenizer_factory.initialize(Settings());
auto tokenizer = tokenizer_factory.create();
- tokenizer->set_reader(&reader);
+ tokenizer->set_reader(reader);
LowerCaseFilterFactory filter_factory;
filter_factory.initialize(settings);
diff --git a/be/test/olap/rowset/segment_v2/inverted_index/token_filter/word_delimiter_filter_test.cpp b/be/test/olap/rowset/segment_v2/inverted_index/token_filter/word_delimiter_filter_test.cpp
index 4d8fa8545da..fbbecb8c021 100644
--- a/be/test/olap/rowset/segment_v2/inverted_index/token_filter/word_delimiter_filter_test.cpp
+++ b/be/test/olap/rowset/segment_v2/inverted_index/token_filter/word_delimiter_filter_test.cpp
@@ -31,14 +31,14 @@ namespace doris::segment_v2::inverted_index {
TokenStreamPtr create_filter(const std::string& text, int32_t flags,
const std::unordered_set<std::string>& prot_words = {}) {
- static lucene::util::SStringReader<char> reader;
- reader.init(text.data(), text.size(), false);
+ ReaderPtr reader = std::make_shared<lucene::util::SStringReader<char>>();
+ reader->init(text.data(), text.size(), false);
Settings settings;
KeywordTokenizerFactory tokenizer_factory;
tokenizer_factory.initialize(settings);
auto tokenizer = tokenizer_factory.create();
- tokenizer->set_reader(&reader);
+ tokenizer->set_reader(reader);
auto token_filter = std::make_shared<WordDelimiterFilter>(
tokenizer, WordDelimiterIterator::DEFAULT_WORD_DELIM_TABLE, flags, prot_words);
diff --git a/be/test/olap/rowset/segment_v2/inverted_index/tokenizer/basic_tokenizer_factory_test.cpp b/be/test/olap/rowset/segment_v2/inverted_index/tokenizer/basic_tokenizer_factory_test.cpp
new file mode 100644
index 00000000000..4087ab5ab26
--- /dev/null
+++ b/be/test/olap/rowset/segment_v2/inverted_index/tokenizer/basic_tokenizer_factory_test.cpp
@@ -0,0 +1,235 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "olap/rowset/segment_v2/inverted_index/tokenizer/basic/basic_tokenizer_factory.h"
+
+#include <gtest/gtest.h>
+
+namespace doris::segment_v2::inverted_index {
+
+TokenStreamPtr create_basic_tokenizer(const std::string& text, Settings settings = Settings()) {
+ ReaderPtr reader = std::make_shared<lucene::util::SStringReader<char>>();
+ reader->init(text.data(), text.size(), false);
+
+ BasicTokenizerFactory factory;
+ factory.initialize(settings);
+ auto tokenizer = factory.create();
+ tokenizer->set_reader(reader);
+ tokenizer->reset();
+ return tokenizer;
+}
+
+struct ExpectedToken {
+ std::string term;
+ int pos_inc;
+};
+
+class BasicTokenizerFactoryTest : public ::testing::Test {
+protected:
+ void assert_tokenizer_output(const std::string& text,
+ const std::vector<ExpectedToken>& expected,
+ const std::string& extra_chars = "") {
+ Settings settings;
+ if (!extra_chars.empty()) {
+ settings.set("extra_chars", extra_chars);
+ }
+ auto tokenizer = create_basic_tokenizer(text, settings);
+
+ Token t;
+ size_t i = 0;
+ while (tokenizer->next(&t)) {
+ ASSERT_LT(i, expected.size()) << "More tokens produced than expected";
+ std::string term(t.termBuffer<char>(), t.termLength<char>());
+ EXPECT_EQ(term, expected[i].term) << "Term mismatch at index " << i;
+ EXPECT_EQ(t.getPositionIncrement(), expected[i].pos_inc)
+ << "Pos increment mismatch at index " << i;
+ ++i;
+ }
+ EXPECT_EQ(i, expected.size()) << "Number of tokens mismatch";
+ }
+};
+
+TEST_F(BasicTokenizerFactoryTest, BasicTokenization) {
+ // Test basic tokenization: English + numbers + Chinese
+ assert_tokenizer_output("Hello world!", {{"Hello", 1}, {"world", 1}});
+}
+
+TEST_F(BasicTokenizerFactoryTest, ChineseTokenization) {
+ // Test Chinese characters tokenization
+ assert_tokenizer_output("你好世界", {{"你", 1}, {"好", 1}, {"世", 1}, {"界", 1}});
+}
+
+TEST_F(BasicTokenizerFactoryTest, MixedLanguage) {
+ // Test mixed English and Chinese
+ assert_tokenizer_output(
+ "Hello你好World世界",
+ {{"Hello", 1}, {"你", 1}, {"好", 1}, {"World", 1}, {"世", 1}, {"界", 1}});
+}
+
+TEST_F(BasicTokenizerFactoryTest, NumbersAndPunctuation) {
+ // Test with numbers, punctuation is ignored by default
+ assert_tokenizer_output("Version 2.0 版本",
+ {{"Version", 1}, {"2", 1}, {"0", 1}, {"版", 1}, {"本", 1}});
+}
+
+TEST_F(BasicTokenizerFactoryTest, ExtraCharsBasic) {
+ // Test extra_chars: tokenize '.' as separate token
+ assert_tokenizer_output("Version 2.0", {{"Version", 1}, {"2", 1}, {".", 1}, {"0", 1}}, ".");
+}
+
+TEST_F(BasicTokenizerFactoryTest, ExtraCharsMultiple) {
+ // Test multiple extra_chars: '[', ']', '.', '_'
+ assert_tokenizer_output(
+ "test[0].value_1",
+ {{"test", 1}, {"[", 1}, {"0", 1}, {"]", 1}, {".", 1}, {"value", 1}, {"_", 1}, {"1", 1}},
+ "[]._");
+}
+
+TEST_F(BasicTokenizerFactoryTest, ExtraCharsWithChinese) {
+ // Test extra_chars with Chinese text
+ assert_tokenizer_output("测试.数据", {{"测", 1}, {"试", 1}, {".", 1}, {"数", 1}, {"据", 1}},
+ ".");
+}
+
+TEST_F(BasicTokenizerFactoryTest, ExtraCharsSpecialSymbols) {
+ // Test various special symbols as extra_chars
+ assert_tokenizer_output("hello@world#2024",
+ {{"hello", 1}, {"@", 1}, {"world", 1}, {"#", 1}, {"2024", 1}}, "@#");
+}
+
+TEST_F(BasicTokenizerFactoryTest, FactoryInitializationDefault) {
+ // Test default initialization without extra_chars
+ Settings settings;
+
+ BasicTokenizerFactory factory;
+ factory.initialize(settings);
+
+ auto tokenizer = factory.create();
+ auto basic_tokenizer = std::dynamic_pointer_cast<BasicTokenizer>(tokenizer);
+ ASSERT_NE(basic_tokenizer, nullptr);
+}
+
+TEST_F(BasicTokenizerFactoryTest, FactoryInitializationWithExtraChars) {
+ // Test initialization with extra_chars
+ Settings settings;
+ settings.set("extra_chars", "[]._");
+
+ BasicTokenizerFactory factory;
+ factory.initialize(settings);
+
+ auto tokenizer = factory.create();
+ auto basic_tokenizer = std::dynamic_pointer_cast<BasicTokenizer>(tokenizer);
+ ASSERT_NE(basic_tokenizer, nullptr);
+}
+
+TEST_F(BasicTokenizerFactoryTest, EmptyExtraChars) {
+ // Test with empty extra_chars
+ Settings settings;
+ settings.set("extra_chars", "");
+
+ BasicTokenizerFactory factory;
+ EXPECT_NO_THROW(factory.initialize(settings));
+}
+
+TEST_F(BasicTokenizerFactoryTest, InvalidExtraCharsNonASCII) {
+ // Test with non-ASCII characters in extra_chars (should throw)
+ Settings settings;
+ settings.set("extra_chars", "你好"); // Chinese characters
+
+ BasicTokenizerFactory factory;
+ EXPECT_THROW(factory.initialize(settings), Exception);
+}
+
+TEST_F(BasicTokenizerFactoryTest, InvalidExtraCharsMixedASCIIAndNonASCII) {
+ // Test with mixed ASCII and non-ASCII (should throw)
+ Settings settings;
+ settings.set("extra_chars", ".中"); // '.' is ASCII, '中' is not
+
+ BasicTokenizerFactory factory;
+ EXPECT_THROW(factory.initialize(settings), Exception);
+}
+
+TEST_F(BasicTokenizerFactoryTest, ValidASCIIExtraChars) {
+ // Test with various valid ASCII characters
+ Settings settings;
+ settings.set("extra_chars", "!@#$%^&*()_+-=[]{}|;:',.<>?/~`");
+
+ BasicTokenizerFactory factory;
+ EXPECT_NO_THROW(factory.initialize(settings));
+}
+
+TEST_F(BasicTokenizerFactoryTest, EdgeCases) {
+ // Test empty string
+ assert_tokenizer_output("", {});
+
+ // Test whitespace only
+ assert_tokenizer_output(" ", {});
+
+ // Test punctuation only without extra_chars (should be ignored)
+ assert_tokenizer_output("...", {});
+
+ // Test punctuation only with extra_chars (should be tokenized)
+ assert_tokenizer_output("...", {{".", 1}, {".", 1}, {".", 1}}, ".");
+}
+
+TEST_F(BasicTokenizerFactoryTest, LongText) {
+ // Test with longer text
+ std::string long_text = "This is a long text with multiple words and 中文 characters";
+ std::vector<ExpectedToken> expected = {
+ {"This", 1}, {"is", 1}, {"a", 1}, {"long", 1}, {"text", 1}, {"with", 1},
+ {"multiple", 1}, {"words", 1}, {"and", 1}, {"中", 1}, {"文", 1}, {"characters", 1}};
+ assert_tokenizer_output(long_text, expected);
+}
+
+TEST_F(BasicTokenizerFactoryTest, LongTextWithExtraChars) {
+ // Test longer text with extra_chars
+ std::string long_text = "user.name=test_123";
+ std::vector<ExpectedToken> expected = {{"user", 1}, {".", 1}, {"name", 1}, {"=", 1},
+ {"test", 1}, {"_", 1}, {"123", 1}};
+ assert_tokenizer_output(long_text, expected, "._=");
+}
+
+TEST_F(BasicTokenizerFactoryTest, ExtraCharsWithAlphanumeric) {
+ assert_tokenizer_output("[123.456_7890abcd]",
+ {{.term = "[", .pos_inc = 1},
+ {.term = "123", .pos_inc = 1},
+ {.term = ".", .pos_inc = 1},
+ {.term = "456", .pos_inc = 1},
+ {.term = "_", .pos_inc = 1},
+ {.term = "7890abcd", .pos_inc = 1},
+ {.term = "]", .pos_inc = 1}},
+ "[]._");
+}
+
+TEST_F(BasicTokenizerFactoryTest, ExtraCharsShouldNotIncludeAlphanumeric) {
+ assert_tokenizer_output("Found OpenMP: TRUE (found version \"5.1\")",
+ {{.term = "Found", .pos_inc = 1},
+ {.term = "OpenMP", .pos_inc = 1},
+ {.term = "TRUE", .pos_inc = 1},
+ {.term = "(", .pos_inc = 1},
+ {.term = "found", .pos_inc = 1},
+ {.term = "version", .pos_inc = 1},
+ {.term = "\"", .pos_inc = 1},
+ {.term = "5", .pos_inc = 1},
+ {.term = ".", .pos_inc = 1},
+ {.term = "1", .pos_inc = 1},
+ {.term = "\"", .pos_inc = 1},
+ {.term = ")", .pos_inc = 1}},
+ "()\".");
+}
+
+} // namespace doris::segment_v2::inverted_index
\ No newline at end of file
diff --git a/be/test/olap/rowset/segment_v2/inverted_index/tokenizer/char_group_tokenizer_factory_test.cpp b/be/test/olap/rowset/segment_v2/inverted_index/tokenizer/char_group_tokenizer_factory_test.cpp
index 7d2782098a1..91d978f83fa 100644
--- a/be/test/olap/rowset/segment_v2/inverted_index/tokenizer/char_group_tokenizer_factory_test.cpp
+++ b/be/test/olap/rowset/segment_v2/inverted_index/tokenizer/char_group_tokenizer_factory_test.cpp
@@ -29,9 +29,9 @@ protected:
std::vector<std::string> tokens;
auto tokenizer = factory.create();
{
- lucene::util::SStringReader<char> reader;
- reader.init(text.data(), text.size(), false);
- tokenizer->set_reader(&reader);
+ ReaderPtr reader = std::make_shared<lucene::util::SStringReader<char>>();
+ reader->init(text.data(), text.size(), false);
+ tokenizer->set_reader(reader);
tokenizer->reset();
Token t;
diff --git a/be/test/olap/rowset/segment_v2/inverted_index/tokenizer/edge_ngram_tokenizer_test.cpp b/be/test/olap/rowset/segment_v2/inverted_index/tokenizer/edge_ngram_tokenizer_test.cpp
index 0d9f1802952..00b22b48449 100644
--- a/be/test/olap/rowset/segment_v2/inverted_index/tokenizer/edge_ngram_tokenizer_test.cpp
+++ b/be/test/olap/rowset/segment_v2/inverted_index/tokenizer/edge_ngram_tokenizer_test.cpp
@@ -33,9 +33,9 @@ std::vector<std::string> tokenize(EdgeNGramTokenizerFactory& factory, const std:
std::vector<std::string> tokens;
auto tokenizer = factory.create();
{
- lucene::util::SStringReader<char> reader;
- reader.init(text.data(), text.size(), false);
- tokenizer->set_reader(&reader);
+ ReaderPtr reader = std::make_shared<lucene::util::SStringReader<char>>();
+ reader->init(text.data(), text.size(), false);
+ tokenizer->set_reader(reader);
tokenizer->reset();
Token t;
diff --git a/be/test/olap/rowset/segment_v2/inverted_index/tokenizer/icu_tokenizer_factory_test.cpp b/be/test/olap/rowset/segment_v2/inverted_index/tokenizer/icu_tokenizer_factory_test.cpp
new file mode 100644
index 00000000000..fb736ecaa76
--- /dev/null
+++ b/be/test/olap/rowset/segment_v2/inverted_index/tokenizer/icu_tokenizer_factory_test.cpp
@@ -0,0 +1,212 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "olap/rowset/segment_v2/inverted_index/tokenizer/icu/icu_tokenizer_factory.h"
+
+#include <gtest/gtest.h>
+
+namespace doris::segment_v2::inverted_index {
+
+TokenStreamPtr create_icu_tokenizer(const std::string& text, Settings settings = Settings()) {
+ ReaderPtr reader = std::make_shared<lucene::util::SStringReader<char>>();
+ reader->init(text.data(), text.size(), false);
+
+ ICUTokenizerFactory factory;
+ factory.initialize(settings);
+ auto tokenizer = factory.create();
+ tokenizer->set_reader(reader);
+ tokenizer->reset();
+ return tokenizer;
+}
+
+struct ExpectedToken {
+ std::string term;
+ int pos_inc;
+};
+
+class ICUTokenizerFactoryTest : public ::testing::Test {
+protected:
+ void SetUp() override {
+ original_dict_path_ = config::inverted_index_dict_path;
+
+ constexpr static uint32_t MAX_PATH_LEN = 1024;
+ char buffer[MAX_PATH_LEN];
+ EXPECT_NE(getcwd(buffer, MAX_PATH_LEN), nullptr);
+ std::string current_dir = std::string(buffer);
+
+ config::inverted_index_dict_path = current_dir + "/be/dict";
+ }
+
+ void TearDown() override { config::inverted_index_dict_path = original_dict_path_; }
+
+ void assert_tokenizer_output(const std::string& text,
+ const std::vector<ExpectedToken>& expected) {
+ auto tokenizer = create_icu_tokenizer(text);
+
+ Token t;
+ size_t i = 0;
+ while (tokenizer->next(&t)) {
+ ASSERT_LT(i, expected.size()) << "More tokens produced than expected";
+ std::string term(t.termBuffer<char>(), t.termLength<char>());
+ EXPECT_EQ(term, expected[i].term) << "Term mismatch at index " << i;
+ EXPECT_EQ(t.getPositionIncrement(), expected[i].pos_inc)
+ << "Pos increment mismatch at index " << i;
+ ++i;
+ }
+ EXPECT_EQ(i, expected.size()) << "Number of tokens mismatch";
+ }
+
+private:
+ std::string original_dict_path_;
+};
+
+TEST_F(ICUTokenizerFactoryTest, BasicEnglishTokenization) {
+ // Test basic English tokenization
+ assert_tokenizer_output("Hello world!", {{"Hello", 1}, {"world", 1}});
+}
+
+TEST_F(ICUTokenizerFactoryTest, EnglishWithPunctuation) {
+ // Test English with punctuation
+ assert_tokenizer_output(
+ "This is a test, with punctuation!",
+ {{"This", 1}, {"is", 1}, {"a", 1}, {"test", 1}, {"with", 1}, {"punctuation", 1}});
+}
+
+TEST_F(ICUTokenizerFactoryTest, EnglishWithNumbers) {
+ // Test English with numbers
+ assert_tokenizer_output(
+ "Version 2.0 was released in 2023",
+ {{"Version", 1}, {"2.0", 1}, {"was", 1}, {"released", 1}, {"in", 1}, {"2023", 1}});
+}
+
+TEST_F(ICUTokenizerFactoryTest, ChineseTokenization) {
+ // Test Chinese character tokenization
+ assert_tokenizer_output("你好世界", {{"你好", 1}, {"世界", 1}});
+}
+
+TEST_F(ICUTokenizerFactoryTest, MixedLanguage) {
+ // Test mixed English and Chinese
+ assert_tokenizer_output("Hello你好World世界",
+ {{"Hello", 1}, {"你好", 1}, {"World", 1}, {"世界", 1}});
+}
+
+TEST_F(ICUTokenizerFactoryTest, JapaneseTokenization) {
+ // Test Japanese tokenization (Hiragana, Katakana, Kanji)
+ assert_tokenizer_output("こんにちは世界", {{"こんにちは", 1}, {"世界", 1}});
+}
+
+TEST_F(ICUTokenizerFactoryTest, KoreanTokenization) {
+ // Test Korean tokenization
+ assert_tokenizer_output("안녕하세요", {{"안녕하세요", 1}});
+}
+
+TEST_F(ICUTokenizerFactoryTest, ArabicTokenization) {
+ // Test Arabic tokenization
+ assert_tokenizer_output("مرحبا بالعالم", {{"مرحبا", 1}, {"بالعالم", 1}});
+}
+
+TEST_F(ICUTokenizerFactoryTest, CyrillicTokenization) {
+ // Test Cyrillic (Russian) tokenization
+ assert_tokenizer_output("Привет мир", {{"Привет", 1}, {"мир", 1}});
+}
+
+TEST_F(ICUTokenizerFactoryTest, EmojiAndSymbols) {
+ // Test emoji and special symbols
+ assert_tokenizer_output("Hello 😀 world 🌍",
+ {{"Hello", 1}, {"😀", 1}, {"world", 1}, {"🌍", 1}});
+}
+
+TEST_F(ICUTokenizerFactoryTest, WhitespaceHandling) {
+ // Test whitespace handling
+ assert_tokenizer_output("Hello world\t\n", {{"Hello", 1}, {"world", 1}});
+}
+
+TEST_F(ICUTokenizerFactoryTest, FactoryInitialization) {
+ Settings settings;
+ ICUTokenizerFactory factory;
+ factory.initialize(settings);
+
+ auto tokenizer = factory.create();
+ auto icu_tokenizer = std::dynamic_pointer_cast<ICUTokenizer>(tokenizer);
+ ASSERT_NE(icu_tokenizer, nullptr);
+}
+
+TEST_F(ICUTokenizerFactoryTest, FactoryCreateMultipleInstances) {
+ ICUTokenizerFactory factory;
+ factory.initialize(Settings {});
+
+ auto tokenizer1 = factory.create();
+ auto tokenizer2 = factory.create();
+
+ ASSERT_NE(tokenizer1, tokenizer2);
+ ASSERT_NE(tokenizer1, nullptr);
+ ASSERT_NE(tokenizer2, nullptr);
+}
+
+TEST_F(ICUTokenizerFactoryTest, EdgeCases) {
+ // Test empty string
+ assert_tokenizer_output("", {});
+
+ // Test whitespace only
+ assert_tokenizer_output(" \t\n", {});
+
+ // Test single character
+ assert_tokenizer_output("a", {{"a", 1}});
+
+ // Test single Chinese character
+ assert_tokenizer_output("中", {{"中", 1}});
+}
+
+TEST_F(ICUTokenizerFactoryTest, LongText) {
+ // Test with longer text
+ std::string long_text =
+ "This is a long text with multiple words and 中文 characters and 日本語 text";
+ std::vector<ExpectedToken> expected = {
+ {"This", 1}, {"is", 1}, {"a", 1}, {"long", 1}, {"text", 1},
+ {"with", 1}, {"multiple", 1}, {"words", 1}, {"and", 1}, {"中文", 1},
+ {"characters", 1}, {"and", 1}, {"日本語", 1}, {"text", 1}};
+ assert_tokenizer_output(long_text, expected);
+}
+
+TEST_F(ICUTokenizerFactoryTest, SpecialCharacters) {
+ // Test special characters and symbols
+ assert_tokenizer_output("Price: $100.50 (USD)", {{"Price", 1}, {"100.50", 1}, {"USD", 1}});
+}
+
+TEST_F(ICUTokenizerFactoryTest, URLAndEmail) {
+ // Test URL and email handling
+ assert_tokenizer_output("Visit https://example.com or email test@example.com",
+ {{"Visit", 1},
+ {"https", 1},
+ {"example.com", 1},
+ {"or", 1},
+ {"email", 1},
+ {"test", 1},
+ {"example.com", 1}});
+}
+
+TEST_F(ICUTokenizerFactoryTest, CaseSensitivity) {
+ // Test case sensitivity (ICU tokenizer should preserve case by default)
+ assert_tokenizer_output("Hello WORLD Test", {{"Hello", 1}, {"WORLD", 1}, {"Test", 1}});
+}
+
+TEST_F(ICUTokenizerFactoryTest, UnicodeNormalization) {
+ // Test Unicode normalization
+ assert_tokenizer_output("café naïve résumé", {{"café", 1}, {"naïve", 1}, {"résumé", 1}});
+}
+
+} // namespace doris::segment_v2::inverted_index
\ No newline at end of file
diff --git a/be/test/olap/rowset/segment_v2/inverted_index/tokenizer/keyword_analyzer_test.cpp b/be/test/olap/rowset/segment_v2/inverted_index/tokenizer/keyword_analyzer_test.cpp
index 1a8283fd1ac..0b334b1492d 100644
--- a/be/test/olap/rowset/segment_v2/inverted_index/tokenizer/keyword_analyzer_test.cpp
+++ b/be/test/olap/rowset/segment_v2/inverted_index/tokenizer/keyword_analyzer_test.cpp
@@ -32,9 +32,9 @@ std::vector<std::string> tokenize(KeywordTokenizerFactory& factory, const std::s
std::vector<std::string> tokens;
auto tokenizer = factory.create();
{
- lucene::util::SStringReader<char> reader;
- reader.init(text.data(), text.size(), false);
- tokenizer->set_reader(&reader);
+ ReaderPtr reader = std::make_shared<lucene::util::SStringReader<char>>();
+ reader->init(text.data(), text.size(), false);
+ tokenizer->set_reader(reader);
tokenizer->reset();
Token t;
diff --git a/be/test/olap/rowset/segment_v2/inverted_index/tokenizer/ngram_tokenizer_test.cpp b/be/test/olap/rowset/segment_v2/inverted_index/tokenizer/ngram_tokenizer_test.cpp
index 9b55aae48aa..2249a45d4d3 100644
--- a/be/test/olap/rowset/segment_v2/inverted_index/tokenizer/ngram_tokenizer_test.cpp
+++ b/be/test/olap/rowset/segment_v2/inverted_index/tokenizer/ngram_tokenizer_test.cpp
@@ -35,9 +35,9 @@ std::vector<std::string> tokenize(NGramTokenizerFactory& factory, const std::str
std::vector<std::string> tokens;
auto tokenizer = factory.create();
{
- lucene::util::SStringReader<char> reader;
- reader.init(text.data(), text.size(), false);
- tokenizer->set_reader(&reader);
+ ReaderPtr reader = std::make_shared<lucene::util::SStringReader<char>>();
+ reader->init(text.data(), text.size(), false);
+ tokenizer->set_reader(reader);
tokenizer->reset();
Token t;
diff --git a/be/test/olap/rowset/segment_v2/inverted_index/tokenizer/standard_tokenizer_factory_test.cpp b/be/test/olap/rowset/segment_v2/inverted_index/tokenizer/standard_tokenizer_factory_test.cpp
index 5cbc5689096..631e95745bb 100644
--- a/be/test/olap/rowset/segment_v2/inverted_index/tokenizer/standard_tokenizer_factory_test.cpp
+++ b/be/test/olap/rowset/segment_v2/inverted_index/tokenizer/standard_tokenizer_factory_test.cpp
@@ -22,13 +22,13 @@
namespace doris::segment_v2::inverted_index {
TokenStreamPtr create_standard_tokenizer(const std::string& text, Settings settings = Settings()) {
- static lucene::util::SStringReader<char> reader;
- reader.init(text.data(), text.size(), false);
+ ReaderPtr reader = std::make_shared<lucene::util::SStringReader<char>>();
+ reader->init(text.data(), text.size(), false);
StandardTokenizerFactory factory;
factory.initialize(settings);
auto tokenizer = factory.create();
- tokenizer->set_reader(&reader);
+ tokenizer->set_reader(reader);
tokenizer->reset();
return tokenizer;
}
diff --git a/be/test/olap/rowset/segment_v2/inverted_index/util/reader_test.cpp b/be/test/olap/rowset/segment_v2/inverted_index/util/reader_test.cpp
new file mode 100644
index 00000000000..36cc52fdb95
--- /dev/null
+++ b/be/test/olap/rowset/segment_v2/inverted_index/util/reader_test.cpp
@@ -0,0 +1,92 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "olap/rowset/segment_v2/inverted_index/util/reader.h"
+
+#include <gtest/gtest.h>
+
+#include <memory>
+#include <string>
+
+#include "common/cast_set.h"
+#include "olap/rowset/segment_v2/inverted_index/analyzer/analyzer.h"
+#include "olap/rowset/segment_v2/inverted_index_common.h"
+#include "util/slice.h"
+
+using namespace lucene::analysis;
+using namespace doris::segment_v2::inverted_index;
+
+namespace doris::segment_v2 {
+
+TEST(ReaderTest, ArrayFieldTokenStreamWorkflow) {
+ CharFilterMap char_filter_map;
+ char_filter_map["char_filter_type"] = "char_replace";
+ char_filter_map["char_filter_pattern"] = ",";
+ char_filter_map["char_filter_replacement"] = " ";
+
+ auto inverted_index_ctx = std::make_shared<InvertedIndexCtx>();
+ inverted_index_ctx->custom_analyzer = "";
+ inverted_index_ctx->parser_type = InvertedIndexParserType::PARSER_STANDARD;
+ inverted_index_ctx->parser_mode = "standard";
+ inverted_index_ctx->support_phrase = "yes";
+ inverted_index_ctx->char_filter_map = char_filter_map;
+ inverted_index_ctx->lower_case = "true";
+ inverted_index_ctx->stop_words = "";
+
+ auto analyzer = InvertedIndexAnalyzer::create_analyzer(inverted_index_ctx.get());
+ ASSERT_NE(analyzer, nullptr);
+
+ std::string test_data = "hello,world,test";
+ Slice slice(test_data);
+
+ std::vector<ReaderPtr> keep_readers;
+ auto dir = std::make_shared<lucene::store::RAMDirectory>();
+ {
+ lucene::index::IndexWriter indexwriter(dir.get(), analyzer.get(), true);
+ indexwriter.setRAMBufferSizeMB(512);
+ indexwriter.setMaxFieldLength(0x7FFFFFFFL);
+ indexwriter.setMergeFactor(1000000000);
+ indexwriter.setUseCompoundFile(false);
+ lucene::document::Document doc;
+ std::unique_ptr<lucene::document::Field> new_field;
+ for (int i = 0; i < 2; i++) {
+ int32_t field_config = lucene::document::Field::STORE_NO;
+ field_config |= lucene::document::Field::INDEX_NONORMS;
+ field_config |= lucene::document::Field::INDEX_TOKENIZED;
+ auto* field = _CLNEW lucene::document::Field(L"name", field_config);
+ new_field.reset(field);
+ {
+ ReaderPtr char_string_reader =
+ InvertedIndexAnalyzer::create_reader(inverted_index_ctx->char_filter_map);
+ char_string_reader->init(slice.get_data(), cast_set<int32_t>(slice.get_size()),
+ false);
+
+ auto* ts = analyzer->tokenStream(new_field->name(), char_string_reader);
+ ASSERT_NE(ts, nullptr);
+
+ new_field->setValue(ts, true);
+ keep_readers.emplace_back(std::move(char_string_reader));
+ }
+ doc.add(*new_field.release());
+ }
+ indexwriter.addDocument(&doc);
+ indexwriter.close();
+ }
+ dir->close();
+}
+
+} // namespace doris::segment_v2
\ No newline at end of file
diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4
index 06c6ac6a082..c2aff94c461 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4
@@ -139,6 +139,7 @@ CATALOGS: 'CATALOGS';
CHAIN: 'CHAIN';
CHAR: 'CHAR' | 'CHARACTER';
CHARSET: 'CHARSET';
+CHAR_FILTER: 'CHAR_FILTER';
CHECK: 'CHECK';
CLEAN: 'CLEAN';
CLUSTER: 'CLUSTER';
diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index 3b81a902670..8c146785a6c 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -195,6 +195,8 @@ supportedCreateStatement
name=identifier properties=propertyClause? #createIndexTokenizer
| CREATE INVERTED INDEX TOKEN_FILTER (IF NOT EXISTS)?
name=identifier properties=propertyClause? #createIndexTokenFilter
+ | CREATE INVERTED INDEX CHAR_FILTER (IF NOT EXISTS)?
+ name=identifier properties=propertyClause? #createIndexCharFilter
;
supportedAlterStatement
@@ -212,6 +214,7 @@ supportedDropStatement
| DROP INVERTED INDEX ANALYZER (IF EXISTS)? name=identifier #dropIndexAnalyzer
| DROP INVERTED INDEX TOKENIZER (IF EXISTS)? name=identifier #dropIndexTokenizer
| DROP INVERTED INDEX TOKEN_FILTER (IF EXISTS)? name=identifier #dropIndexTokenFilter
+ | DROP INVERTED INDEX CHAR_FILTER (IF EXISTS)? name=identifier #dropIndexCharFilter
;
supportedShowStatement
@@ -221,6 +224,7 @@ supportedShowStatement
| SHOW INVERTED INDEX ANALYZER #showIndexAnalyzer
| SHOW INVERTED INDEX TOKENIZER #showIndexTokenizer
| SHOW INVERTED INDEX TOKEN_FILTER #showIndexTokenFilter
+ | SHOW INVERTED INDEX CHAR_FILTER #showIndexCharFilter
;
supportedKillStatement
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java
index a1df981898e..434fe52f495 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java
@@ -266,7 +266,9 @@ public class IndexDef {
+ " or key columns of AGG_KEYS table. invalid index: " + indexName);
} else if (keysType == KeysType.UNIQUE_KEYS && !enableUniqueKeyMergeOnWrite
&& indexType == IndexType.INVERTED && properties != null
- && properties.containsKey(InvertedIndexUtil.INVERTED_INDEX_PARSER_KEY)) {
+ && (properties.containsKey(InvertedIndexUtil.INVERTED_INDEX_PARSER_KEY)
+ || properties.containsKey(InvertedIndexUtil.INVERTED_INDEX_PARSER_KEY_ALIAS)
+ || properties.containsKey(InvertedIndexUtil.INVERTED_INDEX_CUSTOM_ANALYZER_KEY))) {
throw new AnalysisException("INVERTED index with parser can NOT be used in value columns of"
+ " UNIQUE_KEYS table with merge_on_write disable. invalid index: " + indexName);
}
@@ -314,6 +316,7 @@ public class IndexDef {
return indexType == IndexDef.IndexType.INVERTED
&& properties != null
&& (properties.containsKey(InvertedIndexUtil.INVERTED_INDEX_PARSER_KEY)
+ || properties.containsKey(InvertedIndexUtil.INVERTED_INDEX_PARSER_KEY_ALIAS)
|| properties.containsKey(InvertedIndexUtil.INVERTED_INDEX_CUSTOM_ANALYZER_KEY));
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/InvertedIndexUtil.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/InvertedIndexUtil.java
index c66e485ed9e..1bb85a287e4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/InvertedIndexUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/InvertedIndexUtil.java
@@ -35,6 +35,7 @@ import java.util.Set;
public class InvertedIndexUtil {
public static String INVERTED_INDEX_PARSER_KEY = "parser";
+ public static String INVERTED_INDEX_PARSER_KEY_ALIAS = "built_in_analyzer";
public static String INVERTED_INDEX_PARSER_UNKNOWN = "unknown";
public static String INVERTED_INDEX_PARSER_NONE = "none";
public static String INVERTED_INDEX_PARSER_STANDARD = "standard";
@@ -72,11 +73,27 @@ public class InvertedIndexUtil {
public static String INVERTED_INDEX_PARSER_FIELD_PATTERN_KEY = "field_pattern";
public static String getInvertedIndexParser(Map<String, String> properties) {
- String parser = properties == null ? null : properties.get(INVERTED_INDEX_PARSER_KEY);
- // default is "none" if not set
+ if (properties == null) {
+ return INVERTED_INDEX_PARSER_NONE;
+ }
+ String parser = properties.get(INVERTED_INDEX_PARSER_KEY);
+ if (parser == null) {
+ parser = properties.get(INVERTED_INDEX_PARSER_KEY_ALIAS);
+ }
return parser != null ? parser : INVERTED_INDEX_PARSER_NONE;
}
+ public static String getInvertedIndexParserMode(Map<String, String> properties) {
+ String mode = properties == null ? null : properties.get(INVERTED_INDEX_PARSER_MODE_KEY);
+ String parser = properties.get(INVERTED_INDEX_PARSER_KEY);
+ if (parser == null) {
+ parser = properties.get(INVERTED_INDEX_PARSER_KEY_ALIAS);
+ }
+ return mode != null ? mode :
+ INVERTED_INDEX_PARSER_IK.equals(parser) ? INVERTED_INDEX_PARSER_SMART :
+ INVERTED_INDEX_PARSER_COARSE_GRANULARITY;
+ }
+
public static String getInvertedIndexFieldPattern(Map<String, String> properties) {
String fieldPattern = properties == null ? null : properties.get(INVERTED_INDEX_PARSER_FIELD_PATTERN_KEY);
// default is "none" if not set
@@ -93,15 +110,6 @@ public class InvertedIndexUtil {
return customAnalyzer != null ? customAnalyzer : "";
}
- public static String getInvertedIndexParserMode(Map<String, String> properties) {
- String mode = properties == null ? null : properties.get(INVERTED_INDEX_PARSER_MODE_KEY);
- // default is "none" if not set
- String parser = properties.get(INVERTED_INDEX_PARSER_KEY);
- return mode != null ? mode :
- INVERTED_INDEX_PARSER_IK.equals(parser) ? INVERTED_INDEX_PARSER_SMART :
- INVERTED_INDEX_PARSER_COARSE_GRANULARITY;
- }
-
public static Map<String, String> getInvertedIndexCharFilter(Map<String, String> properties) {
if (properties == null) {
return new HashMap<>();
@@ -161,7 +169,9 @@ public class InvertedIndexUtil {
String analyzer = null;
if (properties != null) {
parser = properties.get(INVERTED_INDEX_PARSER_KEY);
- analyzer = properties.get(INVERTED_INDEX_CUSTOM_ANALYZER_KEY);
+ if (parser == null) {
+ parser = properties.get(INVERTED_INDEX_PARSER_KEY_ALIAS);
+ }
checkInvertedIndexProperties(properties, colType, invertedIndexFileStorageFormat);
}
@@ -199,6 +209,7 @@ public class InvertedIndexUtil {
TInvertedIndexFileStorageFormat invertedIndexFileStorageFormat) throws AnalysisException {
Set<String> allowedKeys = new HashSet<>(Arrays.asList(
INVERTED_INDEX_PARSER_KEY,
+ INVERTED_INDEX_PARSER_KEY_ALIAS,
INVERTED_INDEX_PARSER_MODE_KEY,
INVERTED_INDEX_SUPPORT_PHRASE_KEY,
INVERTED_INDEX_PARSER_CHAR_FILTER_TYPE,
@@ -219,6 +230,9 @@ public class InvertedIndexUtil {
}
String parser = properties.get(INVERTED_INDEX_PARSER_KEY);
+ if (parser == null) {
+ parser = properties.get(INVERTED_INDEX_PARSER_KEY_ALIAS);
+ }
String parserMode = properties.get(INVERTED_INDEX_PARSER_MODE_KEY);
String supportPhrase = properties.get(INVERTED_INDEX_SUPPORT_PHRASE_KEY);
String charFilterType = properties.get(INVERTED_INDEX_PARSER_CHAR_FILTER_TYPE);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java
index a4a78706394..03a3c051bf5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java
@@ -77,16 +77,21 @@ public class Index implements Writable {
this.comment = comment;
if (indexType == IndexDef.IndexType.INVERTED) {
if (this.properties != null && !this.properties.isEmpty()) {
- if (this.properties.containsKey(InvertedIndexUtil.INVERTED_INDEX_PARSER_KEY)) {
- String lowerCaseKey = InvertedIndexUtil.INVERTED_INDEX_PARSER_LOWERCASE_KEY;
- if (!properties.containsKey(lowerCaseKey)) {
- this.properties.put(lowerCaseKey, "true");
- }
+ if (this.properties.containsKey(InvertedIndexUtil.INVERTED_INDEX_PARSER_KEY)
+ || this.properties.containsKey(InvertedIndexUtil.INVERTED_INDEX_PARSER_KEY_ALIAS)
+ || this.properties.containsKey(InvertedIndexUtil.INVERTED_INDEX_CUSTOM_ANALYZER_KEY)) {
String supportPhraseKey = InvertedIndexUtil
.INVERTED_INDEX_SUPPORT_PHRASE_KEY;
if (!properties.containsKey(supportPhraseKey)) {
this.properties.put(supportPhraseKey, "true");
}
+ if (this.properties.containsKey(InvertedIndexUtil.INVERTED_INDEX_PARSER_KEY)
+ || this.properties.containsKey(InvertedIndexUtil.INVERTED_INDEX_PARSER_KEY_ALIAS)) {
+ String lowerCaseKey = InvertedIndexUtil.INVERTED_INDEX_PARSER_LOWERCASE_KEY;
+ if (!this.properties.containsKey(lowerCaseKey)) {
+ this.properties.put(lowerCaseKey, "true");
+ }
+ }
}
}
}
@@ -372,6 +377,7 @@ public class Index implements Writable {
return indexType == IndexDef.IndexType.INVERTED
&& properties != null
&& (properties.containsKey(InvertedIndexUtil.INVERTED_INDEX_PARSER_KEY)
+ || properties.containsKey(InvertedIndexUtil.INVERTED_INDEX_PARSER_KEY_ALIAS)
|| properties.containsKey(InvertedIndexUtil.INVERTED_INDEX_CUSTOM_ANALYZER_KEY));
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/indexpolicy/KeywordTokenizerValidator.java b/fe/fe-core/src/main/java/org/apache/doris/indexpolicy/BasicTokenizerValidator.java
similarity index 59%
copy from fe/fe-core/src/main/java/org/apache/doris/indexpolicy/KeywordTokenizerValidator.java
copy to fe/fe-core/src/main/java/org/apache/doris/indexpolicy/BasicTokenizerValidator.java
index 1a9ab2865b2..23e06be15c6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/indexpolicy/KeywordTokenizerValidator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/indexpolicy/BasicTokenizerValidator.java
@@ -24,20 +24,33 @@ import com.google.common.collect.ImmutableSet;
import java.util.Map;
import java.util.Set;
-public class KeywordTokenizerValidator extends BasePolicyValidator {
- private static final Set<String> ALLOWED_PROPS = ImmutableSet.of("type");
+public class BasicTokenizerValidator extends BasePolicyValidator {
+ private static final Set<String> ALLOWED_PROPS = ImmutableSet.of("type", "extra_chars");
- public KeywordTokenizerValidator() {
+ public BasicTokenizerValidator() {
super(ALLOWED_PROPS);
}
@Override
protected String getTypeName() {
- return "keyword tokenizer";
+ return "basic tokenizer";
}
@Override
protected void validateSpecific(Map<String, String> props) throws DdlException {
- // No parameters to validate
+ if (props.containsKey("extra_chars")) {
+ String extraChars = props.get("extra_chars");
+ if (extraChars != null && !extraChars.isEmpty()) {
+ for (int i = 0; i < extraChars.length(); i++) {
+ char c = extraChars.charAt(i);
+ if (c > 127) {
+ throw new DdlException("Invalid extra_chars for basic tokenizer: "
+ + "contains non-ASCII character '"
+ + c + "' at position " + i
+ + ". Only ASCII characters (0-127) are allowed.");
+ }
+ }
+ }
+ }
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/indexpolicy/KeywordTokenizerValidator.java b/fe/fe-core/src/main/java/org/apache/doris/indexpolicy/CharReplaceCharFilterValidator.java
similarity index 53%
copy from fe/fe-core/src/main/java/org/apache/doris/indexpolicy/KeywordTokenizerValidator.java
copy to fe/fe-core/src/main/java/org/apache/doris/indexpolicy/CharReplaceCharFilterValidator.java
index 1a9ab2865b2..2e7fe15b2a2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/indexpolicy/KeywordTokenizerValidator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/indexpolicy/CharReplaceCharFilterValidator.java
@@ -24,20 +24,40 @@ import com.google.common.collect.ImmutableSet;
import java.util.Map;
import java.util.Set;
-public class KeywordTokenizerValidator extends BasePolicyValidator {
- private static final Set<String> ALLOWED_PROPS = ImmutableSet.of("type");
+public class CharReplaceCharFilterValidator extends BasePolicyValidator {
+ private static final Set<String> ALLOWED_PROPS = ImmutableSet.of(
+ "type", "pattern", "replacement");
- public KeywordTokenizerValidator() {
+ public CharReplaceCharFilterValidator() {
super(ALLOWED_PROPS);
}
@Override
protected String getTypeName() {
- return "keyword tokenizer";
+ return "char_replace filter";
}
@Override
protected void validateSpecific(Map<String, String> props) throws DdlException {
- // No parameters to validate
+ if (props.containsKey("pattern")) {
+ String pattern = props.get("pattern");
+ if (pattern != null && !pattern.isEmpty()) {
+ for (int i = 0; i < pattern.length(); i++) {
+ if (pattern.charAt(i) > 255) {
+ throw new DdlException(
+ "pattern must contain only single-byte characters in [0,255]");
+ }
+ }
+ }
+ }
+ if (props.containsKey("replacement")) {
+ String replacement = props.get("replacement");
+ if (replacement == null || replacement.length() != 1) {
+ throw new DdlException("replacement must be exactly one byte");
+ }
+ if (replacement.charAt(0) > 255) {
+ throw new DdlException("replacement must be in [0,255]");
+ }
+ }
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/indexpolicy/LowerCaseTokenFilterValidator.java b/fe/fe-core/src/main/java/org/apache/doris/indexpolicy/ICUTokenizerValidator.java
similarity index 83%
rename from fe/fe-core/src/main/java/org/apache/doris/indexpolicy/LowerCaseTokenFilterValidator.java
rename to fe/fe-core/src/main/java/org/apache/doris/indexpolicy/ICUTokenizerValidator.java
index 02adb9b505d..cb725486357 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/indexpolicy/LowerCaseTokenFilterValidator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/indexpolicy/ICUTokenizerValidator.java
@@ -24,20 +24,21 @@ import com.google.common.collect.ImmutableSet;
import java.util.Map;
import java.util.Set;
-public class LowerCaseTokenFilterValidator extends BasePolicyValidator {
+public class ICUTokenizerValidator extends BasePolicyValidator {
private static final Set<String> ALLOWED_PROPS = ImmutableSet.of("type");
- public LowerCaseTokenFilterValidator() {
+ public ICUTokenizerValidator() {
super(ALLOWED_PROPS);
}
@Override
protected String getTypeName() {
- return "lowercase filter";
+ return "icu tokenizer";
}
@Override
protected void validateSpecific(Map<String, String> props) throws DdlException {
- // No parameters to validate
+ // ICU tokenizer doesn't have additional parameters to validate
+ // It uses default ICU configuration
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/indexpolicy/IndexPolicy.java b/fe/fe-core/src/main/java/org/apache/doris/indexpolicy/IndexPolicy.java
index 74fa1c7f8a8..7abfe53ff4e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/indexpolicy/IndexPolicy.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/indexpolicy/IndexPolicy.java
@@ -56,12 +56,16 @@ public class IndexPolicy implements Writable, GsonPostProcessable {
public static final String PROP_ANALYZER = "analyzer";
public static final String PROP_TOKENIZER = "tokenizer";
public static final String PROP_TOKEN_FILTER = "token_filter";
+ public static final String PROP_CHAR_FILTER = "char_filter";
public static final Set<String> BUILTIN_TOKENIZERS = ImmutableSet.of(
- "ngram", "edge_ngram", "keyword", "standard", "char_group");
+ "empty", "ngram", "edge_ngram", "keyword", "standard", "char_group", "basic", "icu");
public static final Set<String> BUILTIN_TOKEN_FILTERS = ImmutableSet.of(
- "asciifolding", "word_delimiter", "lowercase");
+ "empty", "asciifolding", "word_delimiter", "lowercase");
+
+ public static final Set<String> BUILTIN_CHAR_FILTERS = ImmutableSet.of(
+ "empty", "char_replace");
private static final Logger LOG = LogManager.getLogger(IndexPolicy.class);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/indexpolicy/IndexPolicyMgr.java b/fe/fe-core/src/main/java/org/apache/doris/indexpolicy/IndexPolicyMgr.java
index d3be0f48f85..04205605669 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/indexpolicy/IndexPolicyMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/indexpolicy/IndexPolicyMgr.java
@@ -112,6 +112,9 @@ public class IndexPolicyMgr implements Writable, GsonPostProcessable {
if (IndexPolicy.BUILTIN_TOKEN_FILTERS.contains(policyName)) {
throw new DdlException("Policy name '" + policyName + "' conflicts with built-in token filter name");
}
+ if (IndexPolicy.BUILTIN_CHAR_FILTERS.contains(policyName)) {
+ throw new DdlException("Policy name '" + policyName + "' conflicts with built-in char filter name");
+ }
IndexPolicy indexPolicy = IndexPolicy.create(policyName, type, properties);
@@ -173,6 +176,9 @@ public class IndexPolicyMgr implements Writable, GsonPostProcessable {
case TOKEN_FILTER:
validateTokenFilterProperties(properties);
break;
+ case CHAR_FILTER:
+ validateCharFilterProperties(properties);
+ break;
default:
throw new DdlException("Unknown index policy type: " + type);
}
@@ -181,10 +187,11 @@ public class IndexPolicyMgr implements Writable, GsonPostProcessable {
private void validateAnalyzerProperties(Map<String, String> properties) throws DdlException {
for (String key : properties.keySet()) {
if (!key.equals(IndexPolicy.PROP_TOKENIZER)
- && !key.equals(IndexPolicy.PROP_TOKEN_FILTER)) {
+ && !key.equals(IndexPolicy.PROP_TOKEN_FILTER)
+ && !key.equals(IndexPolicy.PROP_CHAR_FILTER)) {
throw new DdlException("Invalid analyzer property: '" + key + "'. Only '"
+ IndexPolicy.PROP_TOKENIZER + "' and '" + IndexPolicy.PROP_TOKEN_FILTER
- + "' are allowed.");
+ + "' and '" + IndexPolicy.PROP_CHAR_FILTER + "' are allowed.");
}
}
@@ -200,6 +207,13 @@ public class IndexPolicyMgr implements Writable, GsonPostProcessable {
validatePolicyReference(filter, IndexPolicyTypeEnum.TOKEN_FILTER);
}
}
+
+ String charFilters = properties.get(IndexPolicy.PROP_CHAR_FILTER);
+ if (charFilters != null && !charFilters.isEmpty()) {
+ for (String filter : charFilters.split(",\\s*")) {
+ validatePolicyReference(filter, IndexPolicyTypeEnum.CHAR_FILTER);
+ }
+ }
}
private void validatePolicyReference(String name, IndexPolicyTypeEnum expectedType)
@@ -212,6 +226,10 @@ public class IndexPolicyMgr implements Writable, GsonPostProcessable {
&& IndexPolicy.BUILTIN_TOKEN_FILTERS.contains(name)) {
return;
}
+ if (expectedType == IndexPolicyTypeEnum.CHAR_FILTER
+ && IndexPolicy.BUILTIN_CHAR_FILTERS.contains(name)) {
+ return;
+ }
IndexPolicy policy = getPolicyByName(name);
if (policy == null) {
@@ -230,6 +248,9 @@ public class IndexPolicyMgr implements Writable, GsonPostProcessable {
}
PolicyPropertyValidator validator;
switch (type) {
+ case "empty":
+ validator = new NoOperationValidator("empty tokenizer");
+ break;
case "ngram":
validator = new NGramTokenizerValidator();
break;
@@ -240,7 +261,7 @@ public class IndexPolicyMgr implements Writable, GsonPostProcessable {
validator = new StandardTokenizerValidator();
break;
case "keyword":
- validator = new KeywordTokenizerValidator();
+ validator = new NoOperationValidator("keyword tokenizer");
break;
case "char_group":
validator = new CharGroupTokenizerValidator();
@@ -259,6 +280,9 @@ public class IndexPolicyMgr implements Writable, GsonPostProcessable {
}
PolicyPropertyValidator validator;
switch (type) {
+ case "empty":
+ validator = new NoOperationValidator("empty token filter");
+ break;
case "asciifolding":
validator = new AsciiFoldingTokenFilterValidator();
break;
@@ -266,7 +290,7 @@ public class IndexPolicyMgr implements Writable, GsonPostProcessable {
validator = new WordDelimiterTokenFilterValidator();
break;
case "lowercase":
- validator = new LowerCaseTokenFilterValidator();
+ validator = new NoOperationValidator("lowercase token filter");
break;
default:
throw new DdlException("Unsupported token filter type: " + type
@@ -275,6 +299,26 @@ public class IndexPolicyMgr implements Writable, GsonPostProcessable {
validator.validate(properties);
}
+ private void validateCharFilterProperties(Map<String, String> properties) throws DdlException {
+ String type = properties.get(IndexPolicy.PROP_TYPE);
+ if (type == null || type.isEmpty()) {
+ throw new DdlException("CHAR_FILTER must specify a 'type' property");
+ }
+ PolicyPropertyValidator validator;
+ switch (type) {
+ case "empty":
+ validator = new NoOperationValidator("empty char filter");
+ break;
+ case "char_replace":
+ validator = new CharReplaceCharFilterValidator();
+ break;
+ default:
+ throw new DdlException("Unsupported char filter type: " + type
+ + ". Supported types: " + IndexPolicy.BUILTIN_CHAR_FILTERS);
+ }
+ validator.validate(properties);
+ }
+
public void dropIndexPolicy(boolean isIfExists, String indexPolicyName,
IndexPolicyTypeEnum type) throws DdlException, AnalysisException {
writeLock();
@@ -290,7 +334,8 @@ public class IndexPolicyMgr implements Writable, GsonPostProcessable {
checkAnalyzerNotUsedByIndex(policyToDrop.getName());
}
if (policyToDrop.getType() == IndexPolicyTypeEnum.TOKENIZER
- || policyToDrop.getType() == IndexPolicyTypeEnum.TOKEN_FILTER) {
+ || policyToDrop.getType() == IndexPolicyTypeEnum.TOKEN_FILTER
+ || policyToDrop.getType() == IndexPolicyTypeEnum.CHAR_FILTER) {
checkPolicyNotReferenced(policyToDrop);
}
long id = policyToDrop.getId();
@@ -323,7 +368,8 @@ public class IndexPolicyMgr implements Writable, GsonPostProcessable {
if (properties != null
&& analyzerName.equals(properties.get(IndexPolicy.PROP_ANALYZER))) {
throw new DdlException("the analyzer " + analyzerName + " is used by index: "
- + index.getIndexName() + " in table: " + table.getName());
+ + index.getIndexName() + " in table: "
+ + db.getFullName() + "." + table.getName());
}
}
}
@@ -355,6 +401,17 @@ public class IndexPolicyMgr implements Writable, GsonPostProcessable {
}
}
}
+ } else if (policyType == IndexPolicyTypeEnum.CHAR_FILTER) {
+ String charFilters = properties.get(IndexPolicy.PROP_CHAR_FILTER);
+ if (charFilters != null && !charFilters.isEmpty()) {
+ for (String filter : charFilters.split(",\\s*")) {
+ if (policyName.equals(filter)) {
+ throw new DdlException("Cannot drop " + policyType + " policy '"
+ + policyName + "' as it is referenced by ANALYZER policy '"
+ + analyzerPolicy.getName() + "'");
+ }
+ }
+ }
}
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/indexpolicy/IndexPolicyTypeEnum.java b/fe/fe-core/src/main/java/org/apache/doris/indexpolicy/IndexPolicyTypeEnum.java
index 2f146e16884..acda67c9b8c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/indexpolicy/IndexPolicyTypeEnum.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/indexpolicy/IndexPolicyTypeEnum.java
@@ -23,13 +23,14 @@ import org.apache.doris.thrift.TIndexPolicyType;
* Index policy type enum.
**/
public enum IndexPolicyTypeEnum {
- ANALYZER, TOKENIZER, TOKEN_FILTER;
+ ANALYZER, TOKENIZER, TOKEN_FILTER, CHAR_FILTER;
public TIndexPolicyType toThrift() {
switch (this) {
case ANALYZER: return TIndexPolicyType.ANALYZER;
case TOKENIZER: return TIndexPolicyType.TOKENIZER;
case TOKEN_FILTER: return TIndexPolicyType.TOKEN_FILTER;
+ case CHAR_FILTER: return TIndexPolicyType.CHAR_FILTER;
default: throw new IllegalStateException("Unknown type: " + this);
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/indexpolicy/KeywordTokenizerValidator.java b/fe/fe-core/src/main/java/org/apache/doris/indexpolicy/NoOperationValidator.java
similarity index 83%
rename from fe/fe-core/src/main/java/org/apache/doris/indexpolicy/KeywordTokenizerValidator.java
rename to fe/fe-core/src/main/java/org/apache/doris/indexpolicy/NoOperationValidator.java
index 1a9ab2865b2..22baa33388f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/indexpolicy/KeywordTokenizerValidator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/indexpolicy/NoOperationValidator.java
@@ -24,20 +24,22 @@ import com.google.common.collect.ImmutableSet;
import java.util.Map;
import java.util.Set;
-public class KeywordTokenizerValidator extends BasePolicyValidator {
+public class NoOperationValidator extends BasePolicyValidator {
private static final Set<String> ALLOWED_PROPS = ImmutableSet.of("type");
+ private final String typeName;
- public KeywordTokenizerValidator() {
+ public NoOperationValidator(String typeName) {
super(ALLOWED_PROPS);
+ this.typeName = typeName;
}
@Override
protected String getTypeName() {
- return "keyword tokenizer";
+ return typeName;
}
@Override
protected void validateSpecific(Map<String, String> props) throws DdlException {
- // No parameters to validate
+ // No additional parameters to validate
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index 867dc525683..e7cdd8770df 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -80,6 +80,7 @@ import org.apache.doris.nereids.DorisParser.ComplexColTypeListContext;
import org.apache.doris.nereids.DorisParser.ComplexDataTypeContext;
import org.apache.doris.nereids.DorisParser.ConstantContext;
import org.apache.doris.nereids.DorisParser.CreateIndexAnalyzerContext;
+import org.apache.doris.nereids.DorisParser.CreateIndexCharFilterContext;
import org.apache.doris.nereids.DorisParser.CreateIndexTokenFilterContext;
import org.apache.doris.nereids.DorisParser.CreateIndexTokenizerContext;
import org.apache.doris.nereids.DorisParser.CreateMTMVContext;
@@ -96,6 +97,7 @@ import org.apache.doris.nereids.DorisParser.DereferenceContext;
import org.apache.doris.nereids.DorisParser.DropCatalogRecycleBinContext;
import org.apache.doris.nereids.DorisParser.DropConstraintContext;
import org.apache.doris.nereids.DorisParser.DropIndexAnalyzerContext;
+import org.apache.doris.nereids.DorisParser.DropIndexCharFilterContext;
import org.apache.doris.nereids.DorisParser.DropIndexTokenFilterContext;
import org.apache.doris.nereids.DorisParser.DropIndexTokenizerContext;
import org.apache.doris.nereids.DorisParser.DropMTMVContext;
@@ -185,6 +187,7 @@ import org.apache.doris.nereids.DorisParser.ShowConstraintContext;
import org.apache.doris.nereids.DorisParser.ShowCreateMTMVContext;
import org.apache.doris.nereids.DorisParser.ShowCreateProcedureContext;
import org.apache.doris.nereids.DorisParser.ShowIndexAnalyzerContext;
+import org.apache.doris.nereids.DorisParser.ShowIndexCharFilterContext;
import org.apache.doris.nereids.DorisParser.ShowIndexTokenFilterContext;
import org.apache.doris.nereids.DorisParser.ShowIndexTokenizerContext;
import org.apache.doris.nereids.DorisParser.ShowProcedureStatusContext;
@@ -354,6 +357,7 @@ import org.apache.doris.nereids.trees.plans.commands.CancelMTMVTaskCommand;
import org.apache.doris.nereids.trees.plans.commands.Command;
import org.apache.doris.nereids.trees.plans.commands.Constraint;
import org.apache.doris.nereids.trees.plans.commands.CreateIndexAnalyzerCommand;
+import org.apache.doris.nereids.trees.plans.commands.CreateIndexCharFilterCommand;
import org.apache.doris.nereids.trees.plans.commands.CreateIndexTokenFilterCommand;
import org.apache.doris.nereids.trees.plans.commands.CreateIndexTokenizerCommand;
import org.apache.doris.nereids.trees.plans.commands.CreateJobCommand;
@@ -370,6 +374,7 @@ import org.apache.doris.nereids.trees.plans.commands.DropCatalogRecycleBinComman
import org.apache.doris.nereids.trees.plans.commands.DropCatalogRecycleBinCommand.IdType;
import org.apache.doris.nereids.trees.plans.commands.DropConstraintCommand;
import org.apache.doris.nereids.trees.plans.commands.DropIndexAnalyzerCommand;
+import org.apache.doris.nereids.trees.plans.commands.DropIndexCharFilterCommand;
import org.apache.doris.nereids.trees.plans.commands.DropIndexTokenFilterCommand;
import org.apache.doris.nereids.trees.plans.commands.DropIndexTokenizerCommand;
import org.apache.doris.nereids.trees.plans.commands.DropMTMVCommand;
@@ -391,6 +396,7 @@ import org.apache.doris.nereids.trees.plans.commands.ShowConstraintsCommand;
import org.apache.doris.nereids.trees.plans.commands.ShowCreateMTMVCommand;
import org.apache.doris.nereids.trees.plans.commands.ShowCreateProcedureCommand;
import org.apache.doris.nereids.trees.plans.commands.ShowIndexAnalyzerCommand;
+import org.apache.doris.nereids.trees.plans.commands.ShowIndexCharFilterCommand;
import org.apache.doris.nereids.trees.plans.commands.ShowIndexTokenFilterCommand;
import org.apache.doris.nereids.trees.plans.commands.ShowIndexTokenizerCommand;
import org.apache.doris.nereids.trees.plans.commands.ShowProcedureStatusCommand;
@@ -3898,6 +3904,15 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
return new CreateIndexTokenFilterCommand(ifNotExists, policyName, properties);
}
+ @Override
+ public LogicalPlan visitCreateIndexCharFilter(CreateIndexCharFilterContext ctx) {
+ boolean ifNotExists = ctx.IF() != null;
+ String policyName = ctx.name.getText();
+ Map<String, String> properties = visitPropertyClause(ctx.properties);
+
+ return new CreateIndexCharFilterCommand(ifNotExists, policyName, properties);
+ }
+
@Override
public LogicalPlan visitDropIndexAnalyzer(DropIndexAnalyzerContext ctx) {
String policyName = ctx.name.getText();
@@ -3922,6 +3937,14 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
return new DropIndexTokenFilterCommand(policyName, ifExists);
}
+ @Override
+ public LogicalPlan visitDropIndexCharFilter(DropIndexCharFilterContext ctx) {
+ String policyName = ctx.name.getText();
+ boolean ifExists = ctx.IF() != null;
+
+ return new DropIndexCharFilterCommand(policyName, ifExists);
+ }
+
@Override
public LogicalPlan visitShowIndexAnalyzer(ShowIndexAnalyzerContext ctx) {
return new ShowIndexAnalyzerCommand();
@@ -3936,5 +3959,10 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
public LogicalPlan visitShowIndexTokenFilter(ShowIndexTokenFilterContext ctx) {
return new ShowIndexTokenFilterCommand();
}
+
+ @Override
+ public LogicalPlan visitShowIndexCharFilter(ShowIndexCharFilterContext ctx) {
+ return new ShowIndexCharFilterCommand();
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
index 0f2384fe141..d3335628352 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
@@ -190,5 +190,8 @@ public enum PlanType {
DROP_INDEX_TOKEN_FILTER_COMMAND,
SHOW_INDEX_ANALYZER_COMMAND,
SHOW_INDEX_TOKENIZER_COMMAND,
- SHOW_INDEX_TOKEN_FILTER_COMMAND;
+ SHOW_INDEX_TOKEN_FILTER_COMMAND,
+ CREATE_INDEX_CHAR_FILTER_COMMAND,
+ DROP_INDEX_CHAR_FILTER_COMMAND,
+ SHOW_INDEX_CHAR_FILTER_COMMAND
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateIndexCharFilterCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateIndexCharFilterCommand.java
new file mode 100644
index 00000000000..261046d9659
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateIndexCharFilterCommand.java
@@ -0,0 +1,75 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands;
+
+import org.apache.doris.catalog.Env;
+import org.apache.doris.common.ErrorCode;
+import org.apache.doris.common.ErrorReport;
+import org.apache.doris.common.FeNameFormat;
+import org.apache.doris.common.UserException;
+import org.apache.doris.indexpolicy.IndexPolicyTypeEnum;
+import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.StmtExecutor;
+
+import java.util.Map;
+
+/**
+ * CREATE INVERTED INDEX CHAR_FILTER [IF NOT EXISTS] policy_name PROPERTIES (key1 = value1, ...)
+ */
+public class CreateIndexCharFilterCommand extends Command implements ForwardWithSync {
+
+ private final boolean ifNotExists;
+ private final String policyName;
+ private final Map<String, String> properties;
+
+ /**
+ * Constructor
+ */
+ public CreateIndexCharFilterCommand(boolean ifNotExists, String policyName, Map<String, String> properties) {
+ super(PlanType.CREATE_INDEX_CHAR_FILTER_COMMAND);
+ this.ifNotExists = ifNotExists;
+ this.policyName = policyName;
+ this.properties = properties;
+ }
+
+ private void validate(ConnectContext ctx) throws UserException {
+ // check auth
+ if (!Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) {
+ ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN");
+ }
+
+ // check name
+ FeNameFormat.checkIndexPolicyName(policyName);
+ }
+
+ @Override
+ public void run(ConnectContext ctx, StmtExecutor executor) throws UserException {
+ validate(ctx);
+
+ Env.getCurrentEnv().getIndexPolicyMgr().createIndexPolicy(ifNotExists, policyName,
+ IndexPolicyTypeEnum.CHAR_FILTER, properties);
+ }
+
+ @Override
+ public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+ return visitor.visitCreateIndexCharFilterCommand(this, context);
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DropIndexCharFilterCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DropIndexCharFilterCommand.java
new file mode 100644
index 00000000000..62874f54b8c
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DropIndexCharFilterCommand.java
@@ -0,0 +1,62 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands;
+
+import org.apache.doris.catalog.Env;
+import org.apache.doris.common.ErrorCode;
+import org.apache.doris.common.ErrorReport;
+import org.apache.doris.common.FeNameFormat;
+import org.apache.doris.indexpolicy.IndexPolicyTypeEnum;
+import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.StmtExecutor;
+
+/**
+ * DROP INVERTED INDEX CHAR_FILTER [IF EXISTS] policy_name
+ **/
+public class DropIndexCharFilterCommand extends DropCommand {
+ private final boolean ifExists;
+ private final String name;
+
+ public DropIndexCharFilterCommand(String name, boolean ifExists) {
+ super(PlanType.DROP_INDEX_CHAR_FILTER_COMMAND);
+ this.name = name;
+ this.ifExists = ifExists;
+ }
+
+ @Override
+ public void doRun(ConnectContext ctx, StmtExecutor executor) throws Exception {
+ // check auth
+ if (!Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) {
+ ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN");
+ }
+
+ // check name
+ FeNameFormat.checkIndexPolicyName(name);
+
+ Env.getCurrentEnv().getIndexPolicyMgr().dropIndexPolicy(ifExists, name,
+ IndexPolicyTypeEnum.CHAR_FILTER);
+ }
+
+ @Override
+ public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+ return visitor.visitDropIndexCharFilterCommand(this, context);
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowIndexCharFilterCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowIndexCharFilterCommand.java
new file mode 100644
index 00000000000..57f1f362ce3
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowIndexCharFilterCommand.java
@@ -0,0 +1,53 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands;
+
+import org.apache.doris.catalog.Env;
+import org.apache.doris.common.ErrorCode;
+import org.apache.doris.common.ErrorReport;
+import org.apache.doris.indexpolicy.IndexPolicyTypeEnum;
+import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.ShowResultSet;
+import org.apache.doris.qe.StmtExecutor;
+
+/**
+ * SHOW INVERTED INDEX CHAR_FILTER;
+ **/
+public class ShowIndexCharFilterCommand extends Command implements NoForward {
+ public ShowIndexCharFilterCommand() {
+ super(PlanType.SHOW_INDEX_CHAR_FILTER_COMMAND);
+ }
+
+ @Override
+ public void run(ConnectContext ctx, StmtExecutor executor) throws Exception {
+ if (!Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) {
+ ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN");
+ }
+ ShowResultSet showResultSet = Env.getCurrentEnv().getIndexPolicyMgr()
+ .showIndexPolicy(IndexPolicyTypeEnum.CHAR_FILTER);
+ executor.sendResultSet(showResultSet);
+ }
+
+ @Override
+ public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+ return visitor.visitShowIndexCharFilterCommand(this, context);
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java
index 1e449e83916..4e9787a03a2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java
@@ -147,7 +147,9 @@ public class IndexDefinition {
+ " or key columns of AGG_KEYS table. invalid index: " + name);
} else if (keysType == KeysType.UNIQUE_KEYS && !enableUniqueKeyMergeOnWrite
&& indexType == IndexType.INVERTED && properties != null
- && properties.containsKey(InvertedIndexUtil.INVERTED_INDEX_PARSER_KEY)) {
+ && (properties.containsKey(InvertedIndexUtil.INVERTED_INDEX_PARSER_KEY)
+ || properties.containsKey(InvertedIndexUtil.INVERTED_INDEX_PARSER_KEY_ALIAS)
+ || properties.containsKey(InvertedIndexUtil.INVERTED_INDEX_CUSTOM_ANALYZER_KEY))) {
throw new AnalysisException("INVERTED index with parser can NOT be used in value columns of"
+ " UNIQUE_KEYS table with merge_on_write disable. invalid index: " + name);
}
@@ -245,9 +247,10 @@ public class IndexDefinition {
}
public boolean isAnalyzedInvertedIndex() {
- return indexType == IndexDef.IndexType.INVERTED
- && properties != null
- && (properties.containsKey(InvertedIndexUtil.INVERTED_INDEX_PARSER_KEY)
- || properties.containsKey(InvertedIndexUtil.INVERTED_INDEX_CUSTOM_ANALYZER_KEY));
+ return indexType == IndexType.INVERTED
+ && properties != null
+ && (properties.containsKey(InvertedIndexUtil.INVERTED_INDEX_PARSER_KEY)
+ || properties.containsKey(InvertedIndexUtil.INVERTED_INDEX_PARSER_KEY_ALIAS)
+ || properties.containsKey(InvertedIndexUtil.INVERTED_INDEX_CUSTOM_ANALYZER_KEY));
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
index 455f23ed9f2..24f97f08656 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
@@ -24,6 +24,7 @@ import org.apache.doris.nereids.trees.plans.commands.CallCommand;
import org.apache.doris.nereids.trees.plans.commands.CancelMTMVTaskCommand;
import org.apache.doris.nereids.trees.plans.commands.Command;
import org.apache.doris.nereids.trees.plans.commands.CreateIndexAnalyzerCommand;
+import org.apache.doris.nereids.trees.plans.commands.CreateIndexCharFilterCommand;
import org.apache.doris.nereids.trees.plans.commands.CreateIndexTokenFilterCommand;
import org.apache.doris.nereids.trees.plans.commands.CreateIndexTokenizerCommand;
import org.apache.doris.nereids.trees.plans.commands.CreateJobCommand;
@@ -39,6 +40,7 @@ import org.apache.doris.nereids.trees.plans.commands.DropAnalyzeJobCommand;
import org.apache.doris.nereids.trees.plans.commands.DropCatalogRecycleBinCommand;
import org.apache.doris.nereids.trees.plans.commands.DropConstraintCommand;
import org.apache.doris.nereids.trees.plans.commands.DropIndexAnalyzerCommand;
+import org.apache.doris.nereids.trees.plans.commands.DropIndexCharFilterCommand;
import org.apache.doris.nereids.trees.plans.commands.DropIndexTokenFilterCommand;
import org.apache.doris.nereids.trees.plans.commands.DropIndexTokenizerCommand;
import org.apache.doris.nereids.trees.plans.commands.DropMTMVCommand;
@@ -59,6 +61,7 @@ import org.apache.doris.nereids.trees.plans.commands.ShowConstraintsCommand;
import org.apache.doris.nereids.trees.plans.commands.ShowCreateMTMVCommand;
import org.apache.doris.nereids.trees.plans.commands.ShowCreateProcedureCommand;
import org.apache.doris.nereids.trees.plans.commands.ShowIndexAnalyzerCommand;
+import org.apache.doris.nereids.trees.plans.commands.ShowIndexCharFilterCommand;
import org.apache.doris.nereids.trees.plans.commands.ShowIndexTokenFilterCommand;
import org.apache.doris.nereids.trees.plans.commands.ShowIndexTokenizerCommand;
import org.apache.doris.nereids.trees.plans.commands.ShowProcedureStatusCommand;
@@ -246,6 +249,11 @@ public interface CommandVisitor<R, C> {
return visitCommand(createIndexAnalyzerCommand, context);
}
+ default R visitCreateIndexCharFilterCommand(
+ CreateIndexCharFilterCommand createIndexCharFilterCommand, C context) {
+ return visitCommand(createIndexCharFilterCommand, context);
+ }
+
default R visitCreateIndexTokenizerCommand(
CreateIndexTokenizerCommand createIndexTokenizerCommand, C context) {
return visitCommand(createIndexTokenizerCommand, context);
@@ -261,6 +269,11 @@ public interface CommandVisitor<R, C> {
return visitCommand(dropIndexAnalyzerCommand, context);
}
+ default R visitDropIndexCharFilterCommand(
+ DropIndexCharFilterCommand dropIndexCharFilterCommand, C context) {
+ return visitCommand(dropIndexCharFilterCommand, context);
+ }
+
default R visitDropIndexTokenizerCommand(
DropIndexTokenizerCommand dropIndexTokenizerCommand, C context) {
return visitCommand(dropIndexTokenizerCommand, context);
@@ -276,6 +289,11 @@ public interface CommandVisitor<R, C> {
return visitCommand(showIndexAnalyzerCommand, context);
}
+ default R visitShowIndexCharFilterCommand(
+ ShowIndexCharFilterCommand showIndexCharFilterCommand, C context) {
+ return visitCommand(showIndexCharFilterCommand, context);
+ }
+
default R visitShowIndexTokenizerCommand(
ShowIndexTokenizerCommand showIndexTokenizerCommand, C context) {
return visitCommand(showIndexTokenizerCommand, context);
diff --git a/gensrc/thrift/AgentService.thrift b/gensrc/thrift/AgentService.thrift
index ffc09730793..388b7c3751d 100644
--- a/gensrc/thrift/AgentService.thrift
+++ b/gensrc/thrift/AgentService.thrift
@@ -138,7 +138,8 @@ struct TPushStoragePolicyReq {
enum TIndexPolicyType {
ANALYZER,
TOKENIZER,
- TOKEN_FILTER
+ TOKEN_FILTER,
+ CHAR_FILTER
}
struct TIndexPolicy {
diff --git a/regression-test/data/inverted_index_p0/analyzer/test_custom_analyzer.out b/regression-test/data/inverted_index_p0/analyzer/test_custom_analyzer.out
index 4c22ab33fd2..687807afbfd 100644
--- a/regression-test/data/inverted_index_p0/analyzer/test_custom_analyzer.out
+++ b/regression-test/data/inverted_index_p0/analyzer/test_custom_analyzer.out
@@ -29,6 +29,12 @@
-- !tokenize_sql --
[{\n "token": "1080º"\n }, {\n "token": "avalanche"\n }]
+-- !tokenize_sql --
+[{\n "token": "get"\n }, {\n "token": "images"\n }, {\n "token": "hm"\n }, {\n "token": "bg"\n }, {\n "token": "jpg"\n }, {\n "token": "http"\n }, {\n "token": "1"\n }, {\n "token": "0"\n }]
+
+-- !tokenize_sql --
+[{\n "token": "让"\n }, {\n "token": "我们"\n }, {\n "token": "说"\n }, {\n "token": "hello"\n }, {\n "token": "そして"\n }, {\n "token": "世界"\n }, {\n "token": "と"\n }, {\n "token": "つ"\n }, {\n "token": "な"\n }, {\n "token": "が"\n }, {\n "token": "ろう"\n }]
+
-- !sql --
1 abcDEF
diff --git a/regression-test/data/inverted_index_p0/analyzer/test_custom_analyzer1.out b/regression-test/data/inverted_index_p0/analyzer/test_custom_analyzer1.out
index 124a6e37ed1..d976dd5b186 100644
--- a/regression-test/data/inverted_index_p0/analyzer/test_custom_analyzer1.out
+++ b/regression-test/data/inverted_index_p0/analyzer/test_custom_analyzer1.out
@@ -6,3 +6,9 @@
1 A two-hour programme which included many forms of [[jazz]] from classic to Latin as well as a mix of jazz from the younger players of the day.
2 with off-peak shows introducing more commercial breaks into their output, before the concept was dropped altogether in mid-2006.
+-- !sql --
+1 GET /images/hm_bg.jpg HTTP/1.0
+
+-- !sql --
+1 GET /images/hm_bg.jpg HTTP/1.0
+
diff --git a/regression-test/data/inverted_index_p0/analyzer/test_custom_analyzer2.out b/regression-test/data/inverted_index_p0/analyzer/test_custom_analyzer2.out
new file mode 100644
index 00000000000..a06726086f9
--- /dev/null
+++ b/regression-test/data/inverted_index_p0/analyzer/test_custom_analyzer2.out
@@ -0,0 +1,16 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !tokenize_sql --
+[{\n "token": "empty analyzer"\n }]
+
+-- !sql --
+1
+
+-- !sql --
+2
+
+-- !sql --
+2
+
+-- !sql --
+2
+
diff --git a/regression-test/suites/inverted_index_p0/analyzer/test_custom_analyzer.groovy b/regression-test/suites/inverted_index_p0/analyzer/test_custom_analyzer.groovy
index 9c2315b27eb..14ee4c6819c 100644
--- a/regression-test/suites/inverted_index_p0/analyzer/test_custom_analyzer.groovy
+++ b/regression-test/suites/inverted_index_p0/analyzer/test_custom_analyzer.groovy
@@ -63,8 +63,26 @@ suite("test_custom_analyzer", "p0") {
CREATE INVERTED INDEX ANALYZER IF NOT EXISTS keyword_lowercase
PROPERTIES
(
- "tokenizer" = "keyword",
- "token_filter" = "asciifolding, lowercase"
+ "tokenizer" = "keyword",
+ "token_filter" = "asciifolding, lowercase"
+ );
+ """
+
+ sql """
+ CREATE INVERTED INDEX ANALYZER IF NOT EXISTS basic_analyzer
+ PROPERTIES
+ (
+ "tokenizer" = "basic",
+ "token_filter" = "lowercase"
+ );
+ """
+
+ sql """
+ CREATE INVERTED INDEX ANALYZER IF NOT EXISTS icu_analyzer
+ PROPERTIES
+ (
+ "tokenizer" = "icu",
+ "token_filter" = "lowercase"
);
"""
@@ -80,6 +98,8 @@ suite("test_custom_analyzer", "p0") {
qt_tokenize_sql """ select tokenize("β-carbon nitride", '"analyzer"="lowercase_delimited"'); """
qt_tokenize_sql """ select tokenize("ǁŨǁe language", '"analyzer"="lowercase_delimited"'); """
qt_tokenize_sql """ select tokenize("1080º Avalanche", '"analyzer"="lowercase_delimited"'); """
+ qt_tokenize_sql """ select tokenize("GET /images/hm_bg.jpg HTTP/1.0", '"analyzer"="basic_analyzer"'); """
+ qt_tokenize_sql """ select tokenize("让我们说「Hello」そして世界とつながろう！", '"analyzer"="icu_analyzer"'); """
sql "DROP TABLE IF EXISTS ${indexTbName1}"
sql """
@@ -139,8 +159,6 @@ suite("test_custom_analyzer", "p0") {
}
}
-
-
try {
sql "DROP TABLE IF EXISTS test_custom_analyzer_3"
sql """
diff --git a/regression-test/suites/inverted_index_p0/analyzer/test_custom_analyzer1.groovy b/regression-test/suites/inverted_index_p0/analyzer/test_custom_analyzer1.groovy
index 1bede117f4d..665cda3b346 100644
--- a/regression-test/suites/inverted_index_p0/analyzer/test_custom_analyzer1.groovy
+++ b/regression-test/suites/inverted_index_p0/analyzer/test_custom_analyzer1.groovy
@@ -18,8 +18,6 @@
import java.sql.SQLException
suite("test_custom_analyzer1", "p0") {
- def indexTbName1 = "test_custom_analyzer1"
-
sql """
CREATE INVERTED INDEX TOKEN_FILTER IF NOT EXISTS word_splitter_all
PROPERTIES
@@ -34,22 +32,64 @@ suite("test_custom_analyzer1", "p0") {
"""
sql """
- CREATE INVERTED INDEX ANALYZER IF NOT EXISTS custom_standard_analyzer
+ CREATE INVERTED INDEX ANALYZER IF NOT EXISTS custom_standard_analyzer1
+ PROPERTIES
+ (
+ "tokenizer" = "standard",
+ "token_filter" = "asciifolding, word_splitter_all, lowercase"
+ );
+ """
+
+ sql """
+ CREATE INVERTED INDEX CHAR_FILTER IF NOT EXISTS char_replace_char_filter1
+ PROPERTIES
+ (
+ "type" = "char_replace",
+ "pattern" = "_"
+ );
+ """
+
+ sql """
+ CREATE INVERTED INDEX CHAR_FILTER IF NOT EXISTS char_replace_char_filter2
+ PROPERTIES
+ (
+ "type" = "char_replace",
+ "pattern" = "."
+ );
+ """
+
+ sql """
+ CREATE INVERTED INDEX ANALYZER IF NOT EXISTS custom_standard_analyzer2
PROPERTIES
(
- "tokenizer" = "standard",
- "token_filter" = "asciifolding, word_splitter_all, lowercase"
+ "tokenizer" = "standard",
+ "char_filter" = "char_replace_char_filter1, char_replace_char_filter2",
+ "token_filter" = "lowercase"
);
"""
sql """ select sleep(10) """
- sql "DROP TABLE IF EXISTS ${indexTbName1}"
+ sql "DROP TABLE IF EXISTS test_custom_analyzer1"
+ sql """
+ CREATE TABLE test_custom_analyzer1 (
+ `a` bigint NOT NULL AUTO_INCREMENT(1),
+ `ch` text NULL,
+ INDEX idx_ch (`ch`) USING INVERTED PROPERTIES("support_phrase" = "true", "analyzer" = "custom_standard_analyzer1")
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`a`)
+ DISTRIBUTED BY RANDOM BUCKETS 1
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1"
+ );
+ """
+
+ sql "DROP TABLE IF EXISTS test_custom_analyzer2"
sql """
- CREATE TABLE ${indexTbName1} (
+ CREATE TABLE test_custom_analyzer2 (
`a` bigint NOT NULL AUTO_INCREMENT(1),
`ch` text NULL,
- INDEX idx_ch (`ch`) USING INVERTED PROPERTIES("support_phrase" = "true", "analyzer" = "custom_standard_analyzer")
+ INDEX idx_ch (`ch`) USING INVERTED PROPERTIES("support_phrase" = "true", "analyzer" = "custom_standard_analyzer2")
) ENGINE=OLAP
DUPLICATE KEY(`a`)
DISTRIBUTED BY RANDOM BUCKETS 1
@@ -58,15 +98,20 @@ suite("test_custom_analyzer1", "p0") {
);
"""
- sql """ insert into ${indexTbName1} values(1, "A two-hour programme which included many forms of [[jazz]] from classic to Latin as well as a mix of jazz from the younger players of the day."); """
- sql """ insert into ${indexTbName1} values(2, " with off-peak shows introducing more commercial breaks into their output, before the concept was dropped altogether in mid-2006."); """
+ sql """ insert into test_custom_analyzer1 values(1, "A two-hour programme which included many forms of [[jazz]] from classic to Latin as well as a mix of jazz from the younger players of the day."); """
+ sql """ insert into test_custom_analyzer1 values(2, " with off-peak shows introducing more commercial breaks into their output, before the concept was dropped altogether in mid-2006."); """
+
+ sql """ insert into test_custom_analyzer2 values(1, "GET /images/hm_bg.jpg HTTP/1.0"); """
try {
sql "sync"
sql """ set enable_common_expr_pushdown = true; """
- qt_sql """ select * from ${indexTbName1} where ch match 'with'; """
- qt_sql """ select * from ${indexTbName1} where ch match 'the'; """
+ qt_sql """ select * from test_custom_analyzer1 where ch match 'with'; """
+ qt_sql """ select * from test_custom_analyzer1 where ch match 'the'; """
+
+ qt_sql """ select * from test_custom_analyzer2 where ch match 'hm'; """
+ qt_sql """ select * from test_custom_analyzer2 where ch match 'bg'; """
} finally {
}
}
\ No newline at end of file
diff --git a/regression-test/suites/inverted_index_p0/analyzer/test_custom_analyzer2.groovy b/regression-test/suites/inverted_index_p0/analyzer/test_custom_analyzer2.groovy
new file mode 100644
index 00000000000..88fa81a3efc
--- /dev/null
+++ b/regression-test/suites/inverted_index_p0/analyzer/test_custom_analyzer2.groovy
@@ -0,0 +1,62 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import java.sql.SQLException
+
+suite("test_custom_analyzer2", "p0") {
+ sql """
+ CREATE INVERTED INDEX ANALYZER IF NOT EXISTS empty_analyzer
+ PROPERTIES
+ (
+ "tokenizer" = "empty",
+ "char_filter" = "empty",
+ "token_filter" = "empty"
+ );
+ """
+
+ sql """ select sleep(10) """
+
+ qt_tokenize_sql """ select tokenize("empty analyzer", '"analyzer"="empty_analyzer"'); """
+
+ sql "DROP TABLE IF EXISTS test_custom_analyzer2_1"
+ sql """
+ CREATE TABLE test_custom_analyzer2_1 (
+ `a` bigint NOT NULL AUTO_INCREMENT(1),
+ `ch` text NULL,
+ INDEX idx_ch (`ch`) USING INVERTED PROPERTIES("built_in_analyzer" = "standard", "support_phrase" = "true")
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`a`)
+ DISTRIBUTED BY RANDOM BUCKETS 1
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1"
+ );
+ """
+
+ try {
+ sql """ insert into test_custom_analyzer2_1 values(1, "中国人民"); """
+ sql """ insert into test_custom_analyzer2_1 values(2, "美国人民"); """
+
+ sql "sync"
+ sql """ set enable_common_expr_pushdown = true; """
+
+ qt_sql """ select count(1) from test_custom_analyzer2_1 where ch match '中'; """
+ qt_sql """ select count(1) from test_custom_analyzer2_1 where ch match '国'; """
+ qt_sql """ select count(1) from test_custom_analyzer2_1 where ch match '人'; """
+ qt_sql """ select count(1) from test_custom_analyzer2_1 where ch match '民'; """
+ } finally {
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org