You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by lx...@apache.org on 2026/10/13 05:29:21 UTC
(paimon-cpp) branch main updated: feat(global-index): align FullTextSearch with Java JSON DSL query model (#424)
This is an automated email from the ASF dual-hosted git repository.
lxy-9602 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/paimon-cpp.git
The following commit(s) were added to refs/heads/main by this push:
new c75b3eba feat(global-index): align FullTextSearch with Java JSON DSL query model (#424)
c75b3eba is described below
commit c75b3eba79337857be945a2614b4e795e05952ac
Author: Nicholas Jiang <pr...@163.com>
AuthorDate: Tue Oct 13 12:29:14 2026 +0700
feat(global-index): align FullTextSearch with Java JSON DSL query model (#424)
---
docs/source/user_guide/global_index.rst | 75 ++-
include/paimon/global_index/global_index_reader.h | 8 +-
include/paimon/predicate/full_text_search.h | 142 ++---
src/paimon/CMakeLists.txt | 1 +
.../empty/empty_file_index_reader_test.cpp | 4 +-
.../common/file_index/file_index_reader_test.cpp | 4 +-
.../global_index/bitmap/bitmap_index_reader.cpp | 2 +-
.../global_index/bitmap/bitmap_index_reader.h | 2 +-
.../btree/btree_global_index_reader.cpp | 2 +-
.../global_index/btree/btree_global_index_reader.h | 2 +-
.../btree/lazy_filtered_btree_reader.cpp | 2 +-
.../btree/lazy_filtered_btree_reader.h | 2 +-
.../global_index/offset_global_index_reader.cpp | 53 +-
.../global_index/offset_global_index_reader.h | 5 +-
.../offset_global_index_reader_test.cpp | 58 +-
.../global_index/union_global_index_reader.cpp | 27 +-
.../global_index/union_global_index_reader.h | 8 +-
.../union_global_index_reader_test.cpp | 9 +-
.../global_index/wrap/file_index_reader_wrapper.h | 6 +-
.../common/predicate/full_text_search_test.cpp | 63 +++
src/paimon/core/io/file_index_evaluator_test.cpp | 5 +-
.../core/operation/internal_read_context.cpp | 8 +-
.../core/operation/internal_read_context_test.cpp | 7 +-
src/paimon/core/operation/raw_file_split_read.cpp | 8 +-
.../core/operation/raw_file_split_read_test.cpp | 10 +-
src/paimon/core/operation/read_context_test.cpp | 8 +-
.../table/source/primary_key_sorted_index_scan.cpp | 2 +-
src/paimon/core/table/source/table_read_test.cpp | 4 +-
.../full_text/full_text_global_index_reader.cpp | 44 +-
.../full_text/full_text_global_index_reader.h | 2 +-
.../full_text/full_text_global_index_test.cpp | 89 +--
src/paimon/global_index/lucene/lucene_collector.h | 46 --
.../lucene/lucene_global_index_reader.cpp | 621 +++++++++++++++------
.../lucene/lucene_global_index_reader.h | 26 +-
.../lucene/lucene_global_index_test.cpp | 516 ++++++++++-------
src/paimon/indexer/lumina/lumina_global_index.h | 4 +-
src/paimon/testing/mock/mock_global_index_reader.h | 43 +-
src/paimon/testing/mock/mock_search_file_index.cpp | 4 +-
test/inte/global_index_test.cpp | 94 ++--
39 files changed, 1206 insertions(+), 810 deletions(-)
diff --git a/docs/source/user_guide/global_index.rst b/docs/source/user_guide/global_index.rst
index 040d483e..e4839fc4 100644
--- a/docs/source/user_guide/global_index.rst
+++ b/docs/source/user_guide/global_index.rst
@@ -24,7 +24,7 @@ Paimon C++ supports the following global index types:
- **BTree Index**: An efficient index based on multi-level SST files for scalar column lookups.
- **Range Bitmap Index**: A range bitmap index optimized for range predicates on ordered scalar columns. Extends the bitmap approach by encoding value ordering, enabling efficient less-than, greater-than, and range conditions.
-- **Lucene Index**: A full-text search index powered by Lucene++. Supports tokenized text search with multiple modes including match-all, match-any, phrase, prefix, and wildcard queries.
+- **Lucene Index**: A full-text search index powered by Lucene++. Supports tokenized text search with the JSON queries of the full-text index: match, multi-match, phrase and boolean queries.
- **Full-Text Index (experimental)**: A full-text search index powered by the native
``paimon-full-text-index`` engine. It uses the same index file format as Java Paimon's
``full-text`` index.
@@ -66,23 +66,28 @@ less-than, greater-than, and range conditions without scanning all bitmaps.
Lucene Index
------------
-A full-text search index powered by Lucene++. It supports tokenized text search with multiple
-search modes including match-all, match-any, phrase, prefix, and wildcard queries.
-
-**Supported search types:**
-
-- ``MATCH_ALL``: All terms in the query must be present (AND semantics).
-- ``MATCH_ANY``: Any term in the query can match (OR semantics).
-- ``PHRASE``: Matches the exact sequence of words (with proximity).
-- ``PREFIX``: Matches terms starting with the given string (e.g., "run*" → running, runner). The
- query is not tokenized. The original prefix is retained, and a pure ASCII alphanumeric prefix
- is also matched using the lowercase case-normalization applied to pure ASCII terms at indexing
- time. This preserves matches for mixed terms such as ``B超`` while allowing ``THIS`` to match
- terms indexed as ``this...``.
-- ``WILDCARD``: Supports wildcards ``*`` and ``?`` (e.g., "ap*e", "app?e" → "apple"). The query
- is not tokenized, and wildcard operators are preserved. Both the original pattern and an
- alternative with each ASCII alphanumeric fragment lowercased are matched, covering pure ASCII
- and mixed ASCII/non-ASCII terms.
+A full-text search index powered by Lucene++. It uses ``FullTextSearch`` and the JSON DSL
+described in `Full-Text Index (Experimental)`_, translating queries into Lucene queries on the
+indexed field:
+
+- ``match``: one term query per analyzed term, combined with OR, or with AND for
+ ``"operator": "And"``. ``boost`` sets the boost of the query.
+- ``multi_match``: one ``match`` query per column, combined with OR. Every column must be the
+ indexed field.
+- ``match_phrase`` (alias ``phrase``): a phrase query with an optional ``slop``.
+- ``boolean``: a boolean query of the translated ``must``, ``should``, ``must_not`` and
+ ``queries`` clauses.
+
+``boost`` queries and fuzzy matching (a ``fuzziness`` other than 0) are rejected with a
+``NotImplemented`` status, and a ``column`` other than the indexed field is rejected. The query
+text is analyzed with the Jieba tokenizer of the index. Each index shard returns at most ``limit``
+rows with the highest Lucene relevance scores.
+
+Duplicate known fields, raw NUL bytes in the JSON text and boosts that do not round to a finite
+positive float are rejected, while unknown fields are ignored. As in the native DSL, a query may
+nest at most 127 JSON arrays and objects, not counting unknown fields. Leading and trailing Unicode
+whitespace in operators, boolean occurrences and column names is trimmed as in the native DSL, and
+a column name containing only such whitespace selects the indexed field.
**Special Configuration:**
@@ -117,20 +122,36 @@ by it. Rebuild such indexes with the ``full-text`` index type.
**Queries:**
-``FullTextSearch::query`` is passed to the engine unchanged and must be a JSON query, for example:
+``FullTextSearch`` is aligned with Java Paimon: it takes the field name, a JSON query, a positive
+``limit`` and optional ``include_row_ids``. ``FullTextSearch::query`` is passed to the engine
+unchanged and must be a JSON query, for example:
- ``{"match": {"query": "paimon lake"}}``: rows that contain any of the analyzed terms. Add
- ``"operator": "And"`` to require all terms.
+ ``"operator": "And"`` to require all terms. ``boost``, ``fuzziness``, ``max_expansions`` and
+ ``prefix_length`` are optional.
+- ``{"multi_match": {"query": "paimon lake", "columns": ["text"]}}``: a ``match`` query on the
+ default native index field, with optional per-column ``boosts``.
- ``{"match_phrase": {"query": "data lake"}}``: rows that contain the terms as a phrase. An
optional ``"slop"`` allows other terms between them.
- ``{"boolean": {"must": [...], "should": [...], "must_not": [...]}}``: combines other queries.
-
-The query text is analyzed with the analyzer stored in the index file. ``search_type`` is ignored.
-A positive ``limit`` is required. The engine always ranks the matching rows by BM25 score and
-returns the ``limit`` rows with the highest scores; ``pre_filter`` restricts the rows that are
-ranked. ``min_score`` is applied to those rows afterwards and drops rows whose score is not greater
-than it, so a search can return fewer than ``limit`` rows. ``with_score`` only selects whether the
-scores are returned; scores are computed either way.
+- ``{"boost": {"positive": {...}, "negative": {...}, "negative_boost": 0.5}}``: lowers the score
+ of rows that also match the negative query.
+
+The C++ writer indexes one table column. ``FullTextSearch::field_name`` selects that table column,
+while ``column`` and ``columns`` in the JSON query name fields inside the native index. The native
+field defaults to ``text`` and can be renamed with ``full-text.text-field``. These names do not
+enable searches across multiple table columns. For ``lucene-fts``, DSL column names must instead
+match the indexed table column.
+
+The query text is analyzed with the analyzer stored in the index file, and an invalid query is
+reported as an error status. A search always returns a ``ScoredGlobalIndexResult``. Within each
+index shard, the engine ranks matching rows by BM25 score and returns at most ``limit`` rows with
+the highest scores. ``include_row_ids`` holds global row ids and restricts the rows that are
+ranked; an empty set matches no rows.
+
+For both ``full-text`` and ``lucene-fts``, searches across multiple shards currently return the
+union of the shard candidates, which can exceed ``limit``. Applying a final global top-k is tracked
+in `issue #402 <https://github.com/apache/paimon-cpp/issues/402>`_.
**Configuration:**
diff --git a/include/paimon/global_index/global_index_reader.h b/include/paimon/global_index/global_index_reader.h
index 25792465..db271bce 100644
--- a/include/paimon/global_index/global_index_reader.h
+++ b/include/paimon/global_index/global_index_reader.h
@@ -45,8 +45,12 @@ class PAIMON_EXPORT GlobalIndexReader : public FunctionVisitor<std::shared_ptr<G
virtual Result<std::shared_ptr<ScoredGlobalIndexResult>> VisitVectorSearch(
const std::shared_ptr<VectorSearch>& vector_search) = 0;
- /// VisitFullTextSearch performs full text search.
- virtual Result<std::shared_ptr<GlobalIndexResult>> VisitFullTextSearch(
+ /// VisitFullTextSearch performs full text search and returns row ids with relevance scores.
+ /// Each index shard returns at most `full_text_search->limit` rows. Union readers currently
+ /// retain all shard candidates and can return more than that limit.
+ /// @warning `VisitFullTextSearch` may return error status when it is incorrectly invoked (e.g.,
+ /// BTreeGlobalIndexReader call `VisitFullTextSearch`) or the query is invalid.
+ virtual Result<std::shared_ptr<ScoredGlobalIndexResult>> VisitFullTextSearch(
const std::shared_ptr<FullTextSearch>& full_text_search) = 0;
/// @return true if the reader is thread-safe; false otherwise.
diff --git a/include/paimon/predicate/full_text_search.h b/include/paimon/predicate/full_text_search.h
index ddb84d64..1adb0875 100644
--- a/include/paimon/predicate/full_text_search.h
+++ b/include/paimon/predicate/full_text_search.h
@@ -17,122 +17,78 @@
*/
#pragma once
+#include <cstdint>
#include <functional>
#include <map>
#include <memory>
#include <optional>
#include <string>
+#include <utility>
#include <vector>
#include "paimon/predicate/predicate.h"
#include "paimon/utils/roaring_bitmap64.h"
#include "paimon/visibility.h"
namespace paimon {
-/// A configuration structure for full-text search operations.
+/// A full-text search on a full-text indexed field, aligned with Java
+/// `org.apache.paimon.predicate.FullTextSearch`.
struct PAIMON_EXPORT FullTextSearch {
- /// Enumeration of supported full-text search types.
- enum class SearchType {
- /// All terms in the query must be present (AND semantics).
- MATCH_ALL = 1,
- /// Any term in the query can match (OR semantics).
- MATCH_ANY = 2,
- /// Matches the exact sequence of words (with proximity).
- PHRASE = 3,
- /// Matches terms starting with the given string (e.g., "run*" → running, runner).
- PREFIX = 4,
- /// Supports wildcards * and ? (e.g., "ap*e", "app?e" -> "apple").
- WILDCARD = 5,
- /// Default/fallback type for unrecognized or invalid queries.
- UNKNOWN = 128
- };
-
- FullTextSearch(const std::string& _field_name, std::optional<int32_t> _limit,
- const std::string& _query, const SearchType& _search_type,
- const std::optional<RoaringBitmap64>& _pre_filter, bool _with_score = false,
- std::optional<float> _min_score = std::nullopt)
+ FullTextSearch(const std::string& _field_name, const std::string& _query, int32_t _limit,
+ std::optional<RoaringBitmap64> _include_row_ids = std::nullopt)
: field_name(_field_name),
- limit(_limit),
query(_query),
- search_type(_search_type),
- pre_filter(_pre_filter),
- with_score(_with_score),
- min_score(_min_score) {}
+ limit(_limit),
+ include_row_ids(std::move(_include_row_ids)) {}
- std::shared_ptr<FullTextSearch> ReplacePreFilter(
- const std::optional<RoaringBitmap64>& _pre_filter) const {
- return std::make_shared<FullTextSearch>(field_name, limit, query, search_type, _pre_filter,
- with_score, min_score);
+ /// Returns a copy of this search for an index shard covering the global row ids [`from`, `to`]
+ /// (both inclusive), as Java `FullTextSearch#offsetRange` does: `include_row_ids` is clipped to
+ /// the range and shifted to shard-local row ids by subtracting `from`.
+ std::shared_ptr<FullTextSearch> OffsetRange(int64_t from, int64_t to) const {
+ if (!include_row_ids) {
+ return std::make_shared<FullTextSearch>(*this);
+ }
+ const RoaringBitmap64& global_row_ids = include_row_ids.value();
+ RoaringBitmap64 local_row_ids;
+ for (auto iter = global_row_ids.EqualOrLarger(from), end = global_row_ids.End();
+ iter != end && *iter <= to; ++iter) {
+ local_row_ids.Add(*iter - from);
+ }
+ return std::make_shared<FullTextSearch>(field_name, query, limit, std::move(local_row_ids));
}
/// Name of the field to search within (must be a full-text indexed field).
std::string field_name;
- /// Maximum number of documents to return. Purely a truncation switch,
- /// orthogonal to `with_score`: set `with_score = true` to get relevance
- /// scores; a non-empty `limit` does not by itself imply scoring.
- ///
- /// The `full-text` index requires a positive `limit` and always returns at most `limit` rows
- /// with the highest scores.
- std::optional<int32_t> limit;
- /// The query string to search for. The interpretation depends on search_type:
- ///
- /// - For MATCH_ALL/MATCH_ANY: keywords are split into terms using the **same analyzer as
- /// indexing**.
- /// Example: "Hello World" → terms ["hello", "world"] (after lowercasing and tokenization).
+ /// The JSON DSL query defined by the `paimon-full-text` engine (`core/src/query.rs` in
+ /// apache/paimon-full-text). The root object holds exactly one query:
///
- /// - For PHRASE: matches the exact word sequence (with optional slop). Also be analyzed.
+ /// - `match`: `query` (alias `terms`), optional `column`, `operator` (`Or` by default, or
+ /// `And`), `boost` (1.0), `fuzziness` (0, a number, or `"auto"` or `null` for automatic),
+ /// `max_expansions` (50) and `prefix_length` (0). The text is analyzed with the analyzer
+ /// used at indexing time.
+ /// - `multi_match`: `query`, `columns`, `boosts`, `operator`, `fuzziness`, `max_expansions`
+ /// and `prefix_length`.
+ /// - `match_phrase` (alias `phrase`): `query`, optional `column` and `slop` (0).
+ /// - `boolean`: `must`, `should` and `must_not` lists of queries, and `queries` as
+ /// `[occur, query]` pairs.
+ /// - `boost`: `positive`, `negative` and `negative_boost` (0.5).
///
- /// - For PREFIX: matches terms starting with the given string (e.g., "run" → running, runner).
- /// The query is not tokenized or filtered for stop words. The original prefix is retained,
- /// and a prefix consisting entirely of ASCII letters and digits is also matched using the
- /// lowercase case-normalization applied to pure ASCII terms at indexing time.
+ /// Examples: `{"match":{"query":"paimon lake","operator":"And"}}` and
+ /// `{"match_phrase":{"query":"paimon lake","slop":1}}`.
///
- /// - For WILDCARD: supports wildcards * and ? (e.g., "ap*e", "app?e").
- /// The query is not tokenized or filtered for stop words. The wildcard operators are
- /// preserved. Both the original pattern and an alternative with each ASCII alphanumeric
- /// fragment lowercased are matched, covering pure ASCII and mixed ASCII/non-ASCII terms.
+ /// `field_name` selects the table column. DSL column names refer to fields inside the index:
+ /// the `full-text` backend defaults to `text`, while `lucene-fts` uses `field_name`.
///
- /// - For the `full-text` index, `search_type` is ignored and the query is passed to the
- /// native engine unchanged, so it must be a JSON DSL query such as
- /// `{"match":{"query":"paimon lake"}}`.
- ///
- /// @note Analyzer consistency between indexing and querying is critical for correctness.
+ /// The `full-text` index passes the query to the native engine unchanged. The `lucene-fts`
+ /// index translates `match`, `multi_match`, `match_phrase` and `boolean` into Lucene queries
+ /// and rejects `boost` queries and a `fuzziness` other than 0. An invalid query is reported as
+ /// an error `Status`.
std::string query;
- /// Type of search to perform.
- SearchType search_type;
- /// A pre-filter whose row ID domain is determined by the reader: global row IDs for
- /// GlobalIndexReader and physical file-local row IDs for FileIndexReader.
- /// Only rows whose row ID is present in `pre_filter` will be included during search.
- /// If not set, all rows will be included.
- std::optional<RoaringBitmap64> pre_filter;
- /// Whether to compute and return relevance scores (e.g. BM25). The 4-path matrix:
- /// - `with_score=false, limit=nullopt` → BitmapGlobalIndexResult (all rows, no score)
- /// - `with_score=false, limit=N` → BitmapGlobalIndexResult (any N matches, unscored)
- /// - `with_score=true, limit=nullopt` → BitmapScoredGlobalIndexResult (all rows + all scores)
- /// - `with_score=true, limit=N` → BitmapScoredGlobalIndexResult (top-N by score +
- /// scores)
- ///
- /// For plain `LIMIT N` without ORDER BY (the common case when an online
- /// engine, e.g. StarRocks, pushes down a predicate) set `with_score=false,
- /// limit=N` — the unscored fast path, where the backend provides one. For top-N by relevance
- /// use `with_score=true, limit=N` and drop the scores in the caller if unneeded.
- ///
- /// The built-in indexes deviate from this matrix:
- /// - `lucene` ignores `with_score`: a search with `limit` returns the top `limit` rows with
- /// their scores, and a search without `limit` returns all matching rows without scores.
- /// - `full-text` requires `limit` and always ranks the matching rows by BM25 score, keeping the
- /// top `limit` rows. `with_score` only selects whether their scores are returned.
- ///
- /// Default is `false`, which avoids score computation on backends with an unscored path.
- bool with_score = false;
- /// Minimum relevance-score threshold (exclusive); results with score ≤ this value are
- /// excluded. The score is whatever the backend's similarity produces (e.g. BM25 for
- /// full-text, classic TF-IDF for lucene), so a threshold is not directly comparable across
- /// backends.
- ///
- /// Only the `full-text` index supports it; the `lucene` index rejects a search that sets it.
- /// The `full-text` index applies it to the top `limit` rows returned by the engine, so a
- /// search can return fewer than `limit` rows.
- /// Default is nullopt (no threshold filtering).
- std::optional<float> min_score;
+ /// Maximum number of highest-scoring rows requested from each index shard; must be positive.
+ int32_t limit;
+ /// The row ids to search within, aligned with Java `includeRowIds`. The row id domain is
+ /// determined by the reader: global row ids for GlobalIndexReader and physical file-local row
+ /// ids for FileIndexReader. Only these rows are ranked, so an empty set matches no rows. If not
+ /// set, all rows are searched.
+ std::optional<RoaringBitmap64> include_row_ids;
};
} // namespace paimon
diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt
index 91200e8b..ddd0118e 100644
--- a/src/paimon/CMakeLists.txt
+++ b/src/paimon/CMakeLists.txt
@@ -644,6 +644,7 @@ if(PAIMON_BUILD_TESTS)
common/metrics/histogram_windowing_test.cpp
common/options/memory_size_test.cpp
common/options/time_duration_test.cpp
+ common/predicate/full_text_search_test.cpp
common/predicate/literal_converter_test.cpp
common/predicate/literal_test.cpp
common/predicate/multi_literals_leaf_function_test.cpp
diff --git a/src/paimon/common/file_index/empty/empty_file_index_reader_test.cpp b/src/paimon/common/file_index/empty/empty_file_index_reader_test.cpp
index df607529..6d9eb508 100644
--- a/src/paimon/common/file_index/empty/empty_file_index_reader_test.cpp
+++ b/src/paimon/common/file_index/empty/empty_file_index_reader_test.cpp
@@ -66,8 +66,8 @@ TEST(EmptyFileIndexReaderTest, TestVectorSearchReturnsNoMatches) {
TEST(EmptyFileIndexReaderTest, TestFullTextSearchReturnsNoMatches) {
EmptyFileIndexReader reader;
- auto search = std::make_shared<FullTextSearch>(
- "body", /*limit=*/1, "word", FullTextSearch::SearchType::MATCH_ANY, std::nullopt);
+ auto search =
+ std::make_shared<FullTextSearch>("body", R"({"match":{"query":"word"}})", /*limit=*/1);
ASSERT_OK_AND_ASSIGN(std::shared_ptr<FileIndexResult> result,
reader.VisitFullTextSearch(search));
ASSERT_OK_AND_ASSIGN(bool is_remain, result->IsRemain());
diff --git a/src/paimon/common/file_index/file_index_reader_test.cpp b/src/paimon/common/file_index/file_index_reader_test.cpp
index a7a4c7db..86745c3e 100644
--- a/src/paimon/common/file_index/file_index_reader_test.cpp
+++ b/src/paimon/common/file_index/file_index_reader_test.cpp
@@ -71,8 +71,8 @@ TEST(FileIndexReaderTest, TestDefaultIndexReader) {
std::make_shared<VectorSearch>("embedding", 1, std::vector<float>{1.0f}, nullptr, nullptr,
std::nullopt, std::map<std::string, std::string>{});
ASSERT_NOK_WITH_MSG(reader.VisitVectorSearch(vector_search), "does not support vector search");
- std::shared_ptr<FullTextSearch> full_text_search = std::make_shared<FullTextSearch>(
- "body", std::nullopt, "word", FullTextSearch::SearchType::MATCH_ANY, std::nullopt);
+ std::shared_ptr<FullTextSearch> full_text_search =
+ std::make_shared<FullTextSearch>("body", R"({"match":{"query":"word"}})", /*limit=*/1);
ASSERT_NOK_WITH_MSG(reader.VisitFullTextSearch(full_text_search),
"does not support full text search");
}
diff --git a/src/paimon/common/global_index/bitmap/bitmap_index_reader.cpp b/src/paimon/common/global_index/bitmap/bitmap_index_reader.cpp
index 4e3e9dfe..baa41c9a 100644
--- a/src/paimon/common/global_index/bitmap/bitmap_index_reader.cpp
+++ b/src/paimon/common/global_index/bitmap/bitmap_index_reader.cpp
@@ -147,7 +147,7 @@ Result<std::shared_ptr<ScoredGlobalIndexResult>> BitmapIndexReader::VisitVectorS
return Status::Invalid("Vector search is not supported in BitmapIndexReader.");
}
-Result<std::shared_ptr<GlobalIndexResult>> BitmapIndexReader::VisitFullTextSearch(
+Result<std::shared_ptr<ScoredGlobalIndexResult>> BitmapIndexReader::VisitFullTextSearch(
const std::shared_ptr<FullTextSearch>& full_text_search) {
return Status::Invalid("Full text search is not supported in BitmapIndexReader.");
}
diff --git a/src/paimon/common/global_index/bitmap/bitmap_index_reader.h b/src/paimon/common/global_index/bitmap/bitmap_index_reader.h
index 41890050..350a6dad 100644
--- a/src/paimon/common/global_index/bitmap/bitmap_index_reader.h
+++ b/src/paimon/common/global_index/bitmap/bitmap_index_reader.h
@@ -69,7 +69,7 @@ class BitmapIndexReader : public GlobalIndexReader,
Result<std::shared_ptr<ScoredGlobalIndexResult>> VisitVectorSearch(
const std::shared_ptr<VectorSearch>& vector_search) override;
- Result<std::shared_ptr<GlobalIndexResult>> VisitFullTextSearch(
+ Result<std::shared_ptr<ScoredGlobalIndexResult>> VisitFullTextSearch(
const std::shared_ptr<FullTextSearch>& full_text_search) override;
bool IsThreadSafe() const override {
diff --git a/src/paimon/common/global_index/btree/btree_global_index_reader.cpp b/src/paimon/common/global_index/btree/btree_global_index_reader.cpp
index b9b7f4cf..522f1058 100644
--- a/src/paimon/common/global_index/btree/btree_global_index_reader.cpp
+++ b/src/paimon/common/global_index/btree/btree_global_index_reader.cpp
@@ -272,7 +272,7 @@ Result<std::shared_ptr<ScoredGlobalIndexResult>> BTreeGlobalIndexReader::VisitVe
return Status::Invalid("Vector search not supported in BTree index");
}
-Result<std::shared_ptr<GlobalIndexResult>> BTreeGlobalIndexReader::VisitFullTextSearch(
+Result<std::shared_ptr<ScoredGlobalIndexResult>> BTreeGlobalIndexReader::VisitFullTextSearch(
const std::shared_ptr<FullTextSearch>& full_text_search) {
return Status::Invalid("Full text search not supported in BTree index");
}
diff --git a/src/paimon/common/global_index/btree/btree_global_index_reader.h b/src/paimon/common/global_index/btree/btree_global_index_reader.h
index 39bee677..1e8508b4 100644
--- a/src/paimon/common/global_index/btree/btree_global_index_reader.h
+++ b/src/paimon/common/global_index/btree/btree_global_index_reader.h
@@ -79,7 +79,7 @@ class BTreeGlobalIndexReader : public GlobalIndexReader,
Result<std::shared_ptr<ScoredGlobalIndexResult>> VisitVectorSearch(
const std::shared_ptr<VectorSearch>& vector_search) override;
- Result<std::shared_ptr<GlobalIndexResult>> VisitFullTextSearch(
+ Result<std::shared_ptr<ScoredGlobalIndexResult>> VisitFullTextSearch(
const std::shared_ptr<FullTextSearch>& full_text_search) override;
bool IsThreadSafe() const override {
diff --git a/src/paimon/common/global_index/btree/lazy_filtered_btree_reader.cpp b/src/paimon/common/global_index/btree/lazy_filtered_btree_reader.cpp
index ca5508bf..ff676e98 100644
--- a/src/paimon/common/global_index/btree/lazy_filtered_btree_reader.cpp
+++ b/src/paimon/common/global_index/btree/lazy_filtered_btree_reader.cpp
@@ -180,7 +180,7 @@ Result<std::shared_ptr<ScoredGlobalIndexResult>> LazyFilteredBTreeReader::VisitV
return Status::Invalid("LazyFilteredBTreeReader does not support vector search");
}
-Result<std::shared_ptr<GlobalIndexResult>> LazyFilteredBTreeReader::VisitFullTextSearch(
+Result<std::shared_ptr<ScoredGlobalIndexResult>> LazyFilteredBTreeReader::VisitFullTextSearch(
const std::shared_ptr<FullTextSearch>& full_text_search) {
return Status::Invalid("LazyFilteredBTreeReader does not support full text search");
}
diff --git a/src/paimon/common/global_index/btree/lazy_filtered_btree_reader.h b/src/paimon/common/global_index/btree/lazy_filtered_btree_reader.h
index 46681c1e..32546a47 100644
--- a/src/paimon/common/global_index/btree/lazy_filtered_btree_reader.h
+++ b/src/paimon/common/global_index/btree/lazy_filtered_btree_reader.h
@@ -67,7 +67,7 @@ class LazyFilteredBTreeReader : public GlobalIndexReader {
Result<std::shared_ptr<ScoredGlobalIndexResult>> VisitVectorSearch(
const std::shared_ptr<VectorSearch>& vector_search) override;
- Result<std::shared_ptr<GlobalIndexResult>> VisitFullTextSearch(
+ Result<std::shared_ptr<ScoredGlobalIndexResult>> VisitFullTextSearch(
const std::shared_ptr<FullTextSearch>& full_text_search) override;
bool IsThreadSafe() const override {
diff --git a/src/paimon/common/global_index/offset_global_index_reader.cpp b/src/paimon/common/global_index/offset_global_index_reader.cpp
index 5522a0c4..70b6afc6 100644
--- a/src/paimon/common/global_index/offset_global_index_reader.cpp
+++ b/src/paimon/common/global_index/offset_global_index_reader.cpp
@@ -130,38 +130,22 @@ Result<std::shared_ptr<ScoredGlobalIndexResult>> OffsetGlobalIndexReader::VisitV
return original_filter(local_id + offset);
});
}
- PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<GlobalIndexResult> result,
+ PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<ScoredGlobalIndexResult> result,
wrapped_->VisitVectorSearch(rewritten_search));
- if (result == nullptr) {
- return std::shared_ptr<ScoredGlobalIndexResult>();
- }
- PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<GlobalIndexResult> offset_result,
- result->AddOffset(offset_));
- auto scored_result = std::dynamic_pointer_cast<ScoredGlobalIndexResult>(offset_result);
- if (!scored_result) {
- return Status::Invalid(
- "AddOffset on ScoredGlobalIndexResult did not return ScoredGlobalIndexResult");
- }
- return scored_result;
+ return ApplyScoredOffset(result);
}
-Result<std::shared_ptr<GlobalIndexResult>> OffsetGlobalIndexReader::VisitFullTextSearch(
+Result<std::shared_ptr<ScoredGlobalIndexResult>> OffsetGlobalIndexReader::VisitFullTextSearch(
const std::shared_ptr<FullTextSearch>& full_text_search) {
- // Rewrite pre_filter to convert global ids (used externally) to local ids (used by wrapped_).
- // The original bitmap contains global ids; subtract offset_ to get local ids for the
- // underlying reader.
- std::shared_ptr<FullTextSearch> rewritten_search = full_text_search;
- if (full_text_search && full_text_search->pre_filter.has_value()) {
- RoaringBitmap64 local_bitmap;
- const auto& global_bitmap = full_text_search->pre_filter.value();
- for (auto iter = global_bitmap.Begin(); iter != global_bitmap.End(); ++iter) {
- local_bitmap.Add(*iter - offset_);
- }
- rewritten_search = full_text_search->ReplacePreFilter(std::move(local_bitmap));
+ // Convert global include row ids to local row ids. The shard end is unknown here, so row ids
+ // past the shard are kept; they match no local row.
+ std::shared_ptr<FullTextSearch> local_search = full_text_search;
+ if (full_text_search && full_text_search->include_row_ids) {
+ local_search = full_text_search->OffsetRange(offset_, RoaringBitmap64::MAX_VALUE);
}
- PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<GlobalIndexResult> result,
- wrapped_->VisitFullTextSearch(rewritten_search));
- return ApplyOffset(result);
+ PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<ScoredGlobalIndexResult> result,
+ wrapped_->VisitFullTextSearch(local_search));
+ return ApplyScoredOffset(result);
}
Result<std::shared_ptr<GlobalIndexResult>> OffsetGlobalIndexReader::ApplyOffset(
@@ -172,4 +156,19 @@ Result<std::shared_ptr<GlobalIndexResult>> OffsetGlobalIndexReader::ApplyOffset(
return result->AddOffset(offset_);
}
+Result<std::shared_ptr<ScoredGlobalIndexResult>> OffsetGlobalIndexReader::ApplyScoredOffset(
+ const std::shared_ptr<ScoredGlobalIndexResult>& result) {
+ if (result == nullptr) {
+ return result;
+ }
+ PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<GlobalIndexResult> offset_result,
+ result->AddOffset(offset_));
+ auto scored_result = std::dynamic_pointer_cast<ScoredGlobalIndexResult>(offset_result);
+ if (!scored_result) {
+ return Status::Invalid(
+ "AddOffset on ScoredGlobalIndexResult did not return ScoredGlobalIndexResult");
+ }
+ return scored_result;
+}
+
} // namespace paimon
diff --git a/src/paimon/common/global_index/offset_global_index_reader.h b/src/paimon/common/global_index/offset_global_index_reader.h
index 21b4ebfe..08229466 100644
--- a/src/paimon/common/global_index/offset_global_index_reader.h
+++ b/src/paimon/common/global_index/offset_global_index_reader.h
@@ -57,7 +57,7 @@ class OffsetGlobalIndexReader : public GlobalIndexReader {
Result<std::shared_ptr<ScoredGlobalIndexResult>> VisitVectorSearch(
const std::shared_ptr<VectorSearch>& vector_search) override;
- Result<std::shared_ptr<GlobalIndexResult>> VisitFullTextSearch(
+ Result<std::shared_ptr<ScoredGlobalIndexResult>> VisitFullTextSearch(
const std::shared_ptr<FullTextSearch>& full_text_search) override;
bool IsThreadSafe() const override {
@@ -72,6 +72,9 @@ class OffsetGlobalIndexReader : public GlobalIndexReader {
Result<std::shared_ptr<GlobalIndexResult>> ApplyOffset(
const std::shared_ptr<GlobalIndexResult>& result);
+ Result<std::shared_ptr<ScoredGlobalIndexResult>> ApplyScoredOffset(
+ const std::shared_ptr<ScoredGlobalIndexResult>& result);
+
private:
std::shared_ptr<GlobalIndexReader> wrapped_;
int64_t offset_;
diff --git a/src/paimon/common/global_index/offset_global_index_reader_test.cpp b/src/paimon/common/global_index/offset_global_index_reader_test.cpp
index 7f759917..39fec0b1 100644
--- a/src/paimon/common/global_index/offset_global_index_reader_test.cpp
+++ b/src/paimon/common/global_index/offset_global_index_reader_test.cpp
@@ -216,44 +216,44 @@ TEST_F(OffsetGlobalIndexReaderTest, TestGetIndexTypeDelegated) {
TEST_F(OffsetGlobalIndexReaderTest, TestVisitFullTextSearchWithOffset) {
auto fake_reader = std::make_shared<MockGlobalIndexReader>();
- fake_reader->SetDefaultResult({0, 3, 5});
+ fake_reader->SetScoredResult({0, 3, 5}, {0.5f, 0.9f, 0.1f});
auto offset_reader = std::make_shared<OffsetGlobalIndexReader>(fake_reader, 10);
ASSERT_OK_AND_ASSIGN(auto result, offset_reader->VisitFullTextSearch(nullptr));
// row ids {0, 3, 5} + offset 10 -> {10, 13, 15}
- CheckResult(result, {10, 13, 15});
+ CheckScoredResult(result, {10, 13, 15}, {0.5f, 0.9f, 0.1f});
}
-TEST_F(OffsetGlobalIndexReaderTest, TestVisitFullTextSearchPreservesScoreFlags) {
- // Regression (review finding #2): rewriting the pre_filter global->local ids
- // in the offset reader must NOT drop with_score / min_score. Before the fix,
- // FullTextSearch::ReplacePreFilter rebuilt via the 5-arg ctor and silently
- // reset both back to their defaults, turning a scored / min_score query
- // unscored as soon as it crossed any offset shard.
+TEST_F(OffsetGlobalIndexReaderTest, TestVisitFullTextSearchConvertsIncludeRowIds) {
auto fake_reader = std::make_shared<MockGlobalIndexReader>();
- fake_reader->SetDefaultResult({0, 3, 5});
auto offset_reader = std::make_shared<OffsetGlobalIndexReader>(fake_reader, 10);
-
- // pre_filter must be set so the offset reader takes the rewrite path.
- auto fts = std::make_shared<FullTextSearch>(
- "f0", /*limit=*/7, "q", FullTextSearch::SearchType::MATCH_ALL,
- /*pre_filter=*/RoaringBitmap64::From({10l, 13l, 15l}));
- fts->with_score = true;
- fts->min_score = 1.5f;
-
- ASSERT_OK_AND_ASSIGN(auto result, offset_reader->VisitFullTextSearch(fts));
- CheckResult(result, {10, 13, 15});
-
- auto captured_fts = fake_reader->CapturedFullTextSearch();
- ASSERT_TRUE(captured_fts);
- ASSERT_TRUE(captured_fts->with_score) << "with_score must survive the pre_filter rewrite";
- ASSERT_TRUE(captured_fts->min_score.has_value())
- << "min_score must survive the pre_filter rewrite";
- ASSERT_FLOAT_EQ(captured_fts->min_score.value(), 1.5f);
- // limit and the offset-rewritten local pre_filter should still be present.
- ASSERT_EQ(captured_fts->limit, std::optional<int32_t>(7));
- ASSERT_TRUE(captured_fts->pre_filter.has_value());
+ {
+ auto search =
+ std::make_shared<FullTextSearch>("f0", R"({"match":{"query":"paimon"}})", /*limit=*/7,
+ RoaringBitmap64::From({5l, 10l, 13l, 15l}));
+ ASSERT_OK(offset_reader->VisitFullTextSearch(search));
+ auto captured = fake_reader->CapturedFullTextSearch();
+ ASSERT_TRUE(captured);
+ ASSERT_EQ(captured->field_name, "f0");
+ ASSERT_EQ(captured->query, R"({"match":{"query":"paimon"}})");
+ ASSERT_EQ(captured->limit, 7);
+ ASSERT_EQ(captured->include_row_ids, RoaringBitmap64::From({0l, 3l, 5l}));
+ ASSERT_EQ(search->include_row_ids, RoaringBitmap64::From({5l, 10l, 13l, 15l}));
+ }
+ {
+ auto search = std::make_shared<FullTextSearch>("f0", R"({"match":{"query":"paimon"}})",
+ /*limit=*/7, RoaringBitmap64::From({5l}));
+ ASSERT_OK(offset_reader->VisitFullTextSearch(search));
+ ASSERT_TRUE(fake_reader->CapturedFullTextSearch()->include_row_ids);
+ ASSERT_TRUE(fake_reader->CapturedFullTextSearch()->include_row_ids->IsEmpty());
+ }
+ {
+ auto search =
+ std::make_shared<FullTextSearch>("f0", R"({"match":{"query":"paimon"}})", /*limit=*/7);
+ ASSERT_OK(offset_reader->VisitFullTextSearch(search));
+ ASSERT_EQ(fake_reader->CapturedFullTextSearch(), search);
+ }
}
TEST_F(OffsetGlobalIndexReaderTest, TestVisitVectorSearchWithOffset) {
diff --git a/src/paimon/common/global_index/union_global_index_reader.cpp b/src/paimon/common/global_index/union_global_index_reader.cpp
index 1b52d0a9..39d4002d 100644
--- a/src/paimon/common/global_index/union_global_index_reader.cpp
+++ b/src/paimon/common/global_index/union_global_index_reader.cpp
@@ -126,11 +126,21 @@ Result<std::shared_ptr<GlobalIndexResult>> UnionGlobalIndexReader::VisitLike(
Result<std::shared_ptr<ScoredGlobalIndexResult>> UnionGlobalIndexReader::VisitVectorSearch(
const std::shared_ptr<VectorSearch>& vector_search) {
- auto results = ExecuteAllReaders<Result<std::shared_ptr<ScoredGlobalIndexResult>>>(
- [vector_search](const std::shared_ptr<GlobalIndexReader>& reader)
- -> Result<std::shared_ptr<ScoredGlobalIndexResult>> {
- return reader->VisitVectorSearch(vector_search);
- });
+ return UnionScored([vector_search](const std::shared_ptr<GlobalIndexReader>& reader) {
+ return reader->VisitVectorSearch(vector_search);
+ });
+}
+
+Result<std::shared_ptr<ScoredGlobalIndexResult>> UnionGlobalIndexReader::VisitFullTextSearch(
+ const std::shared_ptr<FullTextSearch>& full_text_search) {
+ return UnionScored([full_text_search](const std::shared_ptr<GlobalIndexReader>& reader) {
+ return reader->VisitFullTextSearch(full_text_search);
+ });
+}
+
+Result<std::shared_ptr<ScoredGlobalIndexResult>> UnionGlobalIndexReader::UnionScored(
+ ScoredReaderAction action) {
+ auto results = ExecuteAllReaders<Result<std::shared_ptr<ScoredGlobalIndexResult>>>(action);
std::shared_ptr<ScoredGlobalIndexResult> merged_result = nullptr;
for (auto& result_or_status : results) {
@@ -156,13 +166,6 @@ Result<std::shared_ptr<ScoredGlobalIndexResult>> UnionGlobalIndexReader::VisitVe
return merged_result;
}
-Result<std::shared_ptr<GlobalIndexResult>> UnionGlobalIndexReader::VisitFullTextSearch(
- const std::shared_ptr<FullTextSearch>& full_text_search) {
- return Union([full_text_search](const std::shared_ptr<GlobalIndexReader>& reader) {
- return reader->VisitFullTextSearch(full_text_search);
- });
-}
-
Result<std::shared_ptr<GlobalIndexResult>> UnionGlobalIndexReader::Union(ReaderAction action) {
auto results = ExecuteAllReaders<Result<std::shared_ptr<GlobalIndexResult>>>(action);
diff --git a/src/paimon/common/global_index/union_global_index_reader.h b/src/paimon/common/global_index/union_global_index_reader.h
index efd42df5..420654fb 100644
--- a/src/paimon/common/global_index/union_global_index_reader.h
+++ b/src/paimon/common/global_index/union_global_index_reader.h
@@ -58,7 +58,7 @@ class UnionGlobalIndexReader : public GlobalIndexReader {
Result<std::shared_ptr<ScoredGlobalIndexResult>> VisitVectorSearch(
const std::shared_ptr<VectorSearch>& vector_search) override;
- Result<std::shared_ptr<GlobalIndexResult>> VisitFullTextSearch(
+ Result<std::shared_ptr<ScoredGlobalIndexResult>> VisitFullTextSearch(
const std::shared_ptr<FullTextSearch>& full_text_search) override;
bool IsThreadSafe() const override;
@@ -71,9 +71,15 @@ class UnionGlobalIndexReader : public GlobalIndexReader {
using ReaderAction = std::function<Result<std::shared_ptr<GlobalIndexResult>>(
const std::shared_ptr<GlobalIndexReader>&)>;
+ using ScoredReaderAction = std::function<Result<std::shared_ptr<ScoredGlobalIndexResult>>(
+ const std::shared_ptr<GlobalIndexReader>&)>;
+
/// Executes the given action on all readers and merges results with Union.
Result<std::shared_ptr<GlobalIndexResult>> Union(ReaderAction action);
+ /// Executes the given action on all readers and merges the scored results with Union.
+ Result<std::shared_ptr<ScoredGlobalIndexResult>> UnionScored(ScoredReaderAction action);
+
/// Executes the given action on all readers (parallel or sequential) and collects results.
template <typename R>
std::vector<R> ExecuteAllReaders(
diff --git a/src/paimon/common/global_index/union_global_index_reader_test.cpp b/src/paimon/common/global_index/union_global_index_reader_test.cpp
index 25158602..faffb44a 100644
--- a/src/paimon/common/global_index/union_global_index_reader_test.cpp
+++ b/src/paimon/common/global_index/union_global_index_reader_test.cpp
@@ -351,14 +351,15 @@ TEST_F(UnionGlobalIndexReaderTest, TestVisitIsNullUnion) {
TEST_F(UnionGlobalIndexReaderTest, TestVisitFullTextSearchUnion) {
auto reader1 = std::make_shared<MockGlobalIndexReader>();
auto reader2 = std::make_shared<MockGlobalIndexReader>();
- reader1->SetDefaultResult({1, 5});
- reader2->SetDefaultResult({2, 6});
+ auto reader3 = std::make_shared<MockGlobalIndexReader>();
+ reader1->SetScoredResult({1, 5}, {0.9f, 0.5f});
+ reader3->SetScoredResult({2, 6}, {0.8f, 0.6f});
- std::vector<std::shared_ptr<GlobalIndexReader>> readers = {reader1, reader2};
+ std::vector<std::shared_ptr<GlobalIndexReader>> readers = {reader1, reader2, reader3};
UnionGlobalIndexReader union_reader(std::move(readers), nullptr);
ASSERT_OK_AND_ASSIGN(auto result, union_reader.VisitFullTextSearch(nullptr));
- CheckResult(result, {1, 2, 5, 6});
+ CheckScoredResult(result, {1, 2, 5, 6}, {0.9f, 0.8f, 0.5f, 0.6f});
}
TEST_F(UnionGlobalIndexReaderTest, TestVisitVectorSearchAllNullptr) {
diff --git a/src/paimon/common/global_index/wrap/file_index_reader_wrapper.h b/src/paimon/common/global_index/wrap/file_index_reader_wrapper.h
index d47a7744..8b2154b9 100644
--- a/src/paimon/common/global_index/wrap/file_index_reader_wrapper.h
+++ b/src/paimon/common/global_index/wrap/file_index_reader_wrapper.h
@@ -132,10 +132,10 @@ class FileIndexReaderWrapper : public GlobalIndexReader {
"FileIndexReaderWrapper is not supposed to handle vector search query");
}
- Result<std::shared_ptr<GlobalIndexResult>> VisitFullTextSearch(
+ /// File index full-text search returns unscored rows, so there is no scored result here.
+ Result<std::shared_ptr<ScoredGlobalIndexResult>> VisitFullTextSearch(
const std::shared_ptr<FullTextSearch>& full_text_search) override {
- std::shared_ptr<FileIndexResult> remain = FileIndexResult::Remain();
- return transform_(remain);
+ return std::shared_ptr<ScoredGlobalIndexResult>();
}
/// Converts a `FileIndexResult` to a `GlobalIndexResult` by mapping 32-bit row IDs
diff --git a/src/paimon/common/predicate/full_text_search_test.cpp b/src/paimon/common/predicate/full_text_search_test.cpp
new file mode 100644
index 00000000..2e14dbd3
--- /dev/null
+++ b/src/paimon/common/predicate/full_text_search_test.cpp
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "paimon/predicate/full_text_search.h"
+
+#include <cstdint>
+#include <memory>
+#include <vector>
+
+#include "gtest/gtest.h"
+#include "paimon/utils/roaring_bitmap64.h"
+
+namespace paimon::test {
+
+TEST(FullTextSearchTest, TestOffsetRangeWithoutIncludeRowIds) {
+ FullTextSearch search("f0", R"({"match":{"query":"paimon"}})", /*limit=*/5);
+ ASSERT_FALSE(search.include_row_ids);
+
+ std::shared_ptr<FullTextSearch> shard_search = search.OffsetRange(100, 199);
+ ASSERT_TRUE(shard_search);
+ ASSERT_EQ(shard_search->field_name, "f0");
+ ASSERT_EQ(shard_search->query, R"({"match":{"query":"paimon"}})");
+ ASSERT_EQ(shard_search->limit, 5);
+ ASSERT_FALSE(shard_search->include_row_ids);
+}
+
+TEST(FullTextSearchTest, TestOffsetRangeWithIncludeRowIds) {
+ FullTextSearch search("f0", R"({"match":{"query":"paimon"}})", /*limit=*/5,
+ RoaringBitmap64::From({5l, 99l, 100l, 150l, 199l, 200l}));
+
+ std::shared_ptr<FullTextSearch> shard_search = search.OffsetRange(100, 199);
+ ASSERT_EQ(shard_search->field_name, "f0");
+ ASSERT_EQ(shard_search->query, R"({"match":{"query":"paimon"}})");
+ ASSERT_EQ(shard_search->limit, 5);
+ ASSERT_EQ(shard_search->include_row_ids, RoaringBitmap64::From({0l, 50l, 99l}));
+ ASSERT_EQ(search.OffsetRange(0, 99)->include_row_ids, RoaringBitmap64::From({5l, 99l}));
+ ASSERT_EQ(search.OffsetRange(150, 150)->include_row_ids, RoaringBitmap64::From({0l}));
+ ASSERT_EQ(search.OffsetRange(300, 399)->include_row_ids, RoaringBitmap64());
+ ASSERT_EQ(search.include_row_ids, RoaringBitmap64::From({5l, 99l, 100l, 150l, 199l, 200l}));
+
+ const int64_t from = int64_t{1} << 40;
+ FullTextSearch large_search("f0", R"({"match":{"query":"paimon"}})", /*limit=*/5,
+ RoaringBitmap64::From(std::vector<int64_t>{3, from + 3, from + 7}));
+ ASSERT_EQ(large_search.OffsetRange(from, RoaringBitmap64::MAX_VALUE)->include_row_ids,
+ RoaringBitmap64::From({3l, 7l}));
+}
+
+} // namespace paimon::test
diff --git a/src/paimon/core/io/file_index_evaluator_test.cpp b/src/paimon/core/io/file_index_evaluator_test.cpp
index 2be597ab..adeb5a39 100644
--- a/src/paimon/core/io/file_index_evaluator_test.cpp
+++ b/src/paimon/core/io/file_index_evaluator_test.cpp
@@ -479,9 +479,8 @@ class FileIndexSearchEvaluatorTest : public FileIndexEvaluatorTest,
}
std::shared_ptr<FullTextSearch> MakeFullTextSearch(const std::string& field) const {
- return std::make_shared<FullTextSearch>(field, /*limit=*/2, "paimon",
- FullTextSearch::SearchType::MATCH_ALL,
- /*pre_filter=*/std::nullopt);
+ return std::make_shared<FullTextSearch>(field, R"({"match":{"query":"paimon"}})",
+ /*limit=*/2);
}
void CheckSearchError(SearchIndexType index_type, const std::shared_ptr<DataFileMeta>& meta,
diff --git a/src/paimon/core/operation/internal_read_context.cpp b/src/paimon/core/operation/internal_read_context.cpp
index 66312d13..429302cc 100644
--- a/src/paimon/core/operation/internal_read_context.cpp
+++ b/src/paimon/core/operation/internal_read_context.cpp
@@ -212,10 +212,6 @@ Result<std::unique_ptr<InternalReadContext>> InternalReadContext::Create(
return Status::Invalid(
"File Index read must be enabled for VectorSearch or FullTextSearch");
}
- if (context->GetFullTextSearch() && context->GetFullTextSearch()->with_score) {
- return Status::NotImplemented(
- "File full-text search does not support score output yet");
- }
if (!table_schema->PrimaryKeys().empty() && !core_options.DeletionVectorsEnabled()) {
return Status::NotImplemented(
"VectorSearch or FullTextSearch on primary-key tables requires deletion vectors");
@@ -306,6 +302,10 @@ Result<std::unique_ptr<InternalReadContext>> InternalReadContext::Create(
} else {
PAIMON_RETURN_NOT_OK(ArrowSchemaValidator::ValidateSchemaWithFieldId(*read_schema));
}
+ if (context->GetFullTextSearch() &&
+ read_schema->GetFieldIndex(SpecialFields::IndexScore().Name()) >= 0) {
+ return Status::NotImplemented("File full-text search does not support score output yet");
+ }
// validate predicate
if (context->GetPredicate()) {
PAIMON_RETURN_NOT_OK(PredicateValidator::ValidatePredicateWithSchema(
diff --git a/src/paimon/core/operation/internal_read_context_test.cpp b/src/paimon/core/operation/internal_read_context_test.cpp
index 8878a27a..9e0dc5fb 100644
--- a/src/paimon/core/operation/internal_read_context_test.cpp
+++ b/src/paimon/core/operation/internal_read_context_test.cpp
@@ -229,11 +229,10 @@ TEST(InternalReadContext, TestFileFullTextSearchScoreIsNotSupported) {
std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09";
SchemaManager schema_manager(std::make_shared<LocalFileSystem>(), path);
ASSERT_OK_AND_ASSIGN(std::shared_ptr<TableSchema> table_schema, schema_manager.ReadSchema(0));
- auto search = std::make_shared<FullTextSearch>(
- "f0", /*limit=*/1, "query", FullTextSearch::SearchType::MATCH_ALL,
- /*pre_filter=*/std::nullopt, /*with_score=*/true);
+ auto search =
+ std::make_shared<FullTextSearch>("f0", R"({"match":{"query":"query"}})", /*limit=*/1);
ReadContextBuilder builder(path);
- builder.SetReadFieldNames({"f0"}).SetFullTextSearch(search);
+ builder.SetReadFieldNames({"f0", "_INDEX_SCORE"}).SetFullTextSearch(search);
ASSERT_OK_AND_ASSIGN(std::shared_ptr<ReadContext> context, builder.Finish());
ASSERT_NOK_WITH_MSG(InternalReadContext::Create(context, table_schema, table_schema->Options()),
"does not support score output");
diff --git a/src/paimon/core/operation/raw_file_split_read.cpp b/src/paimon/core/operation/raw_file_split_read.cpp
index e4d1c9d6..5f75ea25 100644
--- a/src/paimon/core/operation/raw_file_split_read.cpp
+++ b/src/paimon/core/operation/raw_file_split_read.cpp
@@ -316,13 +316,15 @@ Result<std::optional<RoaringBitmap32>> RawFileSplitRead::EvaluateFullTextSearchW
RoaringBitmap64 eligible;
eligible.AddRange(0, file->row_count);
eligible -= RoaringBitmap64(*deletion);
- if (full_text_search->pre_filter) {
- eligible &= full_text_search->pre_filter.value();
+ if (full_text_search->include_row_ids) {
+ eligible &= full_text_search->include_row_ids.value();
}
if (eligible.IsEmpty()) {
return std::optional<RoaringBitmap32>();
}
- search_with_dv = full_text_search->ReplacePreFilter(eligible);
+ search_with_dv =
+ std::make_shared<FullTextSearch>(full_text_search->field_name, full_text_search->query,
+ full_text_search->limit, std::move(eligible));
}
PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<FileIndexResult> search_result,
FileIndexEvaluator::EvaluateFullTextSearch(
diff --git a/src/paimon/core/operation/raw_file_split_read_test.cpp b/src/paimon/core/operation/raw_file_split_read_test.cpp
index 7ee9e5fc..c6ad7f98 100644
--- a/src/paimon/core/operation/raw_file_split_read_test.cpp
+++ b/src/paimon/core/operation/raw_file_split_read_test.cpp
@@ -545,8 +545,8 @@ TEST_F(RawFileSplitReadTest, TestFullTextSearchFiltersDeletionVectorBeforeLimit)
RoaringBitmap64 user_filter;
user_filter.Add(0);
user_filter.Add(2);
- auto search = std::make_shared<FullTextSearch>(
- "f3", /*limit=*/1, "match", FullTextSearch::SearchType::MATCH_ALL, user_filter);
+ auto search = std::make_shared<FullTextSearch>("f3", R"({"match":{"query":"match"}})",
+ /*limit=*/1, user_filter);
ReadContextBuilder builder(path);
builder.SetReadFieldNames({"f0"}).SetFullTextSearch(search);
ASSERT_OK_AND_ASSIGN(std::shared_ptr<ReadContext> read_context, builder.Finish());
@@ -657,8 +657,8 @@ TEST_F(RawFileSplitReadTest, TestFullTextSearchRenamedUnprojectedFieldInOldFile)
ASSERT_OK_AND_ASSIGN(std::shared_ptr<TableSchema> current_schema,
TableSchema::Create(1, DataField::ConvertDataFieldsToArrowSchema(fields),
original_schema->PartitionKeys(), {}, options));
- auto search = std::make_shared<FullTextSearch>(
- "renamed_f3", 2, "match", FullTextSearch::SearchType::MATCH_ALL, std::nullopt);
+ auto search =
+ std::make_shared<FullTextSearch>("renamed_f3", R"({"match":{"query":"match"}})", 2);
ReadContextBuilder builder(path);
builder.SetOptions(options).SetReadFieldNames({"f0"}).SetFullTextSearch(search);
ASSERT_OK_AND_ASSIGN(std::shared_ptr<ReadContext> context, builder.Finish());
@@ -680,7 +680,7 @@ TEST_F(RawFileSplitReadTest, TestFullTextSearchRenamedUnprojectedFieldInOldFile)
<< actual->ToString();
}
ASSERT_EQ("renamed_f3", search->field_name);
- ASSERT_FALSE(search->pre_filter);
+ ASSERT_FALSE(search->include_row_ids);
ASSERT_EQ("current-options",
current_schema->Options().at("file-index.raw-options-full-text.renamed_f3.token"));
diff --git a/src/paimon/core/operation/read_context_test.cpp b/src/paimon/core/operation/read_context_test.cpp
index 7ea998a0..b820d245 100644
--- a/src/paimon/core/operation/read_context_test.cpp
+++ b/src/paimon/core/operation/read_context_test.cpp
@@ -290,8 +290,8 @@ TEST(ReadContextTest, TestFileIndexSearchConfigurationAndReset) {
EXPECT_FALSE(vector_context->GetFullTextSearch());
EXPECT_TRUE(vector_context->HasVectorOrFullTextSearch());
- std::shared_ptr<FullTextSearch> full_text_search = std::make_shared<FullTextSearch>(
- "body", /*limit=*/5, "paimon", FullTextSearch::SearchType::MATCH_ANY, std::nullopt);
+ std::shared_ptr<FullTextSearch> full_text_search =
+ std::make_shared<FullTextSearch>("body", R"({"match":{"query":"paimon"}})", /*limit=*/5);
ASSERT_EQ(&builder, &builder.SetFullTextSearch(full_text_search));
ASSERT_OK_AND_ASSIGN(std::unique_ptr<ReadContext> full_text_context, builder.Finish());
EXPECT_FALSE(full_text_context->GetVectorSearch());
@@ -307,8 +307,8 @@ TEST(ReadContextTest, TestRejectConflictingFileIndexSearch) {
std::shared_ptr<VectorSearch> vector_search =
std::make_shared<VectorSearch>("embedding", /*limit=*/3, std::vector<float>{1.0f}, nullptr,
nullptr, std::nullopt, std::map<std::string, std::string>{});
- std::shared_ptr<FullTextSearch> full_text_search = std::make_shared<FullTextSearch>(
- "body", std::nullopt, "paimon", FullTextSearch::SearchType::MATCH_ANY, std::nullopt);
+ std::shared_ptr<FullTextSearch> full_text_search =
+ std::make_shared<FullTextSearch>("body", R"({"match":{"query":"paimon"}})", /*limit=*/5);
ReadContextBuilder conflicting("table_root_path");
conflicting.SetVectorSearch(vector_search).SetFullTextSearch(full_text_search);
ASSERT_NOK_WITH_MSG(conflicting.Finish(), "cannot be configured together");
diff --git a/src/paimon/core/table/source/primary_key_sorted_index_scan.cpp b/src/paimon/core/table/source/primary_key_sorted_index_scan.cpp
index a523b920..8d646610 100644
--- a/src/paimon/core/table/source/primary_key_sorted_index_scan.cpp
+++ b/src/paimon/core/table/source/primary_key_sorted_index_scan.cpp
@@ -285,7 +285,7 @@ class FileLocalGroupReader : public GlobalIndexReader {
return Status::Invalid("Primary-key sorted index does not support vector search.");
}
- Result<std::shared_ptr<GlobalIndexResult>> VisitFullTextSearch(
+ Result<std::shared_ptr<ScoredGlobalIndexResult>> VisitFullTextSearch(
const std::shared_ptr<FullTextSearch>& full_text_search) override {
return Status::Invalid("Primary-key sorted index does not support full text search.");
}
diff --git a/src/paimon/core/table/source/table_read_test.cpp b/src/paimon/core/table/source/table_read_test.cpp
index afd0c05e..4f5c03b5 100644
--- a/src/paimon/core/table/source/table_read_test.cpp
+++ b/src/paimon/core/table/source/table_read_test.cpp
@@ -133,8 +133,8 @@ TEST(TableReadTest, TestCreateKeyValueTableRead) {
TEST(TableReadTest, TestCreateCountReaderRejectsFileIndexSearch) {
{
std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09";
- auto search = std::make_shared<FullTextSearch>(
- "f0", /*limit=*/1, "Alice", FullTextSearch::SearchType::MATCH_ALL, std::nullopt);
+ auto search =
+ std::make_shared<FullTextSearch>("f0", R"({"match":{"query":"Alice"}})", /*limit=*/1);
ReadContextBuilder context_builder(path);
context_builder.SetReadFieldNames({"f0"}).SetFullTextSearch(search);
ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish());
diff --git a/src/paimon/global_index/full_text/full_text_global_index_reader.cpp b/src/paimon/global_index/full_text/full_text_global_index_reader.cpp
index 4c6e6152..d2fd883f 100644
--- a/src/paimon/global_index/full_text/full_text_global_index_reader.cpp
+++ b/src/paimon/global_index/full_text/full_text_global_index_reader.cpp
@@ -20,12 +20,10 @@
#include "paimon/global_index/full_text/full_text_global_index_reader.h"
#include <map>
-#include <optional>
#include <utility>
#include "fmt/format.h"
#include "paimon/fs/file_system.h"
-#include "paimon/global_index/bitmap_global_index_result.h"
#include "paimon/global_index/bitmap_scored_global_index_result.h"
#include "paimon/memory/bytes.h"
#include "paimon/utils/roaring_bitmap64.h"
@@ -72,28 +70,12 @@ Status TakeLastError(const std::string& action) {
}
/// Converts the engine output, ordered by descending score, into a result ordered by row id.
-std::shared_ptr<GlobalIndexResult> ToGlobalIndexResult(const std::vector<int64_t>& row_ids,
- const std::vector<float>& scores,
- bool with_score,
- const std::optional<float>& min_score) {
- auto above_min_score = [&](size_t i) { return !min_score || scores[i] > min_score.value(); };
- if (!with_score) {
- RoaringBitmap64 bitmap;
- for (size_t i = 0; i < row_ids.size(); ++i) {
- if (above_min_score(i)) {
- bitmap.Add(row_ids[i]);
- }
- }
- return std::make_shared<BitmapGlobalIndexResult>(
- [bitmap = std::move(bitmap)]() -> Result<RoaringBitmap64> { return bitmap; });
- }
-
+std::shared_ptr<ScoredGlobalIndexResult> ToScoredResult(const std::vector<int64_t>& row_ids,
+ const std::vector<float>& scores) {
// As in Java, a row id that was indexed more than once keeps the score of its last hit.
std::map<int64_t, float> scores_by_row_id;
for (size_t i = 0; i < row_ids.size(); ++i) {
- if (above_min_score(i)) {
- scores_by_row_id[row_ids[i]] = scores[i];
- }
+ scores_by_row_id[row_ids[i]] = scores[i];
}
RoaringBitmap64 bitmap;
std::vector<float> sorted_scores;
@@ -165,12 +147,12 @@ Result<PaimonFtindexReaderHandle*> FullTextGlobalIndexReader::GetOrOpenReader()
return reader_.get();
}
-Result<std::shared_ptr<GlobalIndexResult>> FullTextGlobalIndexReader::VisitFullTextSearch(
+Result<std::shared_ptr<ScoredGlobalIndexResult>> FullTextGlobalIndexReader::VisitFullTextSearch(
const std::shared_ptr<FullTextSearch>& full_text_search) {
if (!full_text_search) {
return Status::Invalid("VisitFullTextSearch: null FullTextSearch pointer");
}
- if (!full_text_search->limit || full_text_search->limit.value() <= 0) {
+ if (full_text_search->limit <= 0) {
return Status::Invalid("full-text index search requires a positive limit");
}
const std::string& query = full_text_search->query;
@@ -179,20 +161,19 @@ Result<std::shared_ptr<GlobalIndexResult>> FullTextGlobalIndexReader::VisitFullT
}
PAIMON_UNIQUE_PTR<Bytes> filter_bytes;
- if (full_text_search->pre_filter) {
- const RoaringBitmap64& pre_filter = full_text_search->pre_filter.value();
- if (pre_filter.IsEmpty()) {
- return ToGlobalIndexResult({}, {}, full_text_search->with_score,
- full_text_search->min_score);
+ if (full_text_search->include_row_ids) {
+ const RoaringBitmap64& include_row_ids = full_text_search->include_row_ids.value();
+ if (include_row_ids.IsEmpty()) {
+ return ToScoredResult({}, {});
}
// Serialize() optimizes the bitmap in place and the same search may be visited by
// concurrent readers, so serialize a copy.
- RoaringBitmap64 filter = pre_filter;
+ RoaringBitmap64 filter = include_row_ids;
filter_bytes = filter.Serialize(pool_.get());
}
PAIMON_ASSIGN_OR_RAISE(PaimonFtindexReaderHandle * reader, GetOrOpenReader());
- auto limit = static_cast<size_t>(full_text_search->limit.value());
+ auto limit = static_cast<size_t>(full_text_search->limit);
std::vector<int64_t> row_ids(limit);
std::vector<float> scores(limit);
size_t result_len = 0;
@@ -211,8 +192,7 @@ Result<std::shared_ptr<GlobalIndexResult>> FullTextGlobalIndexReader::VisitFullT
}
row_ids.resize(result_len);
scores.resize(result_len);
- return ToGlobalIndexResult(row_ids, scores, full_text_search->with_score,
- full_text_search->min_score);
+ return ToScoredResult(row_ids, scores);
}
} // namespace paimon::full_text
diff --git a/src/paimon/global_index/full_text/full_text_global_index_reader.h b/src/paimon/global_index/full_text/full_text_global_index_reader.h
index acb965ba..6fec68a0 100644
--- a/src/paimon/global_index/full_text/full_text_global_index_reader.h
+++ b/src/paimon/global_index/full_text/full_text_global_index_reader.h
@@ -102,7 +102,7 @@ class FullTextGlobalIndexReader : public GlobalIndexReader {
"FullTextGlobalIndexReader is not supposed to handle vector search query");
}
- Result<std::shared_ptr<GlobalIndexResult>> VisitFullTextSearch(
+ Result<std::shared_ptr<ScoredGlobalIndexResult>> VisitFullTextSearch(
const std::shared_ptr<FullTextSearch>& full_text_search) override;
/// The native reader supports concurrent searches, and opening it is guarded by a mutex.
diff --git a/src/paimon/global_index/full_text/full_text_global_index_test.cpp b/src/paimon/global_index/full_text/full_text_global_index_test.cpp
index e61364c0..110b9d65 100644
--- a/src/paimon/global_index/full_text/full_text_global_index_test.cpp
+++ b/src/paimon/global_index/full_text/full_text_global_index_test.cpp
@@ -41,7 +41,6 @@
#include "paimon/common/utils/string_utils.h"
#include "paimon/core/global_index/global_index_file_manager.h"
#include "paimon/fs/local/local_file_system.h"
-#include "paimon/global_index/bitmap_global_index_result.h"
#include "paimon/global_index/bitmap_scored_global_index_result.h"
#include "paimon/global_index/global_indexer_factory.h"
#include "paimon/global_index/io/global_index_file_writer.h"
@@ -59,7 +58,7 @@ namespace {
struct GoldenSearch {
std::string query;
int32_t limit;
- std::optional<std::vector<int64_t>> pre_filter;
+ std::optional<std::vector<int64_t>> include_row_ids;
std::vector<std::pair<int64_t, float>> hits;
};
@@ -257,12 +256,9 @@ class FullTextGlobalIndexTest : public ::testing::Test {
}
static std::shared_ptr<FullTextSearch> MakeSearch(
- const std::string& query, std::optional<int32_t> limit = 10,
- const std::optional<RoaringBitmap64>& pre_filter = std::nullopt, bool with_score = true,
- std::optional<float> min_score = std::nullopt) {
- return std::make_shared<FullTextSearch>("f0", limit, query,
- FullTextSearch::SearchType::UNKNOWN, pre_filter,
- with_score, min_score);
+ const std::string& query, int32_t limit = 10,
+ const std::optional<RoaringBitmap64>& include_row_ids = std::nullopt) {
+ return std::make_shared<FullTextSearch>("f0", query, limit, include_row_ids);
}
static std::string MatchQuery(const std::string& terms) {
@@ -304,13 +300,13 @@ class FullTextGlobalIndexTest : public ::testing::Test {
void CheckGoldenSearches(const GoldenFixture& fixture, const GlobalIndexIOMeta& meta) const {
ASSERT_OK_AND_ASSIGN(std::shared_ptr<GlobalIndexReader> reader, CreateReader({meta}));
for (const auto& search : fixture.searches) {
- std::optional<RoaringBitmap64> pre_filter;
- if (search.pre_filter) {
- pre_filter = RoaringBitmap64::From(search.pre_filter.value());
+ std::optional<RoaringBitmap64> include_row_ids;
+ if (search.include_row_ids) {
+ include_row_ids = RoaringBitmap64::From(search.include_row_ids.value());
}
- ASSERT_OK_AND_ASSIGN(
- std::shared_ptr<GlobalIndexResult> result,
- reader->VisitFullTextSearch(MakeSearch(search.query, search.limit, pre_filter)));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<ScoredGlobalIndexResult> result,
+ reader->VisitFullTextSearch(
+ MakeSearch(search.query, search.limit, include_row_ids)));
auto hits = ToHits(result);
ASSERT_EQ(hits.size(), search.hits.size())
<< fixture.name << ": " << search.query << ": " << result->ToString();
@@ -395,6 +391,10 @@ TEST_F(FullTextGlobalIndexTest, TestWriteAndSearch) {
ASSERT_OK_AND_ASSIGN(auto result,
reader->VisitFullTextSearch(MakeSearch(MatchQuery("THE LAKE"))));
CheckRowIds(result, {0, 2});
+ ASSERT_OK_AND_ASSIGN(
+ result, reader->VisitFullTextSearch(
+ MakeSearch(R"({"multi_match":{"query":"paimon lake","columns":["text"]}})")));
+ CheckRowIds(result, {0, 2});
}
TEST_F(FullTextGlobalIndexTest, TestWriterWithoutRows) {
@@ -483,7 +483,7 @@ TEST_F(FullTextGlobalIndexTest, TestInvalidWrite) {
"full-text index value of row 1 must not contain NUL characters");
}
-TEST_F(FullTextGlobalIndexTest, TestPreFilterLimitAndMinScore) {
+TEST_F(FullTextGlobalIndexTest, TestIncludeRowIdsAndLimit) {
auto array = MakeArray(R"([
["paimon paimon paimon paimon"],
["paimon paimon"],
@@ -526,19 +526,44 @@ TEST_F(FullTextGlobalIndexTest, TestPreFilterLimitAndMinScore) {
MatchQuery("paimon"), 10, RoaringBitmap64())));
CheckRowIds(result, {});
}
- {
- ASSERT_OK_AND_ASSIGN(auto result, reader->VisitFullTextSearch(
- MakeSearch(MatchQuery("paimon"), 10, std::nullopt,
- /*with_score=*/true, all_hits[1].second)));
- CheckRowIds(result, {0});
- }
- {
- ASSERT_OK_AND_ASSIGN(auto result,
- reader->VisitFullTextSearch(MakeSearch(
- MatchQuery("paimon"), 10, std::nullopt, /*with_score=*/false)));
- ASSERT_TRUE(std::dynamic_pointer_cast<BitmapGlobalIndexResult>(result));
- ASSERT_FALSE(std::dynamic_pointer_cast<BitmapScoredGlobalIndexResult>(result));
- CheckRowIds(result, {0, 1, 2});
+}
+
+TEST_F(FullTextGlobalIndexTest, TestNestedBooleanAndBoost) {
+ auto array = MakeArray(R"([
+ ["paimon streaming lake"],
+ ["paimon batch lake"],
+ ["paimon lake"],
+ ["iceberg streaming lake"],
+ ["iceberg batch"]
+ ])");
+ ASSERT_OK_AND_ASSIGN(std::vector<GlobalIndexIOMeta> metas, WriteIndex(/*options=*/{}, array));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<GlobalIndexReader> reader, CreateReader(metas));
+
+ ASSERT_OK_AND_ASSIGN(auto result, reader->VisitFullTextSearch(MakeSearch(R"({"boolean":{
+ "must":[
+ {"match":{"query":"paimon"}},
+ {"boolean":{"should":[
+ {"match":{"query":"streaming"}},{"match":{"query":"batch"}}]}}]}})")));
+ CheckRowIds(result, {0, 1});
+ ASSERT_OK_AND_ASSIGN(result, reader->VisitFullTextSearch(MakeSearch(R"({"boolean":{
+ "queries":[
+ ["must",{"match":{"query":"paimon"}}],
+ ["must_not",{"boolean":{"should":[
+ {"match":{"query":"streaming"}},{"match":{"query":"batch"}}]}}]]}})")));
+ CheckRowIds(result, {2});
+
+ ASSERT_OK_AND_ASSIGN(result, reader->VisitFullTextSearch(
+ MakeSearch(R"({"match":{"query":"lake","boost":1.0}})")));
+ CheckRowIds(result, {0, 1, 2, 3});
+ ASSERT_OK_AND_ASSIGN(
+ auto boosted_result,
+ reader->VisitFullTextSearch(MakeSearch(R"({"match":{"query":"lake","boost":2.0}})")));
+ CheckRowIds(boosted_result, {0, 1, 2, 3});
+ auto hits = ToHits(result);
+ auto boosted_hits = ToHits(boosted_result);
+ ASSERT_EQ(boosted_hits.size(), hits.size());
+ for (size_t i = 0; i < hits.size(); ++i) {
+ ASSERT_FLOAT_EQ(boosted_hits[i].second, 2.0f * hits[i].second) << "row " << hits[i].first;
}
}
@@ -553,10 +578,10 @@ TEST_F(FullTextGlobalIndexTest, TestInvalidRead) {
ASSERT_OK_AND_ASSIGN(std::shared_ptr<GlobalIndexReader> reader, CreateReader(metas));
ASSERT_NOK_WITH_MSG(reader->VisitFullTextSearch(nullptr), "null FullTextSearch");
- ASSERT_NOK_WITH_MSG(reader->VisitFullTextSearch(MakeSearch(MatchQuery("paimon"), std::nullopt)),
- "positive limit");
ASSERT_NOK_WITH_MSG(reader->VisitFullTextSearch(MakeSearch(MatchQuery("paimon"), 0)),
"positive limit");
+ ASSERT_NOK_WITH_MSG(reader->VisitFullTextSearch(MakeSearch(MatchQuery("paimon"), -1)),
+ "positive limit");
ASSERT_NOK_WITH_MSG(
reader->VisitFullTextSearch(MakeSearch(MatchQuery(std::string("pai\0mon", 7)))),
"must not contain NUL characters");
@@ -629,7 +654,7 @@ TEST_F(FullTextGlobalIndexTest, TestConcurrentSearch) {
std::vector<int64_t> expected_ids =
i % 2 == 0 ? std::vector<int64_t>({1}) : std::vector<int64_t>({2});
for (int32_t round = 0; round < kRoundNum; ++round) {
- Result<std::shared_ptr<GlobalIndexResult>> result =
+ Result<std::shared_ptr<ScoredGlobalIndexResult>> result =
reader->VisitFullTextSearch(search);
if (!result.ok()) {
statuses[i] = result.status();
@@ -674,7 +699,7 @@ TEST_F(FullTextGlobalIndexTest, TestIOException) {
CHECK_HOOK_STATUS(metas.status(), i);
ASSERT_OK_AND_ASSIGN(std::shared_ptr<GlobalIndexReader> reader,
CreateReader(metas.value()));
- Result<std::shared_ptr<GlobalIndexResult>> result =
+ Result<std::shared_ptr<ScoredGlobalIndexResult>> result =
reader->VisitFullTextSearch(MakeSearch(MatchQuery("paimon")));
CHECK_HOOK_STATUS(result.status(), i);
CheckRowIds(result.value(), {0});
diff --git a/src/paimon/global_index/lucene/lucene_collector.h b/src/paimon/global_index/lucene/lucene_collector.h
deleted file mode 100644
index e95eacb4..00000000
--- a/src/paimon/global_index/lucene/lucene_collector.h
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#pragma once
-#include "lucene++/LuceneHeaders.h"
-#include "paimon/utils/roaring_bitmap64.h"
-
-namespace paimon::lucene {
-class LuceneCollector : public Lucene::Collector {
- public:
- LuceneCollector() : Lucene::Collector() {}
- void setScorer(const Lucene::ScorerPtr& scorer) override {
- // ignore scorer
- }
- void collect(int32_t doc) override {
- bitmap_.Add(doc_base_ + doc);
- }
- void setNextReader(const Lucene::IndexReaderPtr& reader, int32_t doc_base) override {
- doc_base_ = doc_base;
- }
- bool acceptsDocsOutOfOrder() override {
- return true;
- }
- const RoaringBitmap64& GetBitmap() const {
- return bitmap_;
- }
-
- private:
- RoaringBitmap64 bitmap_;
- int64_t doc_base_ = 0;
-};
-} // namespace paimon::lucene
diff --git a/src/paimon/global_index/lucene/lucene_global_index_reader.cpp b/src/paimon/global_index/lucene/lucene_global_index_reader.cpp
index 88bbfab4..c8f28b9c 100644
--- a/src/paimon/global_index/lucene/lucene_global_index_reader.cpp
+++ b/src/paimon/global_index/lucene/lucene_global_index_reader.cpp
@@ -17,23 +17,473 @@
*/
#include "paimon/global_index/lucene/lucene_global_index_reader.h"
+#include <algorithm>
+#include <cmath>
+#include <functional>
+#include <initializer_list>
+#include <limits>
+#include <tuple>
+#include <utility>
+
#include "arrow/c/bridge.h"
-#include "lucene++/DisjunctionMaxQuery.h"
#include "lucene++/FileUtils.h"
#include "paimon/common/utils/options_utils.h"
#include "paimon/common/utils/path_util.h"
#include "paimon/common/utils/rapidjson_util.h"
-#include "paimon/global_index/bitmap_global_index_result.h"
+#include "paimon/common/utils/string_utils.h"
#include "paimon/global_index/bitmap_scored_global_index_result.h"
#include "paimon/global_index/lucene/jieba_analyzer.h"
-#include "paimon/global_index/lucene/lucene_collector.h"
#include "paimon/global_index/lucene/lucene_defs.h"
#include "paimon/global_index/lucene/lucene_directory.h"
#include "paimon/global_index/lucene/lucene_filter.h"
#include "paimon/global_index/lucene/lucene_utils.h"
#include "paimon/io/data_input_stream.h"
+#include "rapidjson/document.h"
+#include "rapidjson/error/en.h"
namespace paimon::lucene {
+namespace {
+
+std::string JsonString(const rapidjson::Value& value) {
+ return std::string(value.GetString(), value.GetStringLength());
+}
+
+/// Trims Unicode White_Space as Rust `str::trim` does in the native DSL.
+std::wstring TrimDslWhitespace(const std::wstring& value) {
+ constexpr wchar_t kWhitespace[] =
+ L"\t\n\v\f\r \u0085\u00a0\u1680\u2000\u2001\u2002\u2003\u2004\u2005"
+ L"\u2006\u2007\u2008\u2009\u200a\u2028\u2029\u202f\u205f\u3000";
+ size_t first = value.find_first_not_of(kWhitespace);
+ if (first == std::wstring::npos) {
+ return L"";
+ }
+ size_t last = value.find_last_not_of(kWhitespace);
+ return value.substr(first, last - first + 1);
+}
+
+bool TrimmedIsOneOf(const std::string& value, std::initializer_list<const wchar_t*> candidates) {
+ std::wstring trimmed = TrimDslWhitespace(LuceneUtils::StringToWstring(value));
+ for (const wchar_t* candidate : candidates) {
+ if (trimmed == candidate) {
+ return true;
+ }
+ }
+ return false;
+}
+
+/// Translates the JSON DSL query of `FullTextSearch` into a Lucene query on the indexed field.
+///
+/// The DSL is defined by `core/src/query.rs` of apache/paimon-full-text, whose field names,
+/// aliases and defaults are followed. `match`, `multi_match`, `match_phrase` and `boolean` queries
+/// are translated into term, boolean and phrase queries, while `boost` queries and fuzzy matching
+/// are rejected. As in the engine, duplicate known fields are rejected and unknown fields are
+/// ignored.
+class DslQueryTranslator {
+ public:
+ /// Splits text into index terms with the analyzer used at indexing time.
+ using Analyzer = std::function<std::vector<std::wstring>(const std::string&)>;
+
+ DslQueryTranslator(const std::wstring& wfield_name, Analyzer analyzer)
+ : wfield_name_(wfield_name), analyzer_(std::move(analyzer)) {}
+
+ Result<Lucene::QueryPtr> Translate(const std::string& query) const noexcept(false) {
+ // RapidJSON treats a NUL byte as the end of input even when a length is provided.
+ if (query.find('\0') != std::string::npos) {
+ return Status::Invalid("lucene full-text query must not contain NUL characters");
+ }
+ rapidjson::Document document;
+ // Parse without recursion so that deep input cannot exhaust the stack. The document is
+ // also freed by its memory pool without recursion.
+ document.Parse<rapidjson::kParseIterativeFlag>(query.data(), query.size());
+ if (document.HasParseError()) {
+ return Status::Invalid(fmt::format(
+ "invalid full-text query: {} (at offset {})",
+ rapidjson::GetParseError_En(document.GetParseError()), document.GetErrorOffset()));
+ }
+ return TranslateQuery(document, /*depth=*/1);
+ }
+
+ private:
+ /// The maximum nesting depth of the JSON arrays and objects that a query is translated from,
+ /// counted from 1 for the outermost object. As serde_json does in the engine, the 128th level
+ /// is rejected, while unknown fields may nest at any depth.
+ static constexpr int32_t kMaxDepth = 127;
+
+ Result<Lucene::QueryPtr> TranslateQuery(const rapidjson::Value& spec, int32_t depth) const
+ noexcept(false) {
+ if (!spec.IsObject() || spec.MemberCount() != 1) {
+ return Status::Invalid(
+ "full-text query must be a JSON object with exactly one query type");
+ }
+ const auto& member = *spec.MemberBegin();
+ std::string type = JsonString(member.name);
+ const rapidjson::Value& body = member.value;
+ if (!body.IsObject()) {
+ return Status::Invalid(fmt::format("full-text query `{}` must be a JSON object", type));
+ }
+ const int32_t body_depth = depth + 1;
+ PAIMON_RETURN_NOT_OK(CheckDepth(body_depth));
+ if (type == "match") {
+ return TranslateMatch(body);
+ }
+ if (type == "multi_match") {
+ return TranslateMultiMatch(body, body_depth);
+ }
+ if (type == "match_phrase" || type == "phrase") {
+ return TranslateMatchPhrase(body);
+ }
+ if (type == "boolean") {
+ return TranslateBoolean(body, body_depth);
+ }
+ if (type == "boost") {
+ return Status::NotImplemented("lucene full-text search does not support boost queries");
+ }
+ return Status::Invalid(fmt::format("unknown full-text query type `{}`", type));
+ }
+
+ Result<Lucene::QueryPtr> TranslateMatch(const rapidjson::Value& body) const noexcept(false) {
+ PAIMON_RETURN_NOT_OK(CheckColumn(body));
+ PAIMON_ASSIGN_OR_RAISE(std::string terms, GetTerms(body));
+ PAIMON_ASSIGN_OR_RAISE(bool conjunction, IsConjunction(body));
+ PAIMON_ASSIGN_OR_RAISE(float boost, GetBoost(body));
+ PAIMON_RETURN_NOT_OK(CheckFuzzyOptions(body));
+ return BuildMatchQuery(terms, conjunction, boost);
+ }
+
+ /// The index has a single field, so every column must name it. As in the engine, the query
+ /// is a SHOULD clause of one `match` query per column.
+ Result<Lucene::QueryPtr> TranslateMultiMatch(const rapidjson::Value& body,
+ int32_t body_depth) const noexcept(false) {
+ PAIMON_ASSIGN_OR_RAISE(std::string terms, GetTerms(body));
+ PAIMON_ASSIGN_OR_RAISE(const rapidjson::Value* columns, FindField(body, "columns"));
+ if (columns == nullptr || !columns->IsArray()) {
+ return Status::Invalid("multi_match query field `columns` must be an array of strings");
+ }
+ // Also covers `boosts`, which is nested as deep as `columns`.
+ PAIMON_RETURN_NOT_OK(CheckDepth(body_depth + 1));
+ if (columns->Empty()) {
+ return Status::Invalid("multi_match query must contain at least one column");
+ }
+ PAIMON_ASSIGN_OR_RAISE(const rapidjson::Value* boosts, FindField(body, "boosts", "boost"));
+ if (boosts != nullptr && !boosts->IsArray()) {
+ return Status::Invalid("multi_match query field `boosts` must be an array of numbers");
+ }
+ bool has_boosts = boosts != nullptr && !boosts->Empty();
+ if (has_boosts && boosts->Size() != columns->Size()) {
+ return Status::Invalid(
+ fmt::format("multi_match boosts length {} does not match columns length {}",
+ boosts->Size(), columns->Size()));
+ }
+ PAIMON_ASSIGN_OR_RAISE(bool conjunction, IsConjunction(body));
+ PAIMON_RETURN_NOT_OK(CheckFuzzyOptions(body));
+
+ auto multi_match_query = Lucene::newLucene<Lucene::BooleanQuery>();
+ for (rapidjson::SizeType i = 0; i < columns->Size(); ++i) {
+ const rapidjson::Value& column = (*columns)[i];
+ if (!column.IsString()) {
+ return Status::Invalid(
+ "multi_match query field `columns` must be an array of strings");
+ }
+ PAIMON_RETURN_NOT_OK(CheckColumnName(JsonString(column)));
+ float boost = 1.0f;
+ if (has_boosts) {
+ const rapidjson::Value& boost_value = (*boosts)[i];
+ if (!boost_value.IsNumber()) {
+ return Status::Invalid(
+ "multi_match query field `boosts` must be an array of numbers");
+ }
+ PAIMON_ASSIGN_OR_RAISE(boost, ToFloatBoost(boost_value.GetDouble()));
+ }
+ multi_match_query->add(BuildMatchQuery(terms, conjunction, boost),
+ Lucene::BooleanClause::Occur::SHOULD);
+ }
+ Lucene::QueryPtr query = multi_match_query;
+ return query;
+ }
+
+ Result<Lucene::QueryPtr> TranslateMatchPhrase(const rapidjson::Value& body) const
+ noexcept(false) {
+ PAIMON_RETURN_NOT_OK(CheckColumn(body));
+ PAIMON_ASSIGN_OR_RAISE(std::string terms, GetTerms(body));
+ int32_t slop = 0;
+ PAIMON_ASSIGN_OR_RAISE(const rapidjson::Value* slop_value, FindField(body, "slop"));
+ if (slop_value != nullptr) {
+ if (!slop_value->IsUint() ||
+ slop_value->GetUint() >
+ static_cast<uint32_t>(std::numeric_limits<int32_t>::max())) {
+ return Status::Invalid(
+ "match_phrase query field `slop` must be an integer in [0, 2147483647]");
+ }
+ slop = static_cast<int32_t>(slop_value->GetUint());
+ }
+ auto phrase_query = Lucene::newLucene<Lucene::PhraseQuery>();
+ for (const auto& token : analyzer_(terms)) {
+ phrase_query->add(Lucene::newLucene<Lucene::Term>(wfield_name_, token));
+ }
+ phrase_query->setSlop(slop);
+ Lucene::QueryPtr query = phrase_query;
+ return query;
+ }
+
+ Result<Lucene::QueryPtr> TranslateBoolean(const rapidjson::Value& body,
+ int32_t body_depth) const noexcept(false) {
+ // Each clause holds its occurrence, its query and the nesting depth of that query.
+ std::vector<std::tuple<Lucene::BooleanClause::Occur, const rapidjson::Value*, int32_t>>
+ clauses;
+ const int32_t array_depth = body_depth + 1;
+ auto add_clauses = [&](const char* name, Lucene::BooleanClause::Occur occur) -> Status {
+ PAIMON_ASSIGN_OR_RAISE(const rapidjson::Value* children, FindField(body, name));
+ if (children == nullptr) {
+ return Status::OK();
+ }
+ if (!children->IsArray()) {
+ return Status::Invalid(
+ fmt::format("boolean query field `{}` must be an array of queries", name));
+ }
+ PAIMON_RETURN_NOT_OK(CheckDepth(array_depth));
+ for (auto iter = children->Begin(); iter != children->End(); ++iter) {
+ clauses.emplace_back(occur, &(*iter), array_depth + 1);
+ }
+ return Status::OK();
+ };
+ PAIMON_RETURN_NOT_OK(add_clauses("should", Lucene::BooleanClause::Occur::SHOULD));
+ PAIMON_RETURN_NOT_OK(add_clauses("must", Lucene::BooleanClause::Occur::MUST));
+ PAIMON_RETURN_NOT_OK(add_clauses("must_not", Lucene::BooleanClause::Occur::MUST_NOT));
+ PAIMON_ASSIGN_OR_RAISE(const rapidjson::Value* queries, FindField(body, "queries"));
+ if (queries != nullptr) {
+ if (!queries->IsArray()) {
+ return Status::Invalid(
+ "boolean query field `queries` must be an array of [occur, query] pairs");
+ }
+ PAIMON_RETURN_NOT_OK(CheckDepth(array_depth));
+ for (auto iter = queries->Begin(); iter != queries->End(); ++iter) {
+ if (!iter->IsArray() || iter->Size() != 2 || !iter->Begin()->IsString()) {
+ return Status::Invalid(
+ "boolean query field `queries` must be an array of [occur, query] pairs");
+ }
+ PAIMON_RETURN_NOT_OK(CheckDepth(array_depth + 1));
+ PAIMON_ASSIGN_OR_RAISE(Lucene::BooleanClause::Occur occur,
+ ParseOccur(JsonString(*iter->Begin())));
+ clauses.emplace_back(occur, iter->Begin() + 1, array_depth + 2);
+ }
+ }
+
+ if (clauses.empty()) {
+ return Status::Invalid("boolean query must contain at least one clause");
+ }
+ bool has_positive_clause =
+ std::any_of(clauses.begin(), clauses.end(), [](const auto& clause) {
+ return std::get<0>(clause) != Lucene::BooleanClause::Occur::MUST_NOT;
+ });
+ if (!has_positive_clause) {
+ return Status::Invalid("boolean query must contain at least one should or must clause");
+ }
+ auto boolean_query = Lucene::newLucene<Lucene::BooleanQuery>();
+ for (const auto& [occur, child, child_depth] : clauses) {
+ PAIMON_ASSIGN_OR_RAISE(Lucene::QueryPtr child_query,
+ TranslateQuery(*child, child_depth));
+ boolean_query->add(child_query, occur);
+ }
+ Lucene::QueryPtr query = boolean_query;
+ return query;
+ }
+
+ /// Builds a `match` query on the indexed field: one term query per analyzed token, combined
+ /// with MUST for the `And` operator and SHOULD for `Or`. Text without tokens matches no rows.
+ Lucene::QueryPtr BuildMatchQuery(const std::string& terms, bool conjunction, float boost) const
+ noexcept(false) {
+ std::vector<std::wstring> tokens = analyzer_(terms);
+ Lucene::QueryPtr query;
+ if (tokens.size() == 1) {
+ query = Lucene::newLucene<Lucene::TermQuery>(
+ Lucene::newLucene<Lucene::Term>(wfield_name_, tokens[0]));
+ } else {
+ Lucene::BooleanClause::Occur occur = conjunction ? Lucene::BooleanClause::Occur::MUST
+ : Lucene::BooleanClause::Occur::SHOULD;
+ auto boolean_query = Lucene::newLucene<Lucene::BooleanQuery>();
+ for (const auto& token : tokens) {
+ boolean_query->add(Lucene::newLucene<Lucene::TermQuery>(
+ Lucene::newLucene<Lucene::Term>(wfield_name_, token)),
+ occur);
+ }
+ query = boolean_query;
+ }
+ if (boost != 1.0f) {
+ query->setBoost(boost);
+ }
+ return query;
+ }
+
+ static Status CheckDepth(int32_t depth) {
+ if (depth > kMaxDepth) {
+ return Status::Invalid(
+ fmt::format("full-text query exceeds the maximum nesting depth of {}", kMaxDepth));
+ }
+ return Status::OK();
+ }
+
+ Status CheckColumn(const rapidjson::Value& body) const {
+ PAIMON_ASSIGN_OR_RAISE(const rapidjson::Value* column, FindField(body, "column"));
+ if (column == nullptr || column->IsNull()) {
+ return Status::OK();
+ }
+ if (!column->IsString()) {
+ return Status::Invalid("full-text query field `column` must be a string");
+ }
+ return CheckColumnName(JsonString(*column));
+ }
+
+ /// As in the engine, a blank column selects the indexed field.
+ Status CheckColumnName(const std::string& column) const {
+ std::wstring wcolumn = LuceneUtils::StringToWstring(column);
+ // Lucene returns an empty string on decoding failure; it must not select the default field.
+ bool column_decoded = column.empty() || !wcolumn.empty();
+ wcolumn = TrimDslWhitespace(wcolumn);
+ if (!column_decoded || (!wcolumn.empty() && wcolumn != wfield_name_)) {
+ return Status::Invalid(fmt::format(
+ "full-text query column '{}' is not configured for this index", column));
+ }
+ return Status::OK();
+ }
+
+ /// Returns the text of `terms` or its alias `query`.
+ static Result<std::string> GetTerms(const rapidjson::Value& body) {
+ PAIMON_ASSIGN_OR_RAISE(const rapidjson::Value* terms, FindField(body, "terms", "query"));
+ if (terms == nullptr) {
+ return Status::Invalid("full-text query is missing field `query`");
+ }
+ if (!terms->IsString()) {
+ return Status::Invalid("full-text query field `query` must be a string");
+ }
+ return JsonString(*terms);
+ }
+
+ /// Returns true for the `And` operator and false for `Or`, the default.
+ static Result<bool> IsConjunction(const rapidjson::Value& body) {
+ PAIMON_ASSIGN_OR_RAISE(const rapidjson::Value* match_operator, FindField(body, "operator"));
+ if (match_operator == nullptr) {
+ return false;
+ }
+ if (!match_operator->IsString()) {
+ return Status::Invalid("full-text query field `operator` must be a string");
+ }
+ std::string value = JsonString(*match_operator);
+ if (TrimmedIsOneOf(value, {L"Or", L"or", L"OR"})) {
+ return false;
+ }
+ if (TrimmedIsOneOf(value, {L"And", L"and", L"AND"})) {
+ return true;
+ }
+ return Status::Invalid(fmt::format("invalid full-text query operator: {}", value));
+ }
+
+ static Result<Lucene::BooleanClause::Occur> ParseOccur(const std::string& value) {
+ if (TrimmedIsOneOf(value, {L"Should", L"should", L"SHOULD"})) {
+ return Lucene::BooleanClause::Occur::SHOULD;
+ }
+ if (TrimmedIsOneOf(value, {L"Must", L"must", L"MUST"})) {
+ return Lucene::BooleanClause::Occur::MUST;
+ }
+ if (TrimmedIsOneOf(value, {L"MustNot", L"must_not", L"MUST_NOT", L"mustnot", L"MUSTNOT"})) {
+ return Lucene::BooleanClause::Occur::MUST_NOT;
+ }
+ return Status::Invalid(fmt::format("invalid boolean query occur: {}", value));
+ }
+
+ static Result<float> GetBoost(const rapidjson::Value& body) {
+ PAIMON_ASSIGN_OR_RAISE(const rapidjson::Value* boost, FindField(body, "boost"));
+ if (boost == nullptr) {
+ return 1.0f;
+ }
+ if (!boost->IsNumber()) {
+ return Status::Invalid("full-text query field `boost` must be a number");
+ }
+ return ToFloatBoost(boost->GetDouble());
+ }
+
+ /// The native DSL stores boosts as f32 and validates the rounded value.
+ static Result<float> ToFloatBoost(double boost) {
+ const double max_boost = std::numeric_limits<float>::max();
+ const double overflow_threshold =
+ max_boost + (max_boost - std::nextafter(std::numeric_limits<float>::max(), 0.0f)) / 2.0;
+ float rounded_boost = 0.0f;
+ if (std::isfinite(boost) && boost > 0.0 && boost < overflow_threshold) {
+ // Values just above float's maximum round to that maximum. Clamp them before the cast
+ // so the C++ conversion stays within the representable range.
+ rounded_boost = static_cast<float>(std::min(boost, max_boost));
+ }
+ if (rounded_boost <= 0.0f) {
+ return Status::Invalid(fmt::format(
+ "boost must be a finite positive value after float conversion, got {}", boost));
+ }
+ return rounded_boost;
+ }
+
+ /// Validates `fuzziness`, `max_expansions` and `prefix_length` as the engine does. Only exact
+ /// term matching, `fuzziness` 0, is supported.
+ static Status CheckFuzzyOptions(const rapidjson::Value& body) {
+ PAIMON_ASSIGN_OR_RAISE(const rapidjson::Value* fuzziness, FindField(body, "fuzziness"));
+ if (fuzziness != nullptr) {
+ bool is_auto = fuzziness->IsNull() ||
+ (fuzziness->IsString() &&
+ StringUtils::EqualsIgnoreCase(JsonString(*fuzziness), "auto"));
+ if (!is_auto && (!fuzziness->IsUint64() || fuzziness->GetUint64() > 2)) {
+ return Status::Invalid(
+ "match query fuzziness must be auto/null or a value in [0, 2]");
+ }
+ if (is_auto || fuzziness->GetUint64() != 0) {
+ return Status::NotImplemented(
+ "lucene full-text search does not support fuzzy matching, fuzziness must be 0");
+ }
+ }
+ PAIMON_ASSIGN_OR_RAISE(const rapidjson::Value* max_expansions,
+ FindField(body, "max_expansions", "maxExpansions"));
+ if (max_expansions != nullptr &&
+ (!max_expansions->IsUint64() || max_expansions->GetUint64() == 0)) {
+ return Status::Invalid("match query max_expansions must be positive");
+ }
+ PAIMON_ASSIGN_OR_RAISE(const rapidjson::Value* prefix_length,
+ FindField(body, "prefix_length", "prefixLength"));
+ if (prefix_length != nullptr && !prefix_length->IsUint()) {
+ return Status::Invalid("match query prefix_length must be a non-negative integer");
+ }
+ return Status::OK();
+ }
+
+ /// Rejects repeated known fields while leaving unknown fields ignored, as Serde does.
+ static Result<const rapidjson::Value*> FindField(const rapidjson::Value& body,
+ const char* name) {
+ const rapidjson::Value* value = nullptr;
+ for (auto iter = body.MemberBegin(); iter != body.MemberEnd(); ++iter) {
+ if (iter->name != name) {
+ continue;
+ }
+ if (value != nullptr) {
+ return Status::Invalid(fmt::format("duplicate full-text query field `{}`", name));
+ }
+ value = &iter->value;
+ }
+ return value;
+ }
+
+ /// Returns the field `name` or its alias, or nullptr if neither is present.
+ static Result<const rapidjson::Value*> FindField(const rapidjson::Value& body, const char* name,
+ const char* alias) {
+ PAIMON_ASSIGN_OR_RAISE(const rapidjson::Value* value, FindField(body, name));
+ PAIMON_ASSIGN_OR_RAISE(const rapidjson::Value* alias_value, FindField(body, alias));
+ if (value != nullptr && alias_value != nullptr) {
+ return Status::Invalid(fmt::format(
+ "full-text query sets both field `{}` and its alias `{}`", name, alias));
+ }
+ return value != nullptr ? value : alias_value;
+ }
+
+ const std::wstring& wfield_name_;
+ Analyzer analyzer_;
+};
+
+} // namespace
+
Result<std::shared_ptr<LuceneGlobalIndexReader>> LuceneGlobalIndexReader::Create(
const std::string& field_name, const GlobalIndexIOMeta& io_meta,
const std::shared_ptr<GlobalIndexFileReader>& file_reader,
@@ -119,111 +569,15 @@ std::vector<std::wstring> LuceneGlobalIndexReader::TokenizeQuery(const std::stri
return wterms;
}
-std::string LuceneGlobalIndexReader::NormalizeWildcardQuery(const std::string& query) {
- std::string normalized_query;
- normalized_query.reserve(query.size());
- size_t term_begin = 0;
- for (size_t i = 0; i <= query.size(); i++) {
- if (i != query.size() && query[i] != '*' && query[i] != '?') {
- continue;
- }
- std::string term = query.substr(term_begin, i - term_begin);
- JiebaTokenizer::NormalizeCase(&term);
- normalized_query.append(term);
- if (i != query.size()) {
- normalized_query.push_back(query[i]);
- }
- term_begin = i + 1;
- }
- return normalized_query;
-}
-
-Lucene::QueryPtr LuceneGlobalIndexReader::ConstructMatchQuery(
- const std::shared_ptr<FullTextSearch>& full_text_search) const noexcept(false) {
- assert(full_text_search->search_type == FullTextSearch::SearchType::MATCH_ALL ||
- full_text_search->search_type == FullTextSearch::SearchType::MATCH_ANY);
- Lucene::BooleanClause::Occur occur =
- full_text_search->search_type == FullTextSearch::SearchType::MATCH_ALL
- ? Lucene::BooleanClause::Occur::MUST
- : Lucene::BooleanClause::Occur::SHOULD;
- std::vector<std::wstring> query_terms = TokenizeQuery(full_text_search->query);
- if (query_terms.size() == 1) {
- return Lucene::newLucene<Lucene::TermQuery>(
- Lucene::newLucene<Lucene::Term>(wfield_name_, query_terms[0]));
- } else {
- auto typed_query = Lucene::newLucene<Lucene::BooleanQuery>();
- for (const auto& term : query_terms) {
- typed_query->add(Lucene::newLucene<Lucene::TermQuery>(
- Lucene::newLucene<Lucene::Term>(wfield_name_, term)),
- occur);
- }
- return typed_query;
- }
-}
-
-Lucene::QueryPtr LuceneGlobalIndexReader::ConstructPhraseQuery(
- const std::shared_ptr<FullTextSearch>& full_text_search) const noexcept(false) {
- assert(full_text_search->search_type == FullTextSearch::SearchType::PHRASE);
- std::vector<std::wstring> query_terms = TokenizeQuery(full_text_search->query);
- auto typed_query = Lucene::newLucene<Lucene::PhraseQuery>();
- for (const auto& term : query_terms) {
- typed_query->add(Lucene::newLucene<Lucene::Term>(wfield_name_, term));
- }
- return typed_query;
-}
-
-Lucene::QueryPtr LuceneGlobalIndexReader::ConstructPrefixQuery(
- const std::shared_ptr<FullTextSearch>& full_text_search) const noexcept(false) {
- assert(full_text_search->search_type == FullTextSearch::SearchType::PREFIX);
- auto create_query = [this](const std::string& query) -> Lucene::QueryPtr {
- return Lucene::newLucene<Lucene::PrefixQuery>(
- Lucene::newLucene<Lucene::Term>(wfield_name_, LuceneUtils::StringToWstring(query)));
- };
- std::string normalized_query = full_text_search->query;
- JiebaTokenizer::NormalizeCase(&normalized_query);
- Lucene::QueryPtr query = create_query(full_text_search->query);
- if (normalized_query == full_text_search->query) {
- return query;
- }
-
- // Preserve the original query for mixed ASCII/CJK indexed terms such as "B超", whose
- // complete token is not lowercased by the index analyzer. The normalized alternative still
- // matches pure ASCII terms. DisjunctionMax avoids double-counting scores if both match.
- auto disjunction = Lucene::newLucene<Lucene::DisjunctionMaxQuery>(0.0);
- disjunction->add(query);
- disjunction->add(create_query(normalized_query));
- return disjunction;
-}
-
-Lucene::QueryPtr LuceneGlobalIndexReader::ConstructWildCardQuery(
- const std::shared_ptr<FullTextSearch>& full_text_search) const noexcept(false) {
- assert(full_text_search->search_type == FullTextSearch::SearchType::WILDCARD);
- auto create_query = [this](const std::string& query) -> Lucene::QueryPtr {
- return Lucene::newLucene<Lucene::WildcardQuery>(
- Lucene::newLucene<Lucene::Term>(wfield_name_, LuceneUtils::StringToWstring(query)));
- };
- std::string normalized_query = NormalizeWildcardQuery(full_text_search->query);
- Lucene::QueryPtr query = create_query(full_text_search->query);
- if (normalized_query == full_text_search->query) {
- return query;
- }
-
- auto disjunction = Lucene::newLucene<Lucene::DisjunctionMaxQuery>(0.0);
- disjunction->add(query);
- disjunction->add(create_query(normalized_query));
- return disjunction;
-}
-
-Result<std::shared_ptr<GlobalIndexResult>> LuceneGlobalIndexReader::SearchWithLimit(
+Result<std::shared_ptr<ScoredGlobalIndexResult>> LuceneGlobalIndexReader::Search(
const Lucene::QueryPtr& query, const std::shared_ptr<FullTextSearch>& full_text_search) const
noexcept(false) {
- assert(full_text_search->limit);
Lucene::FilterPtr filter =
- full_text_search->pre_filter
- ? Lucene::newLucene<LuceneFilter>(&(full_text_search->pre_filter.value()))
+ full_text_search->include_row_ids
+ ? Lucene::newLucene<LuceneFilter>(&(full_text_search->include_row_ids.value()))
: Lucene::FilterPtr();
- Lucene::TopDocsPtr results = searcher_->search(query, filter, full_text_search->limit.value());
+ Lucene::TopDocsPtr results = searcher_->search(query, filter, full_text_search->limit);
// prepare BitmapScoredGlobalIndexResult
std::map<int64_t, float> id_to_score;
@@ -240,65 +594,20 @@ Result<std::shared_ptr<GlobalIndexResult>> LuceneGlobalIndexReader::SearchWithLi
return std::make_shared<BitmapScoredGlobalIndexResult>(std::move(bitmap), std::move(scores));
}
-std::shared_ptr<GlobalIndexResult> LuceneGlobalIndexReader::SearchWithNoLimit(
- const Lucene::QueryPtr& query, const std::shared_ptr<FullTextSearch>& full_text_search) const
- noexcept(false) {
- assert(!full_text_search->limit);
- Lucene::FilterPtr filter =
- full_text_search->pre_filter
- ? Lucene::newLucene<LuceneFilter>(&(full_text_search->pre_filter.value()))
- : Lucene::FilterPtr();
-
- // with no limit & no score
- auto collector = Lucene::newLucene<LuceneCollector>();
- searcher_->search(query, filter, collector);
- return std::make_shared<BitmapGlobalIndexResult>(
- [collector]() -> Result<RoaringBitmap64> { return collector->GetBitmap(); });
-}
-
-Result<std::shared_ptr<GlobalIndexResult>> LuceneGlobalIndexReader::VisitFullTextSearch(
+Result<std::shared_ptr<ScoredGlobalIndexResult>> LuceneGlobalIndexReader::VisitFullTextSearch(
const std::shared_ptr<FullTextSearch>& full_text_search) {
if (!full_text_search) {
return Status::Invalid("VisitFullTextSearch: null FullTextSearch pointer");
}
- if (full_text_search->min_score.has_value()) {
- // The lucene backend does not support min_score pushdown. Fail loudly
- // instead of silently ignoring the threshold and returning unfiltered
- // results, which would be a correctness bug for the caller.
- return Status::NotImplemented(
- "lucene full-text search does not support min_score; "
- "min_score pushdown is only available on the full-text backend");
+ if (full_text_search->limit <= 0) {
+ return Status::Invalid("lucene full-text search requires a positive limit");
}
try {
- Lucene::QueryPtr query;
- switch (full_text_search->search_type) {
- case FullTextSearch::SearchType::MATCH_ALL:
- case FullTextSearch::SearchType::MATCH_ANY: {
- query = ConstructMatchQuery(full_text_search);
- break;
- }
- case FullTextSearch::SearchType::PHRASE: {
- query = ConstructPhraseQuery(full_text_search);
- break;
- }
- case FullTextSearch::SearchType::PREFIX: {
- query = ConstructPrefixQuery(full_text_search);
- break;
- }
- case FullTextSearch::SearchType::WILDCARD: {
- query = ConstructWildCardQuery(full_text_search);
- break;
- }
- default:
- return Status::Invalid(
- fmt::format("Not support for FullTextSearch SearchType {}",
- static_cast<int32_t>(full_text_search->search_type)));
- }
- if (full_text_search->limit) {
- return SearchWithLimit(query, full_text_search);
- } else {
- return SearchWithNoLimit(query, full_text_search);
- }
+ DslQueryTranslator translator(
+ wfield_name_, [this](const std::string& text) { return TokenizeQuery(text); });
+ PAIMON_ASSIGN_OR_RAISE(Lucene::QueryPtr query,
+ translator.Translate(full_text_search->query));
+ return Search(query, full_text_search);
} catch (const std::exception& e) {
return Status::Invalid(
fmt::format("visit full text search failed, with {} error.", e.what()));
diff --git a/src/paimon/global_index/lucene/lucene_global_index_reader.h b/src/paimon/global_index/lucene/lucene_global_index_reader.h
index 4fd8051e..59a378e0 100644
--- a/src/paimon/global_index/lucene/lucene_global_index_reader.h
+++ b/src/paimon/global_index/lucene/lucene_global_index_reader.h
@@ -25,7 +25,6 @@
#include "arrow/type.h"
#include "cppjieba/Jieba.hpp"
#include "lucene++/LuceneHeaders.h"
-#include "paimon/global_index/bitmap_global_index_result.h"
#include "paimon/global_index/global_index_io_meta.h"
#include "paimon/global_index/global_index_reader.h"
#include "paimon/global_index/io/global_index_file_reader.h"
@@ -33,6 +32,8 @@
#include "paimon/predicate/full_text_search.h"
namespace paimon::lucene {
+/// Reads a `lucene-fts` index and answers `FullTextSearch` by translating its JSON DSL query into
+/// Lucene queries on the indexed field.
class LuceneGlobalIndexReader : public GlobalIndexReader {
public:
static Result<std::shared_ptr<LuceneGlobalIndexReader>> Create(
@@ -105,7 +106,7 @@ class LuceneGlobalIndexReader : public GlobalIndexReader {
"LuceneGlobalIndexReader is not supposed to handle vector search query");
}
- Result<std::shared_ptr<GlobalIndexResult>> VisitFullTextSearch(
+ Result<std::shared_ptr<ScoredGlobalIndexResult>> VisitFullTextSearch(
const std::shared_ptr<FullTextSearch>& full_text_search) override;
bool IsThreadSafe() const override {
@@ -128,29 +129,12 @@ class LuceneGlobalIndexReader : public GlobalIndexReader {
std::vector<std::wstring> TokenizeQuery(const std::string& query) const;
- static std::string NormalizeWildcardQuery(const std::string& query);
-
std::shared_ptr<GlobalIndexResult> CreateAllResult() const {
return nullptr;
}
- Lucene::QueryPtr ConstructMatchQuery(
- const std::shared_ptr<FullTextSearch>& full_text_search) const noexcept(false);
-
- Lucene::QueryPtr ConstructPhraseQuery(
- const std::shared_ptr<FullTextSearch>& full_text_search) const noexcept(false);
-
- Lucene::QueryPtr ConstructPrefixQuery(
- const std::shared_ptr<FullTextSearch>& full_text_search) const noexcept(false);
-
- Lucene::QueryPtr ConstructWildCardQuery(
- const std::shared_ptr<FullTextSearch>& full_text_search) const noexcept(false);
-
- Result<std::shared_ptr<GlobalIndexResult>> SearchWithLimit(
- const Lucene::QueryPtr& query,
- const std::shared_ptr<FullTextSearch>& full_text_search) const noexcept(false);
-
- std::shared_ptr<GlobalIndexResult> SearchWithNoLimit(
+ /// Returns the top `limit` rows matching `query` with their scores.
+ Result<std::shared_ptr<ScoredGlobalIndexResult>> Search(
const Lucene::QueryPtr& query,
const std::shared_ptr<FullTextSearch>& full_text_search) const noexcept(false);
diff --git a/src/paimon/global_index/lucene/lucene_global_index_test.cpp b/src/paimon/global_index/lucene/lucene_global_index_test.cpp
index a63e96e6..40885e2a 100644
--- a/src/paimon/global_index/lucene/lucene_global_index_test.cpp
+++ b/src/paimon/global_index/lucene/lucene_global_index_test.cpp
@@ -17,6 +17,8 @@
*/
#include "paimon/global_index/lucene/lucene_global_index.h"
+#include <cmath>
+
#include "arrow/c/bridge.h"
#include "arrow/ipc/api.h"
#include "gtest/gtest.h"
@@ -121,15 +123,20 @@ class LuceneGlobalIndexTest : public ::testing::Test,
pool_);
}
- void CheckResult(const std::shared_ptr<GlobalIndexResult>& result,
+ static std::shared_ptr<FullTextSearch> MakeSearch(
+ const std::string& query, int32_t limit = 10,
+ const std::optional<RoaringBitmap64>& include_row_ids = std::nullopt) {
+ return std::make_shared<FullTextSearch>("f0", query, limit, include_row_ids);
+ }
+
+ void CheckResult(const std::shared_ptr<ScoredGlobalIndexResult>& result,
const std::vector<int64_t>& expected_ids) const {
- const RoaringBitmap64* bitmap = nullptr;
- if (auto scored_result = std::dynamic_pointer_cast<BitmapScoredGlobalIndexResult>(result)) {
- ASSERT_OK_AND_ASSIGN(bitmap, scored_result->GetBitmap());
- ASSERT_EQ(scored_result->GetScores().size(), expected_ids.size());
- } else if (auto bitmap_result =
- std::dynamic_pointer_cast<BitmapGlobalIndexResult>(result)) {
- ASSERT_OK_AND_ASSIGN(bitmap, bitmap_result->GetBitmap());
+ auto scored_result = std::dynamic_pointer_cast<BitmapScoredGlobalIndexResult>(result);
+ ASSERT_TRUE(scored_result);
+ ASSERT_OK_AND_ASSIGN(const RoaringBitmap64* bitmap, scored_result->GetBitmap());
+ ASSERT_EQ(scored_result->GetScores().size(), expected_ids.size());
+ for (float score : scored_result->GetScores()) {
+ ASSERT_TRUE(std::isfinite(score)) << score;
}
ASSERT_TRUE(bitmap);
ASSERT_EQ(*bitmap, RoaringBitmap64::From(expected_ids))
@@ -182,167 +189,145 @@ TEST_P(LuceneGlobalIndexTest, TestSimple) {
ASSERT_TRUE(lucene_reader);
// test visit
+ auto search = [&](const std::string& query, int32_t limit = 10,
+ const std::optional<RoaringBitmap64>& include_row_ids = std::nullopt) {
+ return lucene_reader->VisitFullTextSearch(MakeSearch(query, limit, include_row_ids));
+ };
{
- ASSERT_OK_AND_ASSIGN(auto result,
- lucene_reader->VisitFullTextSearch(std::make_shared<FullTextSearch>(
- "f0",
- /*limit=*/10, "document", FullTextSearch::SearchType::MATCH_ALL,
- /*pre_filter=*/std::nullopt)));
+ const std::string query = R"({"match":{"query":"document","operator":"And"}})";
+ ASSERT_OK_AND_ASSIGN(auto result, search(query));
CheckResult(result, {2l, 1l, 0l});
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<ScoredGlobalIndexResult::ScoredIterator> iter,
+ result->CreateScoredIterator());
+ while (iter->HasNext()) {
+ const auto [row_id, score] = iter->NextWithScore();
+ ASSERT_OK_AND_ASSIGN(auto single_row_result,
+ search(query, 1, RoaringBitmap64::From({row_id})));
+ ASSERT_OK_AND_ASSIGN(
+ std::unique_ptr<ScoredGlobalIndexResult::ScoredIterator> single_row_iter,
+ single_row_result->CreateScoredIterator());
+ ASSERT_TRUE(single_row_iter->HasNext());
+ const auto [expected_row_id, expected_score] = single_row_iter->NextWithScore();
+ ASSERT_EQ(row_id, expected_row_id);
+ ASSERT_FLOAT_EQ(score, expected_score);
+ ASSERT_FALSE(single_row_iter->HasNext());
+ }
}
{
- ASSERT_OK_AND_ASSIGN(auto result,
- lucene_reader->VisitFullTextSearch(std::make_shared<FullTextSearch>(
- "f0",
- /*limit=*/1, "document", FullTextSearch::SearchType::MATCH_ANY,
- /*pre_filter=*/std::nullopt)));
- CheckResult(result, {2l});
+ ASSERT_OK_AND_ASSIGN(auto result, search(R"({"match":{"query":"document",)"
+ R"("ignored":1,"ignored":"\u0000"}})"));
+ CheckResult(result, {0l, 1l, 2l});
}
{
- ASSERT_OK_AND_ASSIGN(
- auto result, lucene_reader->VisitFullTextSearch(std::make_shared<FullTextSearch>(
- "f0",
- /*limit=*/10, "test document", FullTextSearch::SearchType::MATCH_ALL,
- /*pre_filter=*/std::nullopt)));
- CheckResult(result, {2l, 0l});
+ ASSERT_OK_AND_ASSIGN(auto result, search(R"({"match":{"query":"document"}})", 1));
+ CheckResult(result, {2l});
}
{
ASSERT_OK_AND_ASSIGN(auto result,
- lucene_reader->VisitFullTextSearch(std::make_shared<FullTextSearch>(
- "f0",
- /*limit=*/10, "test new", FullTextSearch::SearchType::MATCH_ANY,
- /*pre_filter=*/std::nullopt)));
- CheckResult(result, {1l, 0l, 2l});
+ search(R"({"match":{"terms":"test document","operator":"AND"}})"));
+ CheckResult(result, {2l, 0l});
}
{
ASSERT_OK_AND_ASSIGN(auto result,
- lucene_reader->VisitFullTextSearch(std::make_shared<FullTextSearch>(
- "f0",
- /*limit=*/10, "test document", FullTextSearch::SearchType::PHRASE,
- /*pre_filter=*/std::nullopt)));
- CheckResult(result, {0l});
+ search(R"({"match":{"query":"test new","operator":"Or"}})"));
+ CheckResult(result, {1l, 0l, 2l});
}
{
ASSERT_OK_AND_ASSIGN(auto result,
- lucene_reader->VisitFullTextSearch(std::make_shared<FullTextSearch>(
- "f0",
- /*limit=*/10, "unordered", FullTextSearch::SearchType::MATCH_ALL,
- /*pre_filter=*/std::nullopt)));
+ search(R"({"match":{"query":"unordered","column":"f0","boost":2.0,)"
+ R"("fuzziness":0,"max_expansions":10,"prefix_length":1}})"));
CheckResult(result, {3l});
}
+ for (const std::string& query : std::vector<std::string>{
+ R"({"match":{"query":" "}})", R"({"match_phrase":{"query":" "}})"}) {
+ SCOPED_TRACE(query);
+ ASSERT_OK_AND_ASSIGN(auto result, search(query));
+ CheckResult(result, {});
+ }
{
- ASSERT_OK_AND_ASSIGN(auto result,
- lucene_reader->VisitFullTextSearch(std::make_shared<FullTextSearch>(
- "f0",
- /*limit=*/10, "unorder", FullTextSearch::SearchType::PREFIX,
- /*pre_filter=*/std::nullopt)));
- CheckResult(result, {3l});
+ ASSERT_OK_AND_ASSIGN(auto result, search(R"({"match_phrase":{"query":"test document"}})"));
+ CheckResult(result, {0l});
}
{
ASSERT_OK_AND_ASSIGN(auto result,
- lucene_reader->VisitFullTextSearch(std::make_shared<FullTextSearch>(
- "f0",
- /*limit=*/10, "THIS", FullTextSearch::SearchType::PREFIX,
- /*pre_filter=*/std::nullopt)));
- CheckResult(result, {1l, 0l});
+ search(R"({"phrase":{"query":"test document","slop":2}})"));
+ CheckResult(result, {0l, 2l});
}
- // test wildcard query
{
- ASSERT_OK_AND_ASSIGN(auto result,
- lucene_reader->VisitFullTextSearch(std::make_shared<FullTextSearch>(
- "f0",
- /*limit=*/10, "*order*", FullTextSearch::SearchType::WILDCARD,
- /*pre_filter=*/std::nullopt)));
- CheckResult(result, {3l});
+ ASSERT_OK_AND_ASSIGN(
+ auto result,
+ search(R"({"multi_match":{"query":"test new","columns":["f0"],"boosts":[2.0]}})"));
+ CheckResult(result, {1l, 0l, 2l});
}
{
+ ASSERT_OK_AND_ASSIGN(auto result, search(R"({"multi_match":{"query":"test document",)"
+ R"("columns":["f0"],"operator":"And"}})"));
+ CheckResult(result, {0l, 2l});
+ }
+ for (const std::string& boost : std::vector<std::string>{"1e-45", "3.4028235e38"}) {
+ SCOPED_TRACE(boost);
ASSERT_OK_AND_ASSIGN(auto result,
- lucene_reader->VisitFullTextSearch(std::make_shared<FullTextSearch>(
- "f0",
- /*limit=*/10, "*or*er*", FullTextSearch::SearchType::WILDCARD,
- /*pre_filter=*/std::nullopt)));
- CheckResult(result, {3l});
+ search(R"({"match":{"query":"document","boost":)" + boost + "}}"));
+ CheckResult(result, {0l, 1l, 2l});
+ ASSERT_OK_AND_ASSIGN(result,
+ search(R"({"multi_match":{"query":"document","columns":["f0"],)"
+ R"("boosts":[)" +
+ boost + "]}}"));
+ CheckResult(result, {0l, 1l, 2l});
}
{
- ASSERT_OK_AND_ASSIGN(auto result,
- lucene_reader->VisitFullTextSearch(std::make_shared<FullTextSearch>(
- "f0",
- /*limit=*/10, "*THIS*", FullTextSearch::SearchType::WILDCARD,
- /*pre_filter=*/std::nullopt)));
- CheckResult(result, {1l, 0l});
+ ASSERT_OK_AND_ASSIGN(auto result, search(R"({"boolean":{"should":[
+ {"match":{"query":"document","boost":100.0}},
+ {"match":{"query":"unordered"}}]}})",
+ 3));
+ CheckResult(result, {0l, 1l, 2l});
}
{
- ASSERT_OK_AND_ASSIGN(auto result,
- lucene_reader->VisitFullTextSearch(std::make_shared<FullTextSearch>(
- "f0",
- /*limit=*/10, "*?HIS*", FullTextSearch::SearchType::WILDCARD,
- /*pre_filter=*/std::nullopt)));
- CheckResult(result, {1l, 0l});
+ ASSERT_OK_AND_ASSIGN(auto result, search(R"({"boolean":{
+ "must":[{"match":{"query":"document"}}],
+ "must_not":[{"match":{"query":"new"}}]}})"));
+ CheckResult(result, {0l, 2l});
+ }
+ const std::vector<std::pair<std::string, std::vector<int64_t>>> whitespace_queries = {
+ {R"({"match":{"query":"test document","operator":"\u3000And\u00a0",)"
+ R"("column":"\u0085f0\u202f"}})",
+ {0l, 2l}},
+ {R"({"multi_match":{"query":"test new","columns":["\u2007f0\u205f"],)"
+ R"("operator":"\u1680Or\u2009"}})",
+ {0l, 1l, 2l}},
+ {R"({"match_phrase":{"query":"test document","column":"\t\u3000\u00a0\n"}})", {0l}},
+ {R"({"multi_match":{"query":"document","columns":["\u2028\u2029"]}})", {0l, 1l, 2l}},
+ {R"({"match":{"query":"document","column":""}})", {0l, 1l, 2l}},
+ {R"({"boolean":{"queries":[
+ ["\u00a0Must\u3000",{"match":{"query":"document"}}],
+ ["\u202fShould\u0085",{"match_phrase":{"query":"test document"}}],
+ ["\u205fMustNot\u2007",{"match":{"query":"new"}}]]}})",
+ {0l, 2l}}};
+ for (const auto& [query, expected_ids] : whitespace_queries) {
+ SCOPED_TRACE(query);
+ ASSERT_OK_AND_ASSIGN(auto result, search(query));
+ CheckResult(result, expected_ids);
}
// test filter
+ std::string match_all_document = R"({"match":{"query":"document","operator":"And"}})";
{
ASSERT_OK_AND_ASSIGN(auto result,
- lucene_reader->VisitFullTextSearch(std::make_shared<FullTextSearch>(
- "f0",
- /*limit=*/10, "document", FullTextSearch::SearchType::MATCH_ALL,
- /*pre_filter=*/RoaringBitmap64::From({0l, 1l}))));
+ search(match_all_document, 10, RoaringBitmap64::From({0l, 1l})));
CheckResult(result, {0l, 1l});
}
{
ASSERT_OK_AND_ASSIGN(auto result,
- lucene_reader->VisitFullTextSearch(std::make_shared<FullTextSearch>(
- "f0",
- /*limit=*/10, "document", FullTextSearch::SearchType::MATCH_ALL,
- /*pre_filter=*/RoaringBitmap64::From({2l, 100l}))));
+ search(match_all_document, 10, RoaringBitmap64::From({2l, 100l})));
CheckResult(result, {2l});
}
{
ASSERT_OK_AND_ASSIGN(auto result,
- lucene_reader->VisitFullTextSearch(std::make_shared<FullTextSearch>(
- "f0",
- /*limit=*/10, "document", FullTextSearch::SearchType::MATCH_ALL,
- /*pre_filter=*/RoaringBitmap64::From({20l, 100l}))));
+ search(match_all_document, 10, RoaringBitmap64::From({20l, 100l})));
CheckResult(result, {});
}
- // test no limit
- {
- ASSERT_OK_AND_ASSIGN(
- auto result,
- lucene_reader->VisitFullTextSearch(std::make_shared<FullTextSearch>(
- "f0",
- /*limit=*/std::nullopt, "document", FullTextSearch::SearchType::MATCH_ALL,
- /*pre_filter=*/std::nullopt)));
- CheckResult(result, {0l, 1l, 2l});
- }
- {
- ASSERT_OK_AND_ASSIGN(
- auto result,
- lucene_reader->VisitFullTextSearch(std::make_shared<FullTextSearch>(
- "f0",
- /*limit=*/std::nullopt, "document", FullTextSearch::SearchType::MATCH_ALL,
- /*pre_filter=*/RoaringBitmap64::From({2l}))));
- CheckResult(result, {2l});
- }
{
- ASSERT_OK_AND_ASSIGN(
- auto result,
- lucene_reader->VisitFullTextSearch(std::make_shared<FullTextSearch>(
- "f0",
- /*limit=*/std::nullopt, "document test", FullTextSearch::SearchType::MATCH_ALL,
- /*pre_filter=*/RoaringBitmap64::From({1l, 2l, 3l, 100l}))));
- CheckResult(result, {2l});
- }
- // min_score pushdown is not supported by the lucene backend: it must fail
- // loudly rather than silently ignore the threshold.
- {
- auto fts = std::make_shared<FullTextSearch>("f0",
- /*limit=*/10, "document",
- FullTextSearch::SearchType::MATCH_ALL,
- /*pre_filter=*/std::nullopt);
- fts->min_score = 1.5f;
- auto result = lucene_reader->VisitFullTextSearch(fts);
- ASSERT_FALSE(result.ok());
- ASSERT_TRUE(result.status().IsNotImplemented()) << result.status().ToString();
+ ASSERT_OK_AND_ASSIGN(auto result, search(match_all_document, 10, RoaringBitmap64()));
+ CheckResult(result, {});
}
}
@@ -389,119 +374,244 @@ TEST_P(LuceneGlobalIndexTest, TestSimpleChinese) {
// test visit
{
- ASSERT_OK_AND_ASSIGN(auto result,
- lucene_reader->VisitFullTextSearch(std::make_shared<FullTextSearch>(
- "f0",
- /*limit=*/10, "模块", FullTextSearch::SearchType::MATCH_ALL,
- /*pre_filter=*/std::nullopt)));
+ ASSERT_OK_AND_ASSIGN(auto result, lucene_reader->VisitFullTextSearch(MakeSearch(
+ R"({"match":{"query":"模块","operator":"And"}})")));
CheckResult(result, {0l, 2l});
}
{
- ASSERT_OK_AND_ASSIGN(auto result,
- lucene_reader->VisitFullTextSearch(std::make_shared<FullTextSearch>(
- "f0",
- /*limit=*/1, "模块", FullTextSearch::SearchType::MATCH_ANY,
- /*pre_filter=*/std::nullopt)));
+ ASSERT_OK_AND_ASSIGN(auto result, lucene_reader->VisitFullTextSearch(
+ MakeSearch(R"({"match":{"query":"模块"}})", 1)));
CheckResult(result, {0l});
}
{
ASSERT_OK_AND_ASSIGN(auto result,
- lucene_reader->VisitFullTextSearch(std::make_shared<FullTextSearch>(
- "f0",
- /*limit=*/10, "模块技术", FullTextSearch::SearchType::MATCH_ALL,
- /*pre_filter=*/std::nullopt)));
+ lucene_reader->VisitFullTextSearch(
+ MakeSearch(R"({"match":{"query":"模块技术","operator":"And"}})")));
CheckResult(result, {0l});
}
{
- ASSERT_OK_AND_ASSIGN(auto result,
- lucene_reader->VisitFullTextSearch(std::make_shared<FullTextSearch>(
- "f0",
- /*limit=*/10, "模块技术", FullTextSearch::SearchType::MATCH_ANY,
- /*pre_filter=*/std::nullopt)));
+ ASSERT_OK_AND_ASSIGN(auto result, lucene_reader->VisitFullTextSearch(
+ MakeSearch(R"({"match":{"query":"模块技术"}})")));
CheckResult(result, {0l, 1l, 2l, 3l});
}
{
- ASSERT_OK_AND_ASSIGN(auto result,
- lucene_reader->VisitFullTextSearch(std::make_shared<FullTextSearch>(
- "f0",
- /*limit=*/10, "发展方向", FullTextSearch::SearchType::PHRASE,
- /*pre_filter=*/std::nullopt)));
+ ASSERT_OK_AND_ASSIGN(auto result, lucene_reader->VisitFullTextSearch(MakeSearch(
+ R"({"match_phrase":{"query":"发展方向"}})")));
CheckResult(result, {4l});
}
- {
- ASSERT_OK_AND_ASSIGN(auto result,
- lucene_reader->VisitFullTextSearch(std::make_shared<FullTextSearch>(
- "f0",
- /*limit=*/10, "发", FullTextSearch::SearchType::PREFIX,
- /*pre_filter=*/std::nullopt)));
- CheckResult(result, {3l, 4l});
- }
- // test wildcard query
- {
- ASSERT_OK_AND_ASSIGN(auto result,
- lucene_reader->VisitFullTextSearch(std::make_shared<FullTextSearch>(
- "f0",
- /*limit=*/10, "*发*", FullTextSearch::SearchType::WILDCARD,
- /*pre_filter=*/std::nullopt)));
- CheckResult(result, {0l, 1l, 2l, 3l, 4l});
- }
// test filter
{
- ASSERT_OK_AND_ASSIGN(auto result,
- lucene_reader->VisitFullTextSearch(std::make_shared<FullTextSearch>(
- "f0",
- /*limit=*/10, "模块技术", FullTextSearch::SearchType::MATCH_ANY,
- /*pre_filter=*/RoaringBitmap64::From({1l, 3l, 4l}))));
+ ASSERT_OK_AND_ASSIGN(auto result, lucene_reader->VisitFullTextSearch(
+ MakeSearch(R"({"match":{"query":"模块技术"}})", 10,
+ RoaringBitmap64::From({1l, 3l, 4l}))));
CheckResult(result, {1l, 3l});
}
- // test no limit
- {
- ASSERT_OK_AND_ASSIGN(
- auto result,
- lucene_reader->VisitFullTextSearch(std::make_shared<FullTextSearch>(
- "f0",
- /*limit=*/std::nullopt, "模块技术", FullTextSearch::SearchType::MATCH_ANY,
- /*pre_filter=*/std::nullopt)));
- CheckResult(result, {0l, 1l, 2l, 3l});
- }
}
-TEST_P(LuceneGlobalIndexTest, TestMixedAsciiCjkPrefixAndWildcard) {
+TEST_F(LuceneGlobalIndexTest, TestNestedQuery) {
auto test_root_dir = paimon::test::UniqueTestDirectory::Create();
ASSERT_TRUE(test_root_dir);
auto tmp_dir = paimon::test::UniqueTestDirectory::Create();
ASSERT_TRUE(tmp_dir);
-
std::map<std::string, std::string> options = {
- {"lucene-fts.write.omit-term-freq-and-position", "false"},
- {"lucene-fts.read.buffer-size", std::to_string(GetParam())},
- {"lucene-fts.jieba.tokenize-mode", "query"},
{"lucene-fts.write.tmp.directory", tmp_dir->Str()}};
- std::shared_ptr<arrow::Array> array = arrow::ipc::internal::json::ArrayFromJSON(data_type_, R"([
- ["B超检查"],
- ["T恤"]
- ])")
+ std::shared_ptr<arrow::Array> array = arrow::ipc::internal::json::ArrayFromJSON(data_type_,
+ R"([
+ ["paimon streaming lake"],
+ ["paimon batch lake"],
+ ["paimon lake"],
+ ["iceberg streaming lake"],
+ ["iceberg batch"]
+ ])")
.ValueOrDie();
+ ASSERT_OK_AND_ASSIGN(auto meta, WriteGlobalIndex(test_root_dir->Str(), data_type_, options,
+ array, Range(0, 4), tmp_dir->Str()));
+ ASSERT_OK_AND_ASSIGN(auto reader,
+ CreateGlobalIndexReader(test_root_dir->Str(), data_type_, options, meta));
+ auto search = [&](const std::string& query) {
+ return reader->VisitFullTextSearch(MakeSearch(query));
+ };
+ ASSERT_OK_AND_ASSIGN(auto result, search(R"({"boolean":{
+ "must":[
+ {"match":{"query":"paimon"}},
+ {"boolean":{"should":[
+ {"match":{"query":"streaming"}},{"match":{"query":"batch"}}]}}]}})"));
+ CheckResult(result, {0l, 1l});
+ ASSERT_OK_AND_ASSIGN(result, search(R"({"boolean":{
+ "queries":[
+ ["must",{"match":{"query":"paimon"}}],
+ ["must_not",{"boolean":{"should":[
+ {"match":{"query":"streaming"}},{"match":{"query":"batch"}}]}}]]}})"));
+ CheckResult(result, {2l});
+
+ auto nest = [](const std::string& prefix, const std::string& query, const std::string& suffix,
+ int32_t levels) {
+ std::string nested;
+ for (int32_t i = 0; i < levels; ++i) {
+ nested.append(prefix);
+ }
+ nested.append(query);
+ for (int32_t i = 0; i < levels; ++i) {
+ nested.append(suffix);
+ }
+ return nested;
+ };
+ auto nest_must = [&](const std::string& query, int32_t levels) {
+ return nest(R"({"boolean":{"must":[)", query, "]}}", levels);
+ };
+ auto nest_queries = [&](const std::string& query, int32_t levels) {
+ return nest(R"({"boolean":{"queries":[["must",)", query, "]]}}", levels);
+ };
+ const std::string match = R"({"match":{"query":"paimon"}})";
+ const std::string multi_match = R"({"multi_match":{"query":"paimon","columns":["f0"]}})";
+
+ ASSERT_OK_AND_ASSIGN(result, search(nest_must(match, 41)));
+ CheckResult(result, {0l, 1l, 2l});
+ ASSERT_OK_AND_ASSIGN(result, search(nest_queries(nest_must(multi_match, 40), 1)));
+ CheckResult(result, {0l, 1l, 2l});
+ const std::string too_deep = "full-text query exceeds the maximum nesting depth of 127";
+ ASSERT_NOK_WITH_MSG(search(nest_must(match, 42)), too_deep);
+ ASSERT_NOK_WITH_MSG(search(nest_must(nest_queries(multi_match, 2), 39)), too_deep);
+ ASSERT_NOK_WITH_MSG(search(nest_must(match, 100000)), too_deep);
+ ASSERT_OK_AND_ASSIGN(result, search(R"({"match":{"query":"paimon","ignored":)" +
+ nest("[", "", "]", 100000) + "}}"));
+ CheckResult(result, {0l, 1l, 2l});
+}
+
+TEST_F(LuceneGlobalIndexTest, TestInvalidQuery) {
+ auto test_root_dir = paimon::test::UniqueTestDirectory::Create();
+ ASSERT_TRUE(test_root_dir);
+ auto tmp_dir = paimon::test::UniqueTestDirectory::Create();
+ ASSERT_TRUE(tmp_dir);
+ std::map<std::string, std::string> options = {
+ {"lucene-fts.write.tmp.directory", tmp_dir->Str()}};
+ std::shared_ptr<arrow::Array> array =
+ arrow::ipc::internal::json::ArrayFromJSON(data_type_, R"([["This is an new document."]])")
+ .ValueOrDie();
ASSERT_OK_AND_ASSIGN(auto meta, WriteGlobalIndex(test_root_dir->Str(), data_type_, options,
- array, Range(0, 1), tmp_dir->Str()));
+ array, Range(0, 0), tmp_dir->Str()));
ASSERT_OK_AND_ASSIGN(auto reader,
CreateGlobalIndexReader(test_root_dir->Str(), data_type_, options, meta));
- auto lucene_reader = std::dynamic_pointer_cast<LuceneGlobalIndexReader>(reader);
- ASSERT_TRUE(lucene_reader);
+ auto search = [&](const std::string& query, int32_t limit = 10) {
+ return reader->VisitFullTextSearch(MakeSearch(query, limit));
+ };
+
+ {
+ auto result = search(R"({"boost":{"positive":{"match":{"query":"document"}},)"
+ R"("negative":{"match":{"query":"new"}}}})");
+ ASSERT_FALSE(result.ok());
+ ASSERT_TRUE(result.status().IsNotImplemented()) << result.status().ToString();
+ }
+ for (const std::string& fuzziness : std::vector<std::string>{"1", "2", R"("auto")", "null"}) {
+ auto result = search(R"({"match":{"query":"document","fuzziness":)" + fuzziness + "}}");
+ ASSERT_FALSE(result.ok());
+ ASSERT_TRUE(result.status().IsNotImplemented()) << result.status().ToString();
+ }
- ASSERT_OK_AND_ASSIGN(auto prefix_result,
- lucene_reader->VisitFullTextSearch(std::make_shared<FullTextSearch>(
- "f0", /*limit=*/10, "B", FullTextSearch::SearchType::PREFIX,
- /*pre_filter=*/std::nullopt)));
- CheckResult(prefix_result, {0l});
-
- ASSERT_OK_AND_ASSIGN(auto wildcard_result,
- lucene_reader->VisitFullTextSearch(std::make_shared<FullTextSearch>(
- "f0", /*limit=*/10, "*T*", FullTextSearch::SearchType::WILDCARD,
- /*pre_filter=*/std::nullopt)));
- CheckResult(wildcard_result, {1l});
+ ASSERT_NOK_WITH_MSG(reader->VisitFullTextSearch(nullptr), "null FullTextSearch");
+ ASSERT_NOK_WITH_MSG(search(R"({"match":{"query":"document"}})", 0),
+ "requires a positive limit");
+ ASSERT_NOK_WITH_MSG(search("document"), "invalid full-text query");
+ ASSERT_NOK_WITH_MSG(search(R"({"match":{"query":"document"}} trailing)"),
+ "invalid full-text query");
+ std::string query_with_nul = R"({"match":{"query":"document"}})";
+ query_with_nul.push_back('\0');
+ query_with_nul.append("trailing");
+ ASSERT_NOK_WITH_MSG(search(query_with_nul), "must not contain NUL characters");
+ ASSERT_NOK_WITH_MSG(search(R"(["match"])"), "exactly one query type");
+ ASSERT_NOK_WITH_MSG(
+ search(R"({"match":{"query":"document"},"match_phrase":{"query":"document"}})"),
+ "exactly one query type");
+ ASSERT_NOK_WITH_MSG(search(R"({"term":{"query":"document"}})"),
+ "unknown full-text query type `term`");
+ ASSERT_NOK_WITH_MSG(search(R"({"match":"document"})"), "must be a JSON object");
+ ASSERT_NOK_WITH_MSG(search(R"({"match":{}})"), "missing field `query`");
+ ASSERT_NOK_WITH_MSG(search(R"({"match":{"query":1}})"), "field `query` must be a string");
+ ASSERT_NOK_WITH_MSG(search(R"({"match":{"query":"document","terms":"document"}})"),
+ "sets both field `terms` and its alias `query`");
+ const std::vector<std::pair<std::string, std::string>> duplicate_field_cases = {
+ {R"({"match":{"query":"document","query":"new"}})", "query"},
+ {R"({"match":{"query":"document","column":null,"column":"f0"}})", "column"},
+ {R"({"match":{"query":"document","operator":"Or","operator":"And"}})", "operator"},
+ {R"({"match":{"query":"document","boost":1,"boost":2}})", "boost"},
+ {R"({"multi_match":{"query":"document","columns":["f0"],"columns":["f0"]}})", "columns"},
+ {R"({"multi_match":{"query":"document","columns":["f0"],"boosts":[1],"boosts":[2]}})",
+ "boosts"},
+ {R"({"phrase":{"query":"new document","slop":0,"slop":1}})", "slop"},
+ {R"({"boolean":{"must":[{"match":{"query":"document"}}],)"
+ R"("must":[{"match":{"query":"new"}}]}})",
+ "must"},
+ {R"({"boolean":{"should":[{"match":{"query":"document","query":"new"}}]}})", "query"}};
+ for (const auto& [query, field] : duplicate_field_cases) {
+ SCOPED_TRACE(query);
+ ASSERT_NOK_WITH_MSG(search(query), "duplicate full-text query field `" + field + "`");
+ }
+ ASSERT_NOK_WITH_MSG(search(R"({"match":{"query":"document","column":"f1"}})"),
+ "column 'f1' is not configured for this index");
+ ASSERT_NOK_WITH_MSG(search(R"({"match":{"query":"document","operator":"Xor"}})"),
+ "invalid full-text query operator: Xor");
+ for (const std::string& non_whitespace :
+ std::vector<std::string>{R"(\u0000)", R"(\u0001)", R"(\u001c)", R"(\u001f)", R"(\u180e)",
+ R"(\u200b)", R"(\ufeff)", R"(\ufffe)", R"(\uffff)", "\xff"}) {
+ SCOPED_TRACE(non_whitespace);
+ ASSERT_NOK_WITH_MSG(search(R"({"match":{"query":"document","operator":")" + non_whitespace +
+ "And" + non_whitespace + R"("}})"),
+ "invalid full-text query operator");
+ ASSERT_NOK_WITH_MSG(search(R"({"boolean":{"queries":[[")" + non_whitespace + "Must" +
+ non_whitespace + R"(",{"match":{"query":"document"}}]]}})"),
+ "invalid boolean query occur");
+ ASSERT_NOK_WITH_MSG(search(R"({"match":{"query":"document","column":")" + non_whitespace +
+ "f0" + non_whitespace + R"("}})"),
+ "full-text query column");
+ ASSERT_NOK_WITH_MSG(search(R"({"multi_match":{"query":"document","columns":[")" +
+ non_whitespace + R"("]}})"),
+ "full-text query column");
+ }
+ for (const std::string& boost :
+ std::vector<std::string>{"0", "1e40", "1e-100", "3.4028236e38", "7e-46"}) {
+ SCOPED_TRACE(boost);
+ ASSERT_NOK_WITH_MSG(search(R"({"match":{"query":"document","boost":)" + boost + "}}"),
+ "boost must be a finite positive value after float conversion");
+ ASSERT_NOK_WITH_MSG(
+ search(R"({"multi_match":{"query":"document","columns":["f0"],"boosts":[)" + boost +
+ "]}}"),
+ "boost must be a finite positive value after float conversion");
+ }
+ ASSERT_NOK_WITH_MSG(search(R"({"match":{"query":"document","fuzziness":3}})"),
+ "fuzziness must be auto/null or a value in [0, 2]");
+ ASSERT_NOK_WITH_MSG(search(R"({"match":{"query":"document","max_expansions":0}})"),
+ "max_expansions must be positive");
+ ASSERT_NOK_WITH_MSG(search(R"({"match":{"query":"document","prefix_length":-1}})"),
+ "prefix_length must be a non-negative integer");
+ for (const std::string& slop : std::vector<std::string>{"-1", "2147483648"}) {
+ ASSERT_NOK_WITH_MSG(search(R"({"match_phrase":{"query":"document","slop":)" + slop + "}}"),
+ "field `slop` must be an integer in [0, 2147483647]");
+ }
+ ASSERT_NOK_WITH_MSG(search(R"({"multi_match":{"query":"document"}})"),
+ "field `columns` must be an array of strings");
+ ASSERT_NOK_WITH_MSG(search(R"({"multi_match":{"query":"document","columns":[]}})"),
+ "must contain at least one column");
+ ASSERT_NOK_WITH_MSG(
+ search(R"({"multi_match":{"query":"document","columns":["f0"],"boosts":2.0}})"),
+ "field `boosts` must be an array of numbers");
+ ASSERT_NOK_WITH_MSG(
+ search(R"({"multi_match":{"query":"document","columns":["f0"],"boosts":[1.0,2.0]}})"),
+ "boosts length 2 does not match columns length 1");
+ ASSERT_NOK_WITH_MSG(search(R"({"multi_match":{"query":"document","columns":["f0","f1"]}})"),
+ "column 'f1' is not configured for this index");
+ ASSERT_NOK_WITH_MSG(search(R"({"boolean":{}})"), "must contain at least one clause");
+ ASSERT_NOK_WITH_MSG(search(R"({"boolean":{"must":{"match":{"query":"document"}}}})"),
+ "field `must` must be an array of queries");
+ ASSERT_NOK_WITH_MSG(search(R"({"boolean":{"must_not":[{"match":{"query":"new"}}]}})"),
+ "must contain at least one should or must clause");
+ ASSERT_NOK_WITH_MSG(search(R"({"boolean":{"queries":[["should"]]}})"),
+ "must be an array of [occur, query] pairs");
+ ASSERT_NOK_WITH_MSG(search(R"({"boolean":{"queries":[["filter",{"match":{"query":"new"}}]]}})"),
+ "invalid boolean query occur: filter");
+ ASSERT_NOK_WITH_MSG(search(R"({"boolean":{"should":[{"term":{"query":"new"}}]}})"),
+ "unknown full-text query type `term`");
}
TEST_F(LuceneGlobalIndexTest, TestInvalidWithoutTmpDir) {
diff --git a/src/paimon/indexer/lumina/lumina_global_index.h b/src/paimon/indexer/lumina/lumina_global_index.h
index 452f5561..0a9186fe 100644
--- a/src/paimon/indexer/lumina/lumina_global_index.h
+++ b/src/paimon/indexer/lumina/lumina_global_index.h
@@ -136,9 +136,9 @@ class LuminaIndexReader : public GlobalIndexReader {
Result<std::shared_ptr<ScoredGlobalIndexResult>> VisitVectorSearch(
const std::shared_ptr<VectorSearch>& vector_search) override;
- Result<std::shared_ptr<GlobalIndexResult>> VisitFullTextSearch(
+ Result<std::shared_ptr<ScoredGlobalIndexResult>> VisitFullTextSearch(
const std::shared_ptr<FullTextSearch>& full_text_search) override {
- return std::shared_ptr<GlobalIndexResult>();
+ return std::shared_ptr<ScoredGlobalIndexResult>();
}
Result<std::shared_ptr<GlobalIndexResult>> VisitIsNotNull() override {
diff --git a/src/paimon/testing/mock/mock_global_index_reader.h b/src/paimon/testing/mock/mock_global_index_reader.h
index 32a1e78c..c5bd3bef 100644
--- a/src/paimon/testing/mock/mock_global_index_reader.h
+++ b/src/paimon/testing/mock/mock_global_index_reader.h
@@ -66,7 +66,7 @@ class MockGlobalIndexReader : public GlobalIndexReader {
exception_message_ = message;
}
- /// Sets a scored result returned by VisitVectorSearch.
+ /// Sets a scored result returned by VisitVectorSearch and VisitFullTextSearch.
void SetScoredResult(const std::vector<int64_t>& row_ids, const std::vector<float>& scores) {
scored_row_ids_ = row_ids;
scored_scores_ = scores;
@@ -144,35 +144,19 @@ class MockGlobalIndexReader : public GlobalIndexReader {
Result<std::shared_ptr<ScoredGlobalIndexResult>> VisitVectorSearch(
const std::shared_ptr<VectorSearch>& vector_search) override {
- invocation_count_++;
- if (!supports_search_) {
- return Status::Invalid("not supported");
- }
- if (return_error_) {
- return Status::Invalid(error_message_);
- }
- if (!has_scored_result_) {
- return std::shared_ptr<ScoredGlobalIndexResult>(nullptr);
- }
- auto bitmap = RoaringBitmap64::From(scored_row_ids_);
- auto scores = scored_scores_;
- return std::make_shared<BitmapScoredGlobalIndexResult>(std::move(bitmap),
- std::move(scores));
+ return MakeScoredResult();
}
- Result<std::shared_ptr<GlobalIndexResult>> VisitFullTextSearch(
+ Result<std::shared_ptr<ScoredGlobalIndexResult>> VisitFullTextSearch(
const std::shared_ptr<FullTextSearch>& full_text_search) override {
- if (!supports_search_) {
- return Status::Invalid("not supported");
- }
{
std::lock_guard<std::mutex> lock(captured_search_mutex_);
captured_full_text_search_ = full_text_search;
}
- return MakeResult();
+ return MakeScoredResult();
}
- // Captures the (possibly pre_filter-rewritten) FullTextSearch the offset
+ // Captures the (possibly include_row_ids-rewritten) FullTextSearch the offset
// reader forwarded, so tests can assert field propagation.
std::shared_ptr<FullTextSearch> CapturedFullTextSearch() const {
std::lock_guard<std::mutex> lock(captured_search_mutex_);
@@ -188,6 +172,23 @@ class MockGlobalIndexReader : public GlobalIndexReader {
}
private:
+ Result<std::shared_ptr<ScoredGlobalIndexResult>> MakeScoredResult() {
+ invocation_count_++;
+ if (!supports_search_) {
+ return Status::Invalid("not supported");
+ }
+ if (return_error_) {
+ return Status::Invalid(error_message_);
+ }
+ if (!has_scored_result_) {
+ return std::shared_ptr<ScoredGlobalIndexResult>(nullptr);
+ }
+ auto bitmap = RoaringBitmap64::From(scored_row_ids_);
+ auto scores = scored_scores_;
+ return std::make_shared<BitmapScoredGlobalIndexResult>(std::move(bitmap),
+ std::move(scores));
+ }
+
Result<std::shared_ptr<GlobalIndexResult>> MakeResult() {
invocation_count_++;
if (throw_exception_) {
diff --git a/src/paimon/testing/mock/mock_search_file_index.cpp b/src/paimon/testing/mock/mock_search_file_index.cpp
index 577a332f..3e8e39a1 100644
--- a/src/paimon/testing/mock/mock_search_file_index.cpp
+++ b/src/paimon/testing/mock/mock_search_file_index.cpp
@@ -76,10 +76,10 @@ Result<std::shared_ptr<FileIndexResult>> MockSearchFileIndexReader::VisitFullTex
}
RoaringBitmap32 positions;
for (const auto& [row_id, score] : rows_) {
- if (search->limit && positions.Cardinality() >= search->limit.value()) {
+ if (positions.Cardinality() >= search->limit) {
break;
}
- if (!search->pre_filter || search->pre_filter->Contains(row_id)) {
+ if (!search->include_row_ids || search->include_row_ids->Contains(row_id)) {
positions.Add(row_id);
}
}
diff --git a/test/inte/global_index_test.cpp b/test/inte/global_index_test.cpp
index 5bc14602..59f2d6f7 100644
--- a/test/inte/global_index_test.cpp
+++ b/test/inte/global_index_test.cpp
@@ -2517,47 +2517,22 @@ TEST_P(GlobalIndexTest, TestLuceneWriteCommitScanReadIndexWithScore) {
global_index_scan->CreateReaders("f0", /*row_range_index=*/std::nullopt));
ASSERT_EQ(index_readers.size(), 1u);
auto index_reader = index_readers[0];
- {
- ASSERT_OK_AND_ASSIGN(auto index_result,
- index_reader->VisitFullTextSearch(std::make_shared<FullTextSearch>(
- "f0",
- /*limit=*/10, "document", FullTextSearch::SearchType::MATCH_ALL,
- /*pre_filter=*/std::nullopt)));
- ASSERT_TRUE(index_result->ToString().find("row ids: {0,1,2}") != std::string::npos);
- }
- {
- std::optional<RoaringBitmap64> pre_filter = RoaringBitmap64::From({1, 2, 3});
- ASSERT_OK_AND_ASSIGN(
- auto index_result,
- index_reader->VisitFullTextSearch(std::make_shared<FullTextSearch>(
- "f0",
- /*limit=*/10, "document", FullTextSearch::SearchType::MATCH_ALL, pre_filter)));
- ASSERT_TRUE(index_result->ToString().find("row ids: {1,2}") != std::string::npos);
- }
- {
- ASSERT_OK_AND_ASSIGN(auto index_result,
- index_reader->VisitFullTextSearch(std::make_shared<FullTextSearch>(
- "f0",
- /*limit=*/10, "*or*er*", FullTextSearch::SearchType::WILDCARD,
- /*pre_filter=*/std::nullopt)));
- ASSERT_TRUE(index_result->ToString().find("row ids: {3}") != std::string::npos);
- }
- {
- ASSERT_OK_AND_ASSIGN(auto index_result,
- index_reader->VisitFullTextSearch(std::make_shared<FullTextSearch>(
- "f0",
- /*limit=*/10, "THIS", FullTextSearch::SearchType::PREFIX,
- /*pre_filter=*/std::nullopt)));
- ASSERT_TRUE(index_result->ToString().find("row ids: {0,1}") != std::string::npos);
- }
- {
+ auto search = [&](const std::string& query,
+ const std::optional<RoaringBitmap64>& include_row_ids,
+ const std::string& expected) {
ASSERT_OK_AND_ASSIGN(auto index_result,
index_reader->VisitFullTextSearch(std::make_shared<FullTextSearch>(
- "f0",
- /*limit=*/10, "*THIS*", FullTextSearch::SearchType::WILDCARD,
- /*pre_filter=*/std::nullopt)));
- ASSERT_TRUE(index_result->ToString().find("row ids: {0,1}") != std::string::npos);
- }
+ "f0", query, /*limit=*/10, include_row_ids)));
+ ASSERT_TRUE(index_result->ToString().find("row ids: " + expected + ", scores: ") !=
+ std::string::npos)
+ << query << ": " << index_result->ToString();
+ };
+ search(R"({"match":{"query":"document","operator":"And"}})", std::nullopt, "{0,1,2}");
+ search(R"({"match":{"query":"document","operator":"And"}})", RoaringBitmap64::From({1, 2, 3}),
+ "{1,2}");
+ search(R"({"match_phrase":{"query":"test document"}})", std::nullopt, "{0}");
+ search(R"({"boolean":{"should":[{"match":{"query":"unordered"}},{"match":{"query":"new"}}]}})",
+ std::nullopt, "{1,3}");
}
TEST_P(GlobalIndexTest, TestWriteCommitScanReadLuceneIndexWithPartition) {
@@ -2606,7 +2581,7 @@ TEST_P(GlobalIndexTest, TestWriteCommitScanReadLuceneIndexWithPartition) {
auto scan_and_check_result = [&](const std::map<std::string, std::string>& partition,
const std::optional<RowRangeIndex>& row_range_index,
- const std::optional<RoaringBitmap64>& pre_filter,
+ const std::optional<RoaringBitmap64>& include_row_ids,
const std::string& index_expected) {
std::vector<std::map<std::string, std::string>> partitions = {partition};
ASSERT_OK_AND_ASSIGN(
@@ -2616,11 +2591,10 @@ TEST_P(GlobalIndexTest, TestWriteCommitScanReadLuceneIndexWithPartition) {
// check lucene index
ASSERT_OK_AND_ASSIGN(auto readers, global_index_scan->CreateReaders("f0", row_range_index));
ASSERT_EQ(readers.size(), 1u);
- ASSERT_OK_AND_ASSIGN(
- auto index_result,
- readers[0]->VisitFullTextSearch(std::make_shared<FullTextSearch>(
- "f0",
- /*limit=*/10, "document", FullTextSearch::SearchType::MATCH_ALL, pre_filter)));
+ ASSERT_OK_AND_ASSIGN(auto index_result,
+ readers[0]->VisitFullTextSearch(std::make_shared<FullTextSearch>(
+ "f0", R"({"match":{"query":"document","operator":"And"}})",
+ /*limit=*/10, include_row_ids)));
ASSERT_TRUE(index_result->ToString().find(index_expected) != std::string::npos);
};
@@ -2677,14 +2651,15 @@ TEST_P(GlobalIndexTest, TestFullTextWriteCommitScanReadIndex) {
global_index_scan->CreateReaders("f0", /*row_range_index=*/std::nullopt));
ASSERT_EQ(index_readers.size(), 1u);
auto index_reader = index_readers[0];
- auto search = [&](const std::string& query, const std::optional<RoaringBitmap64>& pre_filter,
+ auto search = [&](const std::string& query,
+ const std::optional<RoaringBitmap64>& include_row_ids,
const std::string& expected) {
- ASSERT_OK_AND_ASSIGN(
- auto index_result,
- index_reader->VisitFullTextSearch(std::make_shared<FullTextSearch>(
- "f0",
- /*limit=*/10, query, FullTextSearch::SearchType::UNKNOWN, pre_filter)));
- ASSERT_EQ(index_result->ToString(), expected) << query;
+ ASSERT_OK_AND_ASSIGN(auto index_result,
+ index_reader->VisitFullTextSearch(std::make_shared<FullTextSearch>(
+ "f0", query, /*limit=*/10, include_row_ids)));
+ ASSERT_TRUE(index_result->ToString().find("row ids: " + expected + ", scores: ") !=
+ std::string::npos)
+ << query << ": " << index_result->ToString();
};
search(R"({"match":{"query":"document"}})", std::nullopt, "{0,1,2}");
search(R"({"match":{"query":"document"}})", RoaringBitmap64::From({1, 2, 3}), "{1,2}");
@@ -2732,7 +2707,7 @@ TEST_P(GlobalIndexTest, TestWriteCommitScanReadFullTextIndexWithPartition) {
auto scan_and_check_result = [&](const std::map<std::string, std::string>& partition,
const std::optional<RowRangeIndex>& row_range_index,
- const std::optional<RoaringBitmap64>& pre_filter,
+ const std::optional<RoaringBitmap64>& include_row_ids,
const std::string& index_expected) {
std::vector<std::map<std::string, std::string>> partitions = {partition};
ASSERT_OK_AND_ASSIGN(
@@ -2741,12 +2716,13 @@ TEST_P(GlobalIndexTest, TestWriteCommitScanReadFullTextIndexWithPartition) {
/*options=*/{}, fs_, /*executor=*/nullptr, pool_));
ASSERT_OK_AND_ASSIGN(auto readers, global_index_scan->CreateReaders("f0", row_range_index));
ASSERT_EQ(readers.size(), 1u);
- ASSERT_OK_AND_ASSIGN(auto index_result,
- readers[0]->VisitFullTextSearch(std::make_shared<FullTextSearch>(
- "f0",
- /*limit=*/10, R"({"match":{"query":"document"}})",
- FullTextSearch::SearchType::UNKNOWN, pre_filter)));
- ASSERT_EQ(index_result->ToString(), index_expected);
+ ASSERT_OK_AND_ASSIGN(
+ auto index_result,
+ readers[0]->VisitFullTextSearch(std::make_shared<FullTextSearch>(
+ "f0", R"({"match":{"query":"document"}})", /*limit=*/10, include_row_ids)));
+ ASSERT_TRUE(index_result->ToString().find("row ids: " + index_expected + ", scores: ") !=
+ std::string::npos)
+ << index_result->ToString();
};
{