You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by lx...@apache.org on 2026/10/08 10:21:20 UTC
(paimon-cpp) branch main updated: feat(global-index): replace Tantivy FFI with paimon-full-text-index (#414)
This is an automated email from the ASF dual-hosted git repository.
lxy-9602 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/paimon-cpp.git
The following commit(s) were added to refs/heads/main by this push:
new 1c942ebd feat(global-index): replace Tantivy FFI with paimon-full-text-index (#414)
1c942ebd is described below
commit 1c942ebd78a8dc9ff3e3c320859d443888bc1200
Author: Nicholas Jiang <pr...@163.com>
AuthorDate: Thu Oct 8 17:21:13 2026 +0700
feat(global-index): replace Tantivy FFI with paimon-full-text-index (#414)
---
.devcontainer/Dockerfile.template | 2 +-
.devcontainer/centos7/Dockerfile | 6 +-
.devcontainer/centos7/run.sh | 4 +-
.devcontainer/devcontainer.json.template | 6 +-
.devcontainer/x86_64/devcontainer.json.template | 4 +-
.github/workflows/build_and_test.yaml | 2 +
CMakeLists.txt | 30 +-
ci/scripts/build_paimon.sh | 38 +-
ci/scripts/setup_rust.sh | 8 +-
cmake_modules/CorrosionFetch.cmake | 4 +-
cmake_modules/ThirdpartyToolchain.cmake | 66 +-
crates/lance_ffi/rust-toolchain.toml | 4 +-
crates/tantivy_ffi/Cargo.lock | 1859 --------------------
crates/tantivy_ffi/Cargo.toml | 50 -
crates/tantivy_ffi/build.rs | 56 -
crates/tantivy_ffi/cbindgen.toml | 82 -
crates/tantivy_ffi/rust-toolchain.toml | 25 -
crates/tantivy_ffi/src/buffer.rs | 127 --
crates/tantivy_ffi/src/callback_directory.rs | 531 ------
crates/tantivy_ffi/src/error.rs | 153 --
crates/tantivy_ffi/src/handle.rs | 122 --
crates/tantivy_ffi/src/lib.rs | 95 -
crates/tantivy_ffi/src/log_bridge.rs | 119 --
crates/tantivy_ffi/src/reader.rs | 1398 ---------------
crates/tantivy_ffi/src/tokenizer.rs | 467 -----
crates/tantivy_ffi/src/writer.rs | 788 ---------
docs/source/building.rst | 6 +-
docs/source/user_guide/global_index.rst | 111 +-
include/paimon/predicate/full_text_search.h | 28 +-
src/paimon/common/utils/file_type_test.cpp | 5 +-
src/paimon/global_index/full_text/CMakeLists.txt | 56 +
.../full_text_defs.h} | 19 +-
.../global_index/full_text/full_text_ffi_utils.cpp | 44 +
.../global_index/full_text/full_text_ffi_utils.h | 50 +
.../full_text_global_index.cpp} | 50 +-
.../full_text_global_index.h} | 18 +-
.../full_text_global_index_factory.cpp} | 29 +-
.../full_text_global_index_factory.h} | 16 +-
.../full_text/full_text_global_index_reader.cpp | 218 +++
.../full_text_global_index_reader.h} | 97 +-
.../full_text/full_text_global_index_test.cpp | 737 ++++++++
.../full_text/full_text_global_index_writer.cpp | 226 +++
.../full_text_global_index_writer.h} | 52 +-
.../lucene/lucene_global_index_reader.cpp | 2 +-
src/paimon/global_index/tantivy/CMakeLists.txt | 268 ---
.../tantivy/tantivy_archive_layout.cpp | 93 -
.../global_index/tantivy/tantivy_archive_layout.h | 61 -
src/paimon/global_index/tantivy/tantivy_defs.h | 93 -
.../tantivy/tantivy_equivalence_test.cpp | 436 -----
.../global_index/tantivy/tantivy_ffi_handle.h | 113 --
.../global_index/tantivy/tantivy_ffi_log.cpp | 70 -
.../global_index/tantivy/tantivy_ffi_status.h | 92 -
.../global_index/tantivy/tantivy_ffi_test.cpp | 138 --
.../tantivy/tantivy_filter_limit_test.cpp | 382 ----
.../tantivy/tantivy_global_index_factory.cpp | 45 -
.../tantivy/tantivy_global_index_reader.cpp | 232 ---
.../tantivy/tantivy_global_index_writer.cpp | 172 --
.../global_index/tantivy/tantivy_index_test.cpp | 300 ----
.../tantivy/tantivy_java_compat_test.cpp | 451 -----
.../tantivy/tantivy_lucene_coexist_test.cpp | 277 ---
.../global_index/tantivy/tantivy_smoke_test.cpp | 51 -
.../global_index/tantivy/tantivy_stream_ctx.cpp | 90 -
.../global_index/tantivy/tantivy_stream_ctx.h | 74 -
.../tantivy/tantivy_streaming_test.cpp | 308 ----
.../tantivy/tantivy_tokenizer_test.cpp | 127 --
.../global_index/tantivy/tantivy_writer_test.cpp | 258 ---
test/inte/global_index_test.cpp | 127 ++
.../cpp_tantivy_fixtures/english_default.archive | Bin 6597 -> 0 bytes
test/test_data/full_text_fixtures/README.md | 67 +
test/test_data/full_text_fixtures/default.archive | Bin 0 -> 3355 bytes
.../full_text_fixtures/generate_fixtures.py | 205 +++
test/test_data/full_text_fixtures/jieba.archive | Bin 0 -> 3380 bytes
test/test_data/full_text_fixtures/ngram.archive | Bin 0 -> 3431 bytes
test/test_data/java_tantivy_fixtures/README.md | 51 -
.../java_tantivy_fixtures/english_simple.archive | Bin 6044 -> 0 bytes
.../english_simple.golden.json | 25 -
.../production_sample.archive | Bin 5176 -> 0 bytes
test/test_data/tokenizer_golden/README.md | 29 -
test/test_data/tokenizer_golden/golden_corpus.txt | 20 -
.../tokenizer_golden/golden_synthetic.txt | 38 -
test/test_data/tokenizer_golden/known_diffs.txt | 18 -
third_party/versions.txt | 5 +
82 files changed, 2151 insertions(+), 10375 deletions(-)
diff --git a/.devcontainer/Dockerfile.template b/.devcontainer/Dockerfile.template
index 00f1ee15..0d2e3816 100644
--- a/.devcontainer/Dockerfile.template
+++ b/.devcontainer/Dockerfile.template
@@ -71,7 +71,7 @@ RUN apt update && \
vim \
wget \
sudo \
- # ---- additions for tantivy-fts migration (Rust + Sanitizer + LLVM) ----
+ # ---- additions for the Rust FFI builds (Rust + Sanitizer + LLVM) ----
clang \
clang-format \
clang-tidy \
diff --git a/.devcontainer/centos7/Dockerfile b/.devcontainer/centos7/Dockerfile
index 9a25b5e8..fa0f9c1c 100644
--- a/.devcontainer/centos7/Dockerfile
+++ b/.devcontainer/centos7/Dockerfile
@@ -14,10 +14,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-# CentOS 7 cross-build verification image for paimon-cpp + tantivy-fts.
+# CentOS 7 cross-build verification image for paimon-cpp + the full-text index.
#
# Purpose:
-# Prove the tantivy-fts stack builds on the OLDEST reasonable Linux target
+# Prove the full-text index stack builds on the OLDEST reasonable Linux target
# (glibc 2.17, EOL 2024-06-30). The default Ubuntu 24.04 dev container
# proves nothing about glibc compatibility; this image does.
#
@@ -95,7 +95,7 @@ enabled=0\n' > /etc/yum.repos.d/CentOS-Base.repo \
# EPEL provides ninja-build and a newer python3 than the base 3.6.
# SCL (Software Collections) provides devtoolset-11 (gcc 11) and rh-python38
# without overriding the system gcc/python. CentOS 7's default gcc 4.8 is
-# too old for C++17/20 used by lucene++ and our tantivy wrapper.
+# too old for C++17/20 used by lucene++ and our full-text wrapper.
#
# Same story as CentOS-Base.repo: both epel + SCL default to mirrorlist
# endpoints that are effectively dead; overwrite with aliyun URLs that we
diff --git a/.devcontainer/centos7/run.sh b/.devcontainer/centos7/run.sh
index 2f494f27..a08f3fd2 100755
--- a/.devcontainer/centos7/run.sh
+++ b/.devcontainer/centos7/run.sh
@@ -107,12 +107,12 @@ case "${cmd}" in
-DPAIMON_ENABLE_LUMINA=OFF \
-DPAIMON_ENABLE_JINDO=OFF \
-DPAIMON_ENABLE_LUCENE=ON \
- -DPAIMON_ENABLE_TANTIVY=ON \
+ -DPAIMON_ENABLE_FULL_TEXT=ON \
-DPAIMON_ENABLE_ORC=ON \
-DPAIMON_ENABLE_AVRO=ON
cmake --build build-centos7 -j "$(nproc)"
ctest --test-dir build-centos7 \
- -R "paimon-lucene-index-test|paimon-global-index-test|paimon-tantivy-.*-test" \
+ -R "paimon-lucene-index-test|paimon-global-index-test|paimon-full-text-index-test" \
--output-on-failure
'
;;
diff --git a/.devcontainer/devcontainer.json.template b/.devcontainer/devcontainer.json.template
index f5a6c742..28c7770f 100644
--- a/.devcontainer/devcontainer.json.template
+++ b/.devcontainer/devcontainer.json.template
@@ -48,11 +48,11 @@
"source=${localEnv:HOME}/.ssh,target=/home/paimon/.ssh,type=bind,readonly",
"source=paimon-cargo-registry,target=/home/paimon/.cargo/registry,type=volume",
"source=paimon-cargo-git,target=/home/paimon/.cargo/git,type=volume",
- "source=paimon-rust-target,target=${containerWorkspaceFolder}/crates/tantivy_ffi/target,type=volume",
+ "source=paimon-rust-target,target=${containerWorkspaceFolder}/crates/lance_ffi/target,type=volume",
"source=paimon-build,target=${containerWorkspaceFolder}/build,type=volume",
"source=paimon-ccache,target=/home/paimon/.ccache,type=volume"
],
- "postCreateCommand": "sudo chown -R paimon:paimon ${containerWorkspaceFolder}/build ${containerWorkspaceFolder}/crates/tantivy_ffi/target /home/paimon/.ccache /home/paimon/.cargo/registry /home/paimon/.cargo/git 2>/dev/null || true; cargo install cbindgen --locked || true; rustup component add rust-src rust-analyzer clippy rustfmt || true",
+ "postCreateCommand": "sudo chown -R paimon:paimon ${containerWorkspaceFolder}/build ${containerWorkspaceFolder}/crates/lance_ffi/target /home/paimon/.ccache /home/paimon/.cargo/registry /home/paimon/.cargo/git 2>/dev/null || true; cargo install cbindgen --locked || true; rustup component add rust-src rust-analyzer clippy rustfmt || true",
"customizations": {
"vscode": {
"extensions": [
@@ -66,7 +66,7 @@
"settings": {
"editor.formatOnSave": true,
"rust-analyzer.linkedProjects": [
- "crates/tantivy_ffi/Cargo.toml"
+ "crates/lance_ffi/Cargo.toml"
]
}
}
diff --git a/.devcontainer/x86_64/devcontainer.json.template b/.devcontainer/x86_64/devcontainer.json.template
index 00763314..12c2e35e 100644
--- a/.devcontainer/x86_64/devcontainer.json.template
+++ b/.devcontainer/x86_64/devcontainer.json.template
@@ -54,11 +54,11 @@
"source=${localEnv:HOME}/.ssh,target=/home/paimon/.ssh,type=bind,readonly",
"source=paimon-cargo-registry-amd64,target=/home/paimon/.cargo/registry,type=volume",
"source=paimon-cargo-git-amd64,target=/home/paimon/.cargo/git,type=volume",
- "source=paimon-rust-target-amd64,target=${containerWorkspaceFolder}/crates/tantivy_ffi/target,type=volume",
+ "source=paimon-rust-target-amd64,target=${containerWorkspaceFolder}/crates/lance_ffi/target,type=volume",
"source=paimon-build-amd64,target=${containerWorkspaceFolder}/build,type=volume",
"source=paimon-ccache-amd64,target=/home/paimon/.ccache,type=volume"
],
- "postCreateCommand": "sudo chown -R paimon:paimon ${containerWorkspaceFolder}/build ${containerWorkspaceFolder}/crates/tantivy_ffi/target /home/paimon/.ccache /home/paimon/.cargo/registry /home/paimon/.cargo/git 2>/dev/null || true; cargo install cbindgen --locked || true; rustup component add rust-src rust-analyzer clippy rustfmt || true",
+ "postCreateCommand": "sudo chown -R paimon:paimon ${containerWorkspaceFolder}/build ${containerWorkspaceFolder}/crates/lance_ffi/target /home/paimon/.ccache /home/paimon/.cargo/registry /home/paimon/.cargo/git 2>/dev/null || true; cargo install cbindgen --locked || true; rustup component add rust-src rust-analyzer clippy rustfmt || true",
"customizations": {
"vscode": {
"extensions": [
diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml
index cd9174c0..e4f58fb6 100644
--- a/.github/workflows/build_and_test.yaml
+++ b/.github/workflows/build_and_test.yaml
@@ -67,6 +67,8 @@ jobs:
- name: gcc-debug-x86_64
cc: gcc-14
cxx: g++-14
+ # Also read the full-text archives written by Paimon C++ with the Python reader.
+ build_args: --verify_full_text_archives
- name: clang-debug-x86_64
fetch_depth: '0' # fetch the PR target branch history for clang-tidy
timeout_minutes: 180
diff --git a/CMakeLists.txt b/CMakeLists.txt
index f55bfadf..afaa09a8 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -19,7 +19,7 @@
cmake_minimum_required(VERSION 3.22)
# 3.22 is the minimum required by Corrosion-rs (used for the Rust<->C++ FFI
-# integration, see crates/tantivy_ffi). Ubuntu 24.04 ships CMake 3.28 and
+# integration, see crates/lance_ffi). Ubuntu 24.04 ships CMake 3.28 and
# CentOS 8+/RHEL 9+ ship 3.20+, so older distributions need a newer CMake
# installation.
message(STATUS "Building using CMake version: ${CMAKE_VERSION}")
@@ -69,8 +69,8 @@ option(PAIMON_ENABLE_NETWORK_TESTS
"Whether to enable tests that access real remote services over the network" OFF)
option(PAIMON_ENABLE_LUMINA "Whether to enable lumina vector index" OFF)
option(PAIMON_ENABLE_LUCENE "Whether to enable lucene index" OFF)
-option(PAIMON_ENABLE_TANTIVY
- "Whether to enable tantivy-fulltext global index (Rust FFI, experimental)" OFF)
+option(PAIMON_ENABLE_FULL_TEXT
+ "Whether to enable full-text global index (Rust FFI, experimental)" OFF)
option(PAIMON_ENABLE_REST
"Whether to enable the rest catalog (requires libcurl and OpenSSL)" OFF)
if(PAIMON_ENABLE_ORC)
@@ -126,8 +126,8 @@ if(PAIMON_ENABLE_LUCENE)
add_definitions(-DPAIMON_ENABLE_LUCENE)
endif()
-if(PAIMON_ENABLE_TANTIVY)
- add_definitions(-DPAIMON_ENABLE_TANTIVY)
+if(PAIMON_ENABLE_FULL_TEXT)
+ add_definitions(-DPAIMON_ENABLE_FULL_TEXT)
endif()
add_definitions(-DSNAPPY_CODEC_AVAILABLE)
@@ -333,7 +333,7 @@ add_subdirectory(third_party/roaring_bitmap EXCLUDE_FROM_ALL)
add_subdirectory(third_party/xxhash EXCLUDE_FROM_ALL)
# ---- Rust FFI crates via Corrosion-rs -------------------------------------------
-if(PAIMON_ENABLE_LANCE OR PAIMON_ENABLE_TANTIVY)
+if(PAIMON_ENABLE_LANCE)
include(CorrosionFetch)
endif()
@@ -346,13 +346,6 @@ if(PAIMON_ENABLE_LANCE)
corrosion_set_env_vars(paimon_lance_ffi "PROTOC=${PAIMON_LANCE_PROTOC}")
endif()
-if(PAIMON_ENABLE_TANTIVY)
- corrosion_import_crate(MANIFEST_PATH crates/tantivy_ffi/Cargo.toml CRATES
- paimon_tantivy_ffi)
- corrosion_experimental_cbindgen(TARGET paimon_tantivy_ffi HEADER_NAME
- paimon_tantivy_ffi.h)
-endif()
-
if(PAIMON_ENABLE_LUCENE)
set(PAIMON_DICT_DEST "share/paimon/dict")
@@ -532,6 +525,13 @@ if(PAIMON_BUILD_TESTS)
paimon_lucene_index_shared)
list(APPEND TEST_STATIC_LINK_LIBS ${TEST_PLUGIN_LINK_LIBS})
endif()
+ if(PAIMON_ENABLE_FULL_TEXT)
+ paimon_link_libraries_whole_archive(PAIMON_FULL_TEXT_INDEX_STATIC_LINK_LIBS
+ paimon_full_text_index_static)
+ paimon_link_libraries_no_as_needed(TEST_PLUGIN_LINK_LIBS
+ paimon_full_text_index_shared)
+ list(APPEND TEST_STATIC_LINK_LIBS ${TEST_PLUGIN_LINK_LIBS})
+ endif()
endif()
if(PAIMON_BUILD_BENCHMARKS)
@@ -586,8 +586,8 @@ if(PAIMON_ENABLE_LUMINA)
add_subdirectory(src/paimon/indexer/lumina)
endif()
add_subdirectory(src/paimon/global_index/lucene)
-if(PAIMON_ENABLE_TANTIVY)
- add_subdirectory(src/paimon/global_index/tantivy)
+if(PAIMON_ENABLE_FULL_TEXT)
+ add_subdirectory(src/paimon/global_index/full_text)
endif()
add_subdirectory(src/paimon/testing/mock)
add_subdirectory(src/paimon/testing/utils)
diff --git a/ci/scripts/build_paimon.sh b/ci/scripts/build_paimon.sh
index b012e7f4..4f9f7983 100755
--- a/ci/scripts/build_paimon.sh
+++ b/ci/scripts/build_paimon.sh
@@ -18,7 +18,7 @@
set -eux
usage() {
- echo "Usage: $0 --source_dir <path> [--enable_asan] [--enable_ubsan] [--enable_tsan] [--check_clang_tidy] [--build_type <type>] [--lint_git_target_commit <commit-or-branch>] [--install_smoke]"
+ echo "Usage: $0 --source_dir <path> [--enable_asan] [--enable_ubsan] [--enable_tsan] [--check_clang_tidy] [--build_type <type>] [--lint_git_target_commit <commit-or-branch>] [--install_smoke] [--verify_full_text_archives]"
}
source_dir=""
@@ -29,6 +29,7 @@ check_clang_tidy="false"
build_type="Debug"
lint_git_target_commit="origin/main"
install_smoke="false"
+verify_full_text_archives="false"
while [[ $# -gt 0 ]]; do
case "$1" in
@@ -79,6 +80,10 @@ while [[ $# -gt 0 ]]; do
install_smoke="true"
shift
;;
+ --verify_full_text_archives)
+ verify_full_text_archives="true"
+ shift
+ ;;
-h | --help)
usage
exit 0
@@ -128,15 +133,15 @@ mkdir -p "${build_dir}"
pushd "${build_dir}"
ENABLE_LUMINA="ON"
-ENABLE_TANTIVY="ON"
+ENABLE_FULL_TEXT="ON"
ENABLE_LANCE="ON"
if [[ "${CC:-}" == *"gcc-8"* ]] || [[ "${CXX:-}" == *"g++-8"* ]]; then
ENABLE_LUMINA="OFF"
- ENABLE_TANTIVY="OFF" # tantivy-fts (Rust FFI) is not built on the gcc-8 image.
+ ENABLE_FULL_TEXT="OFF" # The full-text index (Rust FFI) is not built on the gcc-8 image.
fi
if [[ "${enable_tsan}" == "true" ]]; then
- ENABLE_TANTIVY="OFF" # Tantivy's Rust library is not TSAN-instrumented.
- ENABLE_LANCE="OFF" # Lance's Rust library is not TSAN-instrumented.
+ ENABLE_FULL_TEXT="OFF" # The full-text Rust library is not TSAN-instrumented.
+ ENABLE_LANCE="OFF" # Lance's Rust library is not TSAN-instrumented.
fi
# CI always builds natively, so the host architecture is the target architecture.
host_arch=$(uname -m)
@@ -144,6 +149,10 @@ if [[ "${host_arch}" != "x86_64" ]]; then
ENABLE_LUMINA="OFF"
echo "=== Lumina disabled: no prebuilt artifacts for ${host_arch} ==="
fi
+if [[ "${verify_full_text_archives}" == "true" && "${ENABLE_FULL_TEXT}" != "ON" ]]; then
+ echo "--verify_full_text_archives requires the full-text index, which this build disables" >&2
+ exit 1
+fi
CMAKE_ARGS=(
"-G Ninja"
@@ -156,7 +165,7 @@ CMAKE_ARGS=(
"-DPAIMON_ENABLE_S3=ON"
"-DPAIMON_ENABLE_LUMINA=${ENABLE_LUMINA}"
"-DPAIMON_ENABLE_LUCENE=ON"
- "-DPAIMON_ENABLE_TANTIVY=${ENABLE_TANTIVY}"
+ "-DPAIMON_ENABLE_FULL_TEXT=${ENABLE_FULL_TEXT}"
"-DPAIMON_ENABLE_REST=ON"
"-DPAIMON_LINT_GIT_TARGET_COMMIT=${lint_git_target_commit}"
)
@@ -173,8 +182,25 @@ fi
cmake "${CMAKE_ARGS[@]}" "${source_dir}"
cmake --build . -- -j "${build_jobs}"
+if [[ "${verify_full_text_archives}" == "true" ]]; then
+ # TestCrossReadFixtures copies the full-text archives written by Paimon C++ here.
+ full_text_archive_dir="${build_dir}/full_text_archives"
+ mkdir -p "${full_text_archive_dir}"
+ export PAIMON_FULL_TEXT_ARCHIVE_OUTPUT_DIR="${full_text_archive_dir}"
+fi
ctest --output-on-failure -j "${build_jobs}"
+if [[ "${verify_full_text_archives}" == "true" ]]; then
+ # Read them with the Python binding of the same engine and compare the search results with
+ # those of the checked-in fixtures.
+ full_text_venv="${build_dir}/full_text_venv"
+ python3 -m venv "${full_text_venv}"
+ "${full_text_venv}/bin/pip" install paimon-ftindex==0.1.0
+ "${full_text_venv}/bin/python" \
+ "${source_dir}/test/test_data/full_text_fixtures/generate_fixtures.py" \
+ --verify "${full_text_archive_dir}"
+fi
+
if [[ "${check_clang_tidy}" == "true" ]]; then
cmake --build . --target check-clang-tidy
fi
diff --git a/ci/scripts/setup_rust.sh b/ci/scripts/setup_rust.sh
index 5576d523..bd8cc2d6 100755
--- a/ci/scripts/setup_rust.sh
+++ b/ci/scripts/setup_rust.sh
@@ -15,8 +15,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-# Install the Rust toolchain used by the Lance, Mosaic, and tantivy-fts FFI builds, plus
-# cbindgen required by Lance and tantivy-fts.
+# Install the Rust toolchain used by the Lance, Mosaic, and full-text index FFI builds, plus
+# cbindgen required by Lance.
#
# The dev container (see .devcontainer/) already has these preinstalled;
# this script is for the GitHub Actions runners and is called before ci/scripts/build_paimon.sh.
@@ -26,8 +26,8 @@
set -eux
RUSTUP_VERSION=${RUSTUP_VERSION:-1.29.0}
-# 1.88.0 is the minimum required by transitive crates (e.g. time 0.3.47).
-RUST_VERSION=${RUST_VERSION:-1.88.0}
+# 1.90.0 is the minimum required by paimon-full-text 0.1.0 (rust-version in its Cargo.toml).
+RUST_VERSION=${RUST_VERSION:-1.90.0}
CBINDGEN_VERSION=${CBINDGEN_VERSION:-0.29.2}
PROTOC_VERSION=27.4
diff --git a/cmake_modules/CorrosionFetch.cmake b/cmake_modules/CorrosionFetch.cmake
index b56b2795..8c341cad 100644
--- a/cmake_modules/CorrosionFetch.cmake
+++ b/cmake_modules/CorrosionFetch.cmake
@@ -14,8 +14,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-# Pull Corrosion-rs via FetchContent so we can import crates/tantivy_ffi as
-# CMake targets for the tantivy-fulltext global index.
+# Pull Corrosion-rs via FetchContent so we can import crates/lance_ffi as
+# CMake targets for the Lance file format.
#
# Pinned to v0.5.2 (stable release). Requires CMake >= 3.22.
diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake
index 0c64819c..f92f9abc 100644
--- a/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cmake_modules/ThirdpartyToolchain.cmake
@@ -103,6 +103,18 @@ else()
endif()
endif()
+if(DEFINED ENV{PAIMON_FULL_TEXT_URL})
+ set(FULL_TEXT_SOURCE_URL "$ENV{PAIMON_FULL_TEXT_URL}")
+else()
+ if(EXISTS "${THIRDPARTY_DIR}/${PAIMON_FULL_TEXT_PKG_NAME}")
+ set_urls(FULL_TEXT_SOURCE_URL "${THIRDPARTY_DIR}/${PAIMON_FULL_TEXT_PKG_NAME}")
+ else()
+ set_urls(FULL_TEXT_SOURCE_URL
+ "https://downloads.apache.org/paimon/paimon-full-text-${PAIMON_FULL_TEXT_BUILD_VERSION}/${PAIMON_FULL_TEXT_PKG_NAME}"
+ )
+ endif()
+endif()
+
if(DEFINED ENV{PAIMON_RAPIDJSON_URL})
set(RAPIDJSON_SOURCE_URL "$ENV{PAIMON_RAPIDJSON_URL}")
else()
@@ -1403,6 +1415,53 @@ macro(build_mosaic)
install(FILES "${MOSAIC_DYNAMIC_LIB}" DESTINATION ${CMAKE_INSTALL_LIBDIR})
endmacro()
+macro(build_full_text)
+ message(STATUS "Building Apache Paimon Full Text Rust FFI from source")
+ find_program(PAIMON_CARGO_EXECUTABLE cargo REQUIRED)
+
+ set(FULL_TEXT_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/full_text_ep-install")
+ set(FULL_TEXT_INCLUDE_DIR "${FULL_TEXT_PREFIX}/include")
+ set(FULL_TEXT_LIB_DIR "${FULL_TEXT_PREFIX}/${CMAKE_INSTALL_LIBDIR}")
+ set(FULL_TEXT_DYNAMIC_LIB
+ "${FULL_TEXT_LIB_DIR}/${CMAKE_SHARED_LIBRARY_PREFIX}paimon_ftindex_ffi${CMAKE_SHARED_LIBRARY_SUFFIX}"
+ )
+ set(FULL_TEXT_CARGO_TARGET_DIR "${CMAKE_CURRENT_BINARY_DIR}/full_text_ep-cargo")
+ set(FULL_TEXT_CARGO_DYNAMIC_LIB
+ "${FULL_TEXT_CARGO_TARGET_DIR}/release/${CMAKE_SHARED_LIBRARY_PREFIX}paimon_ftindex_ffi${CMAKE_SHARED_LIBRARY_SUFFIX}"
+ )
+
+ file(MAKE_DIRECTORY "${FULL_TEXT_INCLUDE_DIR}")
+ file(MAKE_DIRECTORY "${FULL_TEXT_LIB_DIR}")
+
+ # Build with the lock file shipped in the source release: a reader rejects index files whose
+ # recorded Tantivy version differs from the linked one, so the Tantivy version must match the
+ # one used by Java Paimon.
+ externalproject_add(full_text_ep
+ URL ${FULL_TEXT_SOURCE_URL}
+ URL_HASH "SHA256=${PAIMON_FULL_TEXT_BUILD_SHA256_CHECKSUM}"
+ ${THIRDPARTY_LOG_OPTIONS}
+ CONFIGURE_COMMAND ""
+ BUILD_COMMAND ${CMAKE_COMMAND} -E env
+ CARGO_TARGET_DIR=${FULL_TEXT_CARGO_TARGET_DIR}
+ ${PAIMON_CARGO_EXECUTABLE} build --release --locked
+ --manifest-path <SOURCE_DIR>/ffi/Cargo.toml
+ COMMAND ${CMAKE_COMMAND} -E copy_if_different
+ ${FULL_TEXT_CARGO_DYNAMIC_LIB} ${FULL_TEXT_DYNAMIC_LIB}
+ INSTALL_COMMAND ${CMAKE_COMMAND} -E copy_directory
+ <SOURCE_DIR>/include ${FULL_TEXT_INCLUDE_DIR}
+ BUILD_BYPRODUCTS "${FULL_TEXT_DYNAMIC_LIB}")
+
+ add_library(paimon_ftindex_ffi SHARED IMPORTED GLOBAL)
+ set_target_properties(paimon_ftindex_ffi
+ PROPERTIES IMPORTED_LOCATION "${FULL_TEXT_DYNAMIC_LIB}"
+ IMPORTED_NO_SONAME TRUE
+ INTERFACE_INCLUDE_DIRECTORIES
+ "${FULL_TEXT_INCLUDE_DIR}")
+ add_dependencies(paimon_ftindex_ffi full_text_ep)
+
+ install(FILES "${FULL_TEXT_DYNAMIC_LIB}" DESTINATION ${CMAKE_INSTALL_LIBDIR})
+endmacro()
+
macro(build_jindosdk_nextarch)
message(STATUS "Building jindosdk-nextarch from local source")
@@ -2128,6 +2187,9 @@ resolve_dependency(glog)
if(PAIMON_ENABLE_MOSAIC)
build_mosaic()
endif()
+if(PAIMON_ENABLE_FULL_TEXT)
+ build_full_text()
+endif()
if(PAIMON_ENABLE_AVRO)
resolve_dependency(Avro)
endif()
@@ -2155,9 +2217,5 @@ endif()
if(PAIMON_ENABLE_LUCENE)
build_boost()
build_lucene()
-endif()
-# jieba (dict + headers) is needed by BOTH lucene-fts and the tantivy jieba
-# tokenizer; build it whenever either backend is on, not only under lucene.
-if(PAIMON_ENABLE_LUCENE OR PAIMON_ENABLE_TANTIVY)
build_jieba()
endif()
diff --git a/crates/lance_ffi/rust-toolchain.toml b/crates/lance_ffi/rust-toolchain.toml
index 0b1a652b..ed2bc124 100644
--- a/crates/lance_ffi/rust-toolchain.toml
+++ b/crates/lance_ffi/rust-toolchain.toml
@@ -15,6 +15,8 @@
# specific language governing permissions and limitations
# under the License.
+# Keep in sync with RUST_VERSION in ci/scripts/setup_rust.sh, so that rustup does not install
+# another toolchain while the build runs.
[toolchain]
-channel = "1.88.0"
+channel = "1.90.0"
profile = "minimal"
diff --git a/crates/tantivy_ffi/Cargo.lock b/crates/tantivy_ffi/Cargo.lock
deleted file mode 100644
index be9056ad..00000000
--- a/crates/tantivy_ffi/Cargo.lock
+++ /dev/null
@@ -1,1859 +0,0 @@
-# This file is automatically @generated by Cargo.
-# It is not intended for manual editing.
-version = 4
-
-[[package]]
-name = "adler32"
-version = "1.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234"
-
-[[package]]
-name = "aho-corasick"
-version = "1.1.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301"
-dependencies = [
- "memchr",
-]
-
-[[package]]
-name = "allocator-api2"
-version = "0.2.21"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923"
-
-[[package]]
-name = "allocator-api2"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c880a97d28a3681c0267bd29cff89621202715b065127cd445fa0f0fe0aa2880"
-
-[[package]]
-name = "anstream"
-version = "1.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d"
-dependencies = [
- "anstyle",
- "anstyle-parse",
- "anstyle-query",
- "anstyle-wincon",
- "colorchoice",
- "is_terminal_polyfill",
- "utf8parse",
-]
-
-[[package]]
-name = "anstyle"
-version = "1.0.14"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000"
-
-[[package]]
-name = "anstyle-parse"
-version = "1.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e"
-dependencies = [
- "utf8parse",
-]
-
-[[package]]
-name = "anstyle-query"
-version = "1.1.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
-dependencies = [
- "windows-sys 0.61.2",
-]
-
-[[package]]
-name = "anstyle-wincon"
-version = "3.0.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
-dependencies = [
- "anstyle",
- "once_cell_polyfill",
- "windows-sys 0.61.2",
-]
-
-[[package]]
-name = "anyhow"
-version = "1.0.102"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c"
-
-[[package]]
-name = "arc-swap"
-version = "1.9.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6a3a1fd6f75306b68087b831f025c712524bcb19aad54e557b1129cfa0a2b207"
-dependencies = [
- "rustversion",
-]
-
-[[package]]
-name = "async-trait"
-version = "0.1.89"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
-
-[[package]]
-name = "autocfg"
-version = "1.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8"
-
-[[package]]
-name = "base64"
-version = "0.22.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
-
-[[package]]
-name = "bitflags"
-version = "2.11.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3"
-
-[[package]]
-name = "bitpacking"
-version = "0.9.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "96a7139abd3d9cebf8cd6f920a389cf3dc9576172e32f4563f188cae3c3eb019"
-dependencies = [
- "crunchy",
-]
-
-[[package]]
-name = "bumpalo"
-version = "3.20.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb"
-
-[[package]]
-name = "byteorder"
-version = "1.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
-
-[[package]]
-name = "cbindgen"
-version = "0.29.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "befbfd072a8e81c02f8c507aefce431fe5e7d051f83d48a23ffc9b9fe5a11799"
-dependencies = [
- "clap",
- "heck",
- "indexmap",
- "log",
- "proc-macro2",
- "quote",
- "serde",
- "serde_json",
- "syn",
- "tempfile",
- "toml",
-]
-
-[[package]]
-name = "cc"
-version = "1.2.60"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "43c5703da9466b66a946814e1adf53ea2c90f10063b86290cc9eb67ce3478a20"
-dependencies = [
- "find-msvc-tools",
- "jobserver",
- "libc",
- "shlex",
-]
-
-[[package]]
-name = "cedarwood"
-version = "0.4.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6d910bedd62c24733263d0bed247460853c9d22e8956bd4cd964302095e04e90"
-dependencies = [
- "smallvec",
-]
-
-[[package]]
-name = "census"
-version = "0.4.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4f4c707c6a209cbe82d10abd08e1ea8995e9ea937d2550646e02798948992be0"
-
-[[package]]
-name = "cfg-if"
-version = "1.0.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
-
-[[package]]
-name = "clap"
-version = "4.6.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51"
-dependencies = [
- "clap_builder",
-]
-
-[[package]]
-name = "clap_builder"
-version = "4.6.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f"
-dependencies = [
- "anstream",
- "anstyle",
- "clap_lex",
- "strsim",
-]
-
-[[package]]
-name = "clap_lex"
-version = "1.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9"
-
-[[package]]
-name = "colorchoice"
-version = "1.0.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570"
-
-[[package]]
-name = "crc32fast"
-version = "1.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511"
-dependencies = [
- "cfg-if",
-]
-
-[[package]]
-name = "croaring"
-version = "2.6.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d0e813b58ac55ac5ccea5ec63beb8c80f37dedd78da3f594c848313415a08c8c"
-dependencies = [
- "allocator-api2 0.4.0",
- "croaring-sys",
-]
-
-[[package]]
-name = "croaring-sys"
-version = "4.6.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f34e9ee8e65c0d46c9d0fe55ce80b477d0bfae4c786c6694687b9c70e8267027"
-dependencies = [
- "cc",
-]
-
-[[package]]
-name = "crossbeam-channel"
-version = "0.5.15"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2"
-dependencies = [
- "crossbeam-utils",
-]
-
-[[package]]
-name = "crossbeam-deque"
-version = "0.8.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51"
-dependencies = [
- "crossbeam-epoch",
- "crossbeam-utils",
-]
-
-[[package]]
-name = "crossbeam-epoch"
-version = "0.9.18"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
-dependencies = [
- "crossbeam-utils",
-]
-
-[[package]]
-name = "crossbeam-utils"
-version = "0.8.21"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"
-
-[[package]]
-name = "crunchy"
-version = "0.2.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5"
-
-[[package]]
-name = "dary_heap"
-version = "0.3.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8b1e3a325bc115f096c8b77bbf027a7c2592230e70be2d985be950d3d5e60ebe"
-
-[[package]]
-name = "deranged"
-version = "0.5.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c"
-dependencies = [
- "powerfmt",
- "serde_core",
-]
-
-[[package]]
-name = "downcast-rs"
-version = "1.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2"
-
-[[package]]
-name = "either"
-version = "1.15.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"
-
-[[package]]
-name = "equivalent"
-version = "1.0.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
-
-[[package]]
-name = "errno"
-version = "0.3.14"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
-dependencies = [
- "libc",
- "windows-sys 0.61.2",
-]
-
-[[package]]
-name = "fastdivide"
-version = "0.4.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9afc2bd4d5a73106dd53d10d73d3401c2f32730ba2c0b93ddb888a8983680471"
-
-[[package]]
-name = "fastrand"
-version = "2.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6"
-
-[[package]]
-name = "find-msvc-tools"
-version = "0.1.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
-
-[[package]]
-name = "fnv"
-version = "1.0.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
-
-[[package]]
-name = "foldhash"
-version = "0.1.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
-
-[[package]]
-name = "foldhash"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb"
-
-[[package]]
-name = "fs4"
-version = "0.8.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f7e180ac76c23b45e767bd7ae9579bc0bb458618c4bc71835926e098e61d15f8"
-dependencies = [
- "rustix 0.38.44",
- "windows-sys 0.52.0",
-]
-
-[[package]]
-name = "fxhash"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c"
-dependencies = [
- "byteorder",
-]
-
-[[package]]
-name = "getrandom"
-version = "0.2.17"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0"
-dependencies = [
- "cfg-if",
- "libc",
- "wasi",
-]
-
-[[package]]
-name = "getrandom"
-version = "0.3.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd"
-dependencies = [
- "cfg-if",
- "libc",
- "r-efi 5.3.0",
- "wasip2",
-]
-
-[[package]]
-name = "getrandom"
-version = "0.4.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555"
-dependencies = [
- "cfg-if",
- "libc",
- "r-efi 6.0.0",
- "wasip2",
- "wasip3",
-]
-
-[[package]]
-name = "hashbrown"
-version = "0.15.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1"
-dependencies = [
- "allocator-api2 0.2.21",
- "equivalent",
- "foldhash 0.1.5",
-]
-
-[[package]]
-name = "hashbrown"
-version = "0.16.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100"
-dependencies = [
- "allocator-api2 0.2.21",
- "equivalent",
- "foldhash 0.2.0",
-]
-
-[[package]]
-name = "hashbrown"
-version = "0.17.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51"
-
-[[package]]
-name = "heck"
-version = "0.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
-
-[[package]]
-name = "hermit-abi"
-version = "0.5.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c"
-
-[[package]]
-name = "htmlescape"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e9025058dae765dee5070ec375f591e2ba14638c63feff74f13805a72e523163"
-
-[[package]]
-name = "id-arena"
-version = "2.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954"
-
-[[package]]
-name = "include-flate"
-version = "0.3.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "23e233413926ef735f7d87024466cfda5a4b87467730846bd82ea7d504121347"
-dependencies = [
- "include-flate-codegen",
- "include-flate-compress",
-]
-
-[[package]]
-name = "include-flate-codegen"
-version = "0.3.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5e7148f24ef8922cc0e5574ebb908729ccdd3a110c440a45165733fedadd9969"
-dependencies = [
- "include-flate-compress",
- "proc-macro-error2",
- "proc-macro2",
- "quote",
- "syn",
-]
-
-[[package]]
-name = "include-flate-compress"
-version = "0.3.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "74783a9ed407e844e99d5e7a57bd650acbfa124cf6e97ffd790ba59d8ab8e7ff"
-dependencies = [
- "libflate",
- "zstd",
-]
-
-[[package]]
-name = "indexmap"
-version = "2.14.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9"
-dependencies = [
- "equivalent",
- "hashbrown 0.17.0",
- "serde",
- "serde_core",
-]
-
-[[package]]
-name = "instant"
-version = "0.1.13"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222"
-dependencies = [
- "cfg-if",
- "js-sys",
- "wasm-bindgen",
- "web-sys",
-]
-
-[[package]]
-name = "is_terminal_polyfill"
-version = "1.70.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695"
-
-[[package]]
-name = "itertools"
-version = "0.12.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569"
-dependencies = [
- "either",
-]
-
-[[package]]
-name = "itoa"
-version = "1.0.18"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
-
-[[package]]
-name = "jieba-macros"
-version = "0.7.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7c676b32a471d3cfae8dac2ad2f8334cd52e53377733cca8c1fb0a5062fec192"
-dependencies = [
- "phf_codegen",
-]
-
-[[package]]
-name = "jieba-rs"
-version = "0.7.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f5dd552bbb95d578520ee68403bf8aaf0dbbb2ce55b0854d019f9350ad61040a"
-dependencies = [
- "cedarwood",
- "fxhash",
- "include-flate",
- "jieba-macros",
- "lazy_static",
- "phf",
- "regex",
-]
-
-[[package]]
-name = "jobserver"
-version = "0.1.34"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33"
-dependencies = [
- "getrandom 0.3.4",
- "libc",
-]
-
-[[package]]
-name = "js-sys"
-version = "0.3.95"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2964e92d1d9dc3364cae4d718d93f227e3abb088e747d92e0395bfdedf1c12ca"
-dependencies = [
- "once_cell",
- "wasm-bindgen",
-]
-
-[[package]]
-name = "lazy_static"
-version = "1.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
-
-[[package]]
-name = "leb128fmt"
-version = "0.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2"
-
-[[package]]
-name = "levenshtein_automata"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c2cdeb66e45e9f36bfad5bbdb4d2384e70936afbee843c6f6543f0c551ebb25"
-
-[[package]]
-name = "libc"
-version = "0.2.185"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "52ff2c0fe9bc6cb6b14a0592c2ff4fa9ceb83eea9db979b0487cd054946a2b8f"
-
-[[package]]
-name = "libflate"
-version = "2.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cd96e993e5f3368b0cb8497dae6c860c22af8ff18388c61c6c0b86c58d86b5df"
-dependencies = [
- "adler32",
- "crc32fast",
- "dary_heap",
- "libflate_lz77",
- "no_std_io2",
-]
-
-[[package]]
-name = "libflate_lz77"
-version = "2.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ff7a10e427698aef6eef269482776debfef63384d30f13aad39a1a95e0e098fd"
-dependencies = [
- "hashbrown 0.16.1",
- "no_std_io2",
- "rle-decode-fast",
-]
-
-[[package]]
-name = "libm"
-version = "0.2.16"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981"
-
-[[package]]
-name = "linux-raw-sys"
-version = "0.4.15"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab"
-
-[[package]]
-name = "linux-raw-sys"
-version = "0.12.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53"
-
-[[package]]
-name = "log"
-version = "0.4.29"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897"
-
-[[package]]
-name = "lru"
-version = "0.12.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38"
-dependencies = [
- "hashbrown 0.15.5",
-]
-
-[[package]]
-name = "lz4_flex"
-version = "0.11.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "373f5eceeeab7925e0c1098212f2fbc4d416adec9d35051a6ab251e824c1854a"
-
-[[package]]
-name = "measure_time"
-version = "0.8.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dbefd235b0aadd181626f281e1d684e116972988c14c264e42069d5e8a5775cc"
-dependencies = [
- "instant",
- "log",
-]
-
-[[package]]
-name = "memchr"
-version = "2.8.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"
-
-[[package]]
-name = "memmap2"
-version = "0.9.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3"
-dependencies = [
- "libc",
-]
-
-[[package]]
-name = "minimal-lexical"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
-
-[[package]]
-name = "murmurhash32"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2195bf6aa996a481483b29d62a7663eed3fe39600c460e323f8ff41e90bdd89b"
-
-[[package]]
-name = "no_std_io2"
-version = "0.9.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b51ed7824b6e07d354605f4abb3d9d300350701299da96642ee084f5ce631550"
-dependencies = [
- "memchr",
-]
-
-[[package]]
-name = "nom"
-version = "7.1.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"
-dependencies = [
- "memchr",
- "minimal-lexical",
-]
-
-[[package]]
-name = "num-conv"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967"
-
-[[package]]
-name = "num-traits"
-version = "0.2.19"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
-dependencies = [
- "autocfg",
- "libm",
-]
-
-[[package]]
-name = "num_cpus"
-version = "1.17.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b"
-dependencies = [
- "hermit-abi",
- "libc",
-]
-
-[[package]]
-name = "once_cell"
-version = "1.21.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
-
-[[package]]
-name = "once_cell_polyfill"
-version = "1.70.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
-
-[[package]]
-name = "oneshot"
-version = "0.1.13"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "269bca4c2591a28585d6bf10d9ed0332b7d76900a1b02bec41bdc3a2cdcda107"
-
-[[package]]
-name = "ownedbytes"
-version = "0.7.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c3a059efb063b8f425b948e042e6b9bd85edfe60e913630ed727b23e2dfcc558"
-dependencies = [
- "stable_deref_trait",
-]
-
-[[package]]
-name = "paimon_tantivy_ffi"
-version = "0.1.0"
-dependencies = [
- "cbindgen",
- "croaring",
- "jieba-rs",
- "log",
- "tantivy",
- "tempfile",
-]
-
-[[package]]
-name = "phf"
-version = "0.11.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078"
-dependencies = [
- "phf_shared",
-]
-
-[[package]]
-name = "phf_codegen"
-version = "0.11.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a"
-dependencies = [
- "phf_generator",
- "phf_shared",
-]
-
-[[package]]
-name = "phf_generator"
-version = "0.11.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d"
-dependencies = [
- "phf_shared",
- "rand",
-]
-
-[[package]]
-name = "phf_shared"
-version = "0.11.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5"
-dependencies = [
- "siphasher",
-]
-
-[[package]]
-name = "pkg-config"
-version = "0.3.33"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e"
-
-[[package]]
-name = "powerfmt"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
-
-[[package]]
-name = "ppv-lite86"
-version = "0.2.21"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9"
-dependencies = [
- "zerocopy",
-]
-
-[[package]]
-name = "prettyplease"
-version = "0.2.37"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b"
-dependencies = [
- "proc-macro2",
- "syn",
-]
-
-[[package]]
-name = "proc-macro-error-attr2"
-version = "2.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5"
-dependencies = [
- "proc-macro2",
- "quote",
-]
-
-[[package]]
-name = "proc-macro-error2"
-version = "2.0.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802"
-dependencies = [
- "proc-macro-error-attr2",
- "proc-macro2",
- "quote",
- "syn",
-]
-
-[[package]]
-name = "proc-macro2"
-version = "1.0.106"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934"
-dependencies = [
- "unicode-ident",
-]
-
-[[package]]
-name = "quote"
-version = "1.0.45"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924"
-dependencies = [
- "proc-macro2",
-]
-
-[[package]]
-name = "r-efi"
-version = "5.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
-
-[[package]]
-name = "r-efi"
-version = "6.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf"
-
-[[package]]
-name = "rand"
-version = "0.8.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a"
-dependencies = [
- "libc",
- "rand_chacha",
- "rand_core",
-]
-
-[[package]]
-name = "rand_chacha"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
-dependencies = [
- "ppv-lite86",
- "rand_core",
-]
-
-[[package]]
-name = "rand_core"
-version = "0.6.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
-dependencies = [
- "getrandom 0.2.17",
-]
-
-[[package]]
-name = "rand_distr"
-version = "0.4.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31"
-dependencies = [
- "num-traits",
- "rand",
-]
-
-[[package]]
-name = "rayon"
-version = "1.12.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d"
-dependencies = [
- "either",
- "rayon-core",
-]
-
-[[package]]
-name = "rayon-core"
-version = "1.13.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91"
-dependencies = [
- "crossbeam-deque",
- "crossbeam-utils",
-]
-
-[[package]]
-name = "regex"
-version = "1.12.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276"
-dependencies = [
- "aho-corasick",
- "memchr",
- "regex-automata",
- "regex-syntax",
-]
-
-[[package]]
-name = "regex-automata"
-version = "0.4.14"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f"
-dependencies = [
- "aho-corasick",
- "memchr",
- "regex-syntax",
-]
-
-[[package]]
-name = "regex-syntax"
-version = "0.8.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a"
-
-[[package]]
-name = "rle-decode-fast"
-version = "1.0.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3582f63211428f83597b51b2ddb88e2a91a9d52d12831f9d08f5e624e8977422"
-
-[[package]]
-name = "rust-stemmers"
-version = "1.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e46a2036019fdb888131db7a4c847a1063a7493f971ed94ea82c67eada63ca54"
-dependencies = [
- "serde",
- "serde_derive",
-]
-
-[[package]]
-name = "rustc-hash"
-version = "1.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
-
-[[package]]
-name = "rustix"
-version = "0.38.44"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154"
-dependencies = [
- "bitflags",
- "errno",
- "libc",
- "linux-raw-sys 0.4.15",
- "windows-sys 0.59.0",
-]
-
-[[package]]
-name = "rustix"
-version = "1.1.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190"
-dependencies = [
- "bitflags",
- "errno",
- "libc",
- "linux-raw-sys 0.12.1",
- "windows-sys 0.61.2",
-]
-
-[[package]]
-name = "rustversion"
-version = "1.0.22"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d"
-
-[[package]]
-name = "semver"
-version = "1.0.28"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd"
-
-[[package]]
-name = "serde"
-version = "1.0.228"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e"
-dependencies = [
- "serde_core",
- "serde_derive",
-]
-
-[[package]]
-name = "serde_core"
-version = "1.0.228"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad"
-dependencies = [
- "serde_derive",
-]
-
-[[package]]
-name = "serde_derive"
-version = "1.0.228"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
-
-[[package]]
-name = "serde_json"
-version = "1.0.149"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86"
-dependencies = [
- "itoa",
- "memchr",
- "serde",
- "serde_core",
- "zmij",
-]
-
-[[package]]
-name = "serde_spanned"
-version = "1.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26"
-dependencies = [
- "serde_core",
-]
-
-[[package]]
-name = "shlex"
-version = "1.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
-
-[[package]]
-name = "siphasher"
-version = "1.0.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b2aa850e253778c88a04c3d7323b043aeda9d3e30d5971937c1855769763678e"
-
-[[package]]
-name = "sketches-ddsketch"
-version = "0.2.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "85636c14b73d81f541e525f585c0a2109e6744e1565b5c1668e31c70c10ed65c"
-dependencies = [
- "serde",
-]
-
-[[package]]
-name = "smallvec"
-version = "1.15.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03"
-
-[[package]]
-name = "stable_deref_trait"
-version = "1.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596"
-
-[[package]]
-name = "strsim"
-version = "0.11.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
-
-[[package]]
-name = "syn"
-version = "2.0.117"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99"
-dependencies = [
- "proc-macro2",
- "quote",
- "unicode-ident",
-]
-
-[[package]]
-name = "tantivy"
-version = "0.22.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "96599ea6fccd844fc833fed21d2eecac2e6a7c1afd9e044057391d78b1feb141"
-dependencies = [
- "aho-corasick",
- "arc-swap",
- "base64",
- "bitpacking",
- "byteorder",
- "census",
- "crc32fast",
- "crossbeam-channel",
- "downcast-rs",
- "fastdivide",
- "fnv",
- "fs4",
- "htmlescape",
- "itertools",
- "levenshtein_automata",
- "log",
- "lru",
- "lz4_flex",
- "measure_time",
- "memmap2",
- "num_cpus",
- "once_cell",
- "oneshot",
- "rayon",
- "regex",
- "rust-stemmers",
- "rustc-hash",
- "serde",
- "serde_json",
- "sketches-ddsketch",
- "smallvec",
- "tantivy-bitpacker",
- "tantivy-columnar",
- "tantivy-common",
- "tantivy-fst",
- "tantivy-query-grammar",
- "tantivy-stacker",
- "tantivy-tokenizer-api",
- "tempfile",
- "thiserror",
- "time",
- "uuid",
- "winapi",
-]
-
-[[package]]
-name = "tantivy-bitpacker"
-version = "0.6.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "284899c2325d6832203ac6ff5891b297fc5239c3dc754c5bc1977855b23c10df"
-dependencies = [
- "bitpacking",
-]
-
-[[package]]
-name = "tantivy-columnar"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "12722224ffbe346c7fec3275c699e508fd0d4710e629e933d5736ec524a1f44e"
-dependencies = [
- "downcast-rs",
- "fastdivide",
- "itertools",
- "serde",
- "tantivy-bitpacker",
- "tantivy-common",
- "tantivy-sstable",
- "tantivy-stacker",
-]
-
-[[package]]
-name = "tantivy-common"
-version = "0.7.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8019e3cabcfd20a1380b491e13ff42f57bb38bf97c3d5fa5c07e50816e0621f4"
-dependencies = [
- "async-trait",
- "byteorder",
- "ownedbytes",
- "serde",
- "time",
-]
-
-[[package]]
-name = "tantivy-fst"
-version = "0.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d60769b80ad7953d8a7b2c70cdfe722bbcdcac6bccc8ac934c40c034d866fc18"
-dependencies = [
- "byteorder",
- "regex-syntax",
- "utf8-ranges",
-]
-
-[[package]]
-name = "tantivy-query-grammar"
-version = "0.22.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "847434d4af57b32e309f4ab1b4f1707a6c566656264caa427ff4285c4d9d0b82"
-dependencies = [
- "nom",
-]
-
-[[package]]
-name = "tantivy-sstable"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c69578242e8e9fc989119f522ba5b49a38ac20f576fc778035b96cc94f41f98e"
-dependencies = [
- "tantivy-bitpacker",
- "tantivy-common",
- "tantivy-fst",
- "zstd",
-]
-
-[[package]]
-name = "tantivy-stacker"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c56d6ff5591fc332739b3ce7035b57995a3ce29a93ffd6012660e0949c956ea8"
-dependencies = [
- "murmurhash32",
- "rand_distr",
- "tantivy-common",
-]
-
-[[package]]
-name = "tantivy-tokenizer-api"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2a0dcade25819a89cfe6f17d932c9cedff11989936bf6dd4f336d50392053b04"
-dependencies = [
- "serde",
-]
-
-[[package]]
-name = "tempfile"
-version = "3.27.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
-dependencies = [
- "fastrand",
- "getrandom 0.4.2",
- "once_cell",
- "rustix 1.1.4",
- "windows-sys 0.61.2",
-]
-
-[[package]]
-name = "thiserror"
-version = "1.0.69"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52"
-dependencies = [
- "thiserror-impl",
-]
-
-[[package]]
-name = "thiserror-impl"
-version = "1.0.69"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
-
-[[package]]
-name = "time"
-version = "0.3.47"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c"
-dependencies = [
- "deranged",
- "itoa",
- "num-conv",
- "powerfmt",
- "serde_core",
- "time-core",
- "time-macros",
-]
-
-[[package]]
-name = "time-core"
-version = "0.1.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca"
-
-[[package]]
-name = "time-macros"
-version = "0.2.27"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215"
-dependencies = [
- "num-conv",
- "time-core",
-]
-
-[[package]]
-name = "toml"
-version = "0.9.12+spec-1.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cf92845e79fc2e2def6a5d828f0801e29a2f8acc037becc5ab08595c7d5e9863"
-dependencies = [
- "indexmap",
- "serde_core",
- "serde_spanned",
- "toml_datetime",
- "toml_parser",
- "toml_writer",
- "winnow 0.7.15",
-]
-
-[[package]]
-name = "toml_datetime"
-version = "0.7.5+spec-1.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "92e1cfed4a3038bc5a127e35a2d360f145e1f4b971b551a2ba5fd7aedf7e1347"
-dependencies = [
- "serde_core",
-]
-
-[[package]]
-name = "toml_parser"
-version = "1.1.2+spec-1.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526"
-dependencies = [
- "winnow 1.0.1",
-]
-
-[[package]]
-name = "toml_writer"
-version = "1.1.1+spec-1.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "756daf9b1013ebe47a8776667b466417e2d4c5679d441c26230efd9ef78692db"
-
-[[package]]
-name = "unicode-ident"
-version = "1.0.24"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
-
-[[package]]
-name = "unicode-xid"
-version = "0.2.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"
-
-[[package]]
-name = "utf8-ranges"
-version = "1.0.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7fcfc827f90e53a02eaef5e535ee14266c1d569214c6aa70133a624d8a3164ba"
-
-[[package]]
-name = "utf8parse"
-version = "0.2.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
-
-[[package]]
-name = "uuid"
-version = "1.23.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ddd74a9687298c6858e9b88ec8935ec45d22e8fd5e6394fa1bd4e99a87789c76"
-dependencies = [
- "getrandom 0.4.2",
- "js-sys",
- "serde_core",
- "wasm-bindgen",
-]
-
-[[package]]
-name = "wasi"
-version = "0.11.1+wasi-snapshot-preview1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b"
-
-[[package]]
-name = "wasip2"
-version = "1.0.3+wasi-0.2.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6"
-dependencies = [
- "wit-bindgen 0.57.1",
-]
-
-[[package]]
-name = "wasip3"
-version = "0.4.0+wasi-0.3.0-rc-2026-01-06"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5"
-dependencies = [
- "wit-bindgen 0.51.0",
-]
-
-[[package]]
-name = "wasm-bindgen"
-version = "0.2.118"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0bf938a0bacb0469e83c1e148908bd7d5a6010354cf4fb73279b7447422e3a89"
-dependencies = [
- "cfg-if",
- "once_cell",
- "rustversion",
- "wasm-bindgen-macro",
- "wasm-bindgen-shared",
-]
-
-[[package]]
-name = "wasm-bindgen-macro"
-version = "0.2.118"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eeff24f84126c0ec2db7a449f0c2ec963c6a49efe0698c4242929da037ca28ed"
-dependencies = [
- "quote",
- "wasm-bindgen-macro-support",
-]
-
-[[package]]
-name = "wasm-bindgen-macro-support"
-version = "0.2.118"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9d08065faf983b2b80a79fd87d8254c409281cf7de75fc4b773019824196c904"
-dependencies = [
- "bumpalo",
- "proc-macro2",
- "quote",
- "syn",
- "wasm-bindgen-shared",
-]
-
-[[package]]
-name = "wasm-bindgen-shared"
-version = "0.2.118"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5fd04d9e306f1907bd13c6361b5c6bfc7b3b3c095ed3f8a9246390f8dbdee129"
-dependencies = [
- "unicode-ident",
-]
-
-[[package]]
-name = "wasm-encoder"
-version = "0.244.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319"
-dependencies = [
- "leb128fmt",
- "wasmparser",
-]
-
-[[package]]
-name = "wasm-metadata"
-version = "0.244.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909"
-dependencies = [
- "anyhow",
- "indexmap",
- "wasm-encoder",
- "wasmparser",
-]
-
-[[package]]
-name = "wasmparser"
-version = "0.244.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe"
-dependencies = [
- "bitflags",
- "hashbrown 0.15.5",
- "indexmap",
- "semver",
-]
-
-[[package]]
-name = "web-sys"
-version = "0.3.95"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4f2dfbb17949fa2088e5d39408c48368947b86f7834484e87b73de55bc14d97d"
-dependencies = [
- "js-sys",
- "wasm-bindgen",
-]
-
-[[package]]
-name = "winapi"
-version = "0.3.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
-dependencies = [
- "winapi-i686-pc-windows-gnu",
- "winapi-x86_64-pc-windows-gnu",
-]
-
-[[package]]
-name = "winapi-i686-pc-windows-gnu"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
-
-[[package]]
-name = "winapi-x86_64-pc-windows-gnu"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
-
-[[package]]
-name = "windows-link"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
-
-[[package]]
-name = "windows-sys"
-version = "0.52.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
-dependencies = [
- "windows-targets",
-]
-
-[[package]]
-name = "windows-sys"
-version = "0.59.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
-dependencies = [
- "windows-targets",
-]
-
-[[package]]
-name = "windows-sys"
-version = "0.61.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc"
-dependencies = [
- "windows-link",
-]
-
-[[package]]
-name = "windows-targets"
-version = "0.52.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
-dependencies = [
- "windows_aarch64_gnullvm",
- "windows_aarch64_msvc",
- "windows_i686_gnu",
- "windows_i686_gnullvm",
- "windows_i686_msvc",
- "windows_x86_64_gnu",
- "windows_x86_64_gnullvm",
- "windows_x86_64_msvc",
-]
-
-[[package]]
-name = "windows_aarch64_gnullvm"
-version = "0.52.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
-
-[[package]]
-name = "windows_aarch64_msvc"
-version = "0.52.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
-
-[[package]]
-name = "windows_i686_gnu"
-version = "0.52.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
-
-[[package]]
-name = "windows_i686_gnullvm"
-version = "0.52.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
-
-[[package]]
-name = "windows_i686_msvc"
-version = "0.52.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
-
-[[package]]
-name = "windows_x86_64_gnu"
-version = "0.52.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
-
-[[package]]
-name = "windows_x86_64_gnullvm"
-version = "0.52.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
-
-[[package]]
-name = "windows_x86_64_msvc"
-version = "0.52.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
-
-[[package]]
-name = "winnow"
-version = "0.7.15"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945"
-
-[[package]]
-name = "winnow"
-version = "1.0.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "09dac053f1cd375980747450bfc7250c264eaae0583872e845c0c7cd578872b5"
-
-[[package]]
-name = "wit-bindgen"
-version = "0.51.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5"
-dependencies = [
- "wit-bindgen-rust-macro",
-]
-
-[[package]]
-name = "wit-bindgen"
-version = "0.57.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e"
-
-[[package]]
-name = "wit-bindgen-core"
-version = "0.51.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc"
-dependencies = [
- "anyhow",
- "heck",
- "wit-parser",
-]
-
-[[package]]
-name = "wit-bindgen-rust"
-version = "0.51.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21"
-dependencies = [
- "anyhow",
- "heck",
- "indexmap",
- "prettyplease",
- "syn",
- "wasm-metadata",
- "wit-bindgen-core",
- "wit-component",
-]
-
-[[package]]
-name = "wit-bindgen-rust-macro"
-version = "0.51.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a"
-dependencies = [
- "anyhow",
- "prettyplease",
- "proc-macro2",
- "quote",
- "syn",
- "wit-bindgen-core",
- "wit-bindgen-rust",
-]
-
-[[package]]
-name = "wit-component"
-version = "0.244.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2"
-dependencies = [
- "anyhow",
- "bitflags",
- "indexmap",
- "log",
- "serde",
- "serde_derive",
- "serde_json",
- "wasm-encoder",
- "wasm-metadata",
- "wasmparser",
- "wit-parser",
-]
-
-[[package]]
-name = "wit-parser"
-version = "0.244.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736"
-dependencies = [
- "anyhow",
- "id-arena",
- "indexmap",
- "log",
- "semver",
- "serde",
- "serde_derive",
- "serde_json",
- "unicode-xid",
- "wasmparser",
-]
-
-[[package]]
-name = "zerocopy"
-version = "0.8.48"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9"
-dependencies = [
- "zerocopy-derive",
-]
-
-[[package]]
-name = "zerocopy-derive"
-version = "0.8.48"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
-
-[[package]]
-name = "zmij"
-version = "1.0.21"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa"
-
-[[package]]
-name = "zstd"
-version = "0.13.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a"
-dependencies = [
- "zstd-safe",
-]
-
-[[package]]
-name = "zstd-safe"
-version = "7.2.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d"
-dependencies = [
- "zstd-sys",
-]
-
-[[package]]
-name = "zstd-sys"
-version = "2.0.16+zstd.1.5.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748"
-dependencies = [
- "cc",
- "pkg-config",
-]
diff --git a/crates/tantivy_ffi/Cargo.toml b/crates/tantivy_ffi/Cargo.toml
deleted file mode 100644
index ea843bd9..00000000
--- a/crates/tantivy_ffi/Cargo.toml
+++ /dev/null
@@ -1,50 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-[package]
-name = "paimon_tantivy_ffi"
-version = "0.1.0"
-edition = "2021"
-description = "C FFI layer wrapping tantivy + jieba-rs for paimon-cpp tantivy-fts global index"
-license = "Apache-2.0"
-publish = false
-
-[lib]
-name = "paimon_tantivy_ffi"
-# staticlib: linked by CMake + Corrosion into libpaimon_tantivy_ffi.a
-# rlib: lets `cargo test` use native Rust linkage when building test binaries
-crate-type = ["staticlib", "rlib"]
-
-[dependencies]
-tantivy = "0.22"
-jieba-rs = "0.7"
-croaring = "2.0"
-log = "0.4"
-tempfile = "3"
-
-[build-dependencies]
-cbindgen = "0.29"
-
-[profile.release]
-opt-level = 3
-lto = "thin"
-codegen-units = 1
-panic = "abort"
-
-[profile.dev]
-# Errors propagate across the FFI via status codes; a Rust panic must abort
-# rather than unwind across the FFI boundary.
-panic = "abort"
diff --git a/crates/tantivy_ffi/build.rs b/crates/tantivy_ffi/build.rs
deleted file mode 100644
index 72d76b6f..00000000
--- a/crates/tantivy_ffi/build.rs
+++ /dev/null
@@ -1,56 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//! build.rs: runs cbindgen to generate the C header paimon_tantivy_ffi.h.
-//!
-//! Output path: $OUT_DIR/paimon_tantivy_ffi.h
-//! Corrosion (on the CMake side) reads OUT_DIR from cargo metadata and adds the
-//! header to the C++ include path.
-
-use std::env;
-use std::path::PathBuf;
-
-fn main() {
- let crate_dir = env::var("CARGO_MANIFEST_DIR").unwrap();
- let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap());
- let header_path = out_dir.join("paimon_tantivy_ffi.h");
-
- let cfg = cbindgen::Config::from_file(PathBuf::from(&crate_dir).join("cbindgen.toml"))
- .expect("cbindgen.toml must exist at crate root");
-
- match cbindgen::Builder::new()
- .with_crate(&crate_dir)
- .with_config(cfg)
- .generate()
- {
- Ok(bindings) => {
- bindings.write_to_file(&header_path);
- println!(
- "cargo:rerun-if-changed={}",
- PathBuf::from(&crate_dir).join("src").display()
- );
- println!("cargo:rerun-if-changed=cbindgen.toml");
- // Expose the header directory to Corrosion / the upstream CMake build.
- println!("cargo:include={}", out_dir.display());
- eprintln!("cbindgen: wrote {}", header_path.display());
- }
- Err(e) => {
- // cbindgen failure is not necessarily fatal (e.g. CI skips it when the
- // Rust code is unchanged); log a warning and continue.
- eprintln!("cbindgen generation failed: {e:?}");
- }
- }
-}
diff --git a/crates/tantivy_ffi/cbindgen.toml b/crates/tantivy_ffi/cbindgen.toml
deleted file mode 100644
index c2ab5f13..00000000
--- a/crates/tantivy_ffi/cbindgen.toml
+++ /dev/null
@@ -1,82 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# cbindgen config: Rust FFI -> C header generator.
-# Invoked by build.rs, outputs to $OUT_DIR/paimon_tantivy_ffi.h.
-# CMake picks up $OUT_DIR via Corrosion and adds it to the C++ include path.
-
-language = "C"
-
-# Header banner written at the top of the generated file.
-header = """
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * \"License\"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * AUTO-GENERATED by cbindgen from Rust sources under crates/tantivy_ffi - DO NOT EDIT.
- *
- * C ABI for paimon_tantivy_ffi.
- */
-#pragma once
-"""
-
-include_guard = "PAIMON_TANTIVY_FFI_H"
-cpp_compat = true
-pragma_once = false # already written by hand in the header banner above
-documentation = true
-documentation_style = "c"
-line_length = 100
-tab_width = 4
-
-[export]
-# No type prefix (Rust type names already carry the PaimonTantivy... prefix).
-# Function names already start with paimon_tantivy_ (named that way in the Rust source).
-prefix = ""
-# Force-export types that only appear as handles / return values; cbindgen does
-# not export them by default when no FFI function directly takes/returns them,
-# so list them explicitly here.
-include = ["PaimonTantivyStatus"]
-
-[export.rename]
-# Rust enum name -> C typedef name (to avoid duplicated prefixes, etc.)
-
-[fn]
-prefix = ""
-args = "auto"
-rename_args = "None"
-
-[enum]
-rename_variants = "ScreamingSnakeCase"
-prefix_with_name = true
-derive_helper_methods = false
-
-[parse]
-parse_deps = false
diff --git a/crates/tantivy_ffi/rust-toolchain.toml b/crates/tantivy_ffi/rust-toolchain.toml
deleted file mode 100644
index 4c1d88ae..00000000
--- a/crates/tantivy_ffi/rust-toolchain.toml
+++ /dev/null
@@ -1,25 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Pin the Rust toolchain used to build paimon_tantivy_ffi so Corrosion can
-# resolve a non-empty toolchain name during a fresh CMake configure.
-#
-# Only the `channel` is pinned — no extra components, because rustup in
-# CI/containers may lack network access to fetch clippy/rustfmt, and build
-# doesn't need them.
-[toolchain]
-channel = "stable"
-profile = "minimal"
diff --git a/crates/tantivy_ffi/src/buffer.rs b/crates/tantivy_ffi/src/buffer.rs
deleted file mode 100644
index 2d96e89d..00000000
--- a/crates/tantivy_ffi/src/buffer.rs
+++ /dev/null
@@ -1,127 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//! `paimon_tantivy_buffer_t`: Rust-allocated byte buffer returned to C++.
-//!
-//! Contract:
-//! - Buffer is allocated by Rust (as a `Box<[u8]>`)
-//! - C++ reads `data[0..len]`, **must not** write past len
-//! - C++ must call `paimon_tantivy_buffer_free()` exactly once per non-empty buffer
-//! - Empty (len=0) buffer has null `data`; buffer_free accepts it as no-op
-//!
-//! This struct is #[repr(C)] so cbindgen generates a matching C struct.
-
-use std::ptr;
-
-#[repr(C)]
-pub struct PaimonTantivyBuffer {
- /// Pointer to `len` bytes. Null iff len == 0.
- pub data: *mut u8,
- /// Number of valid bytes.
- pub len: usize,
- /// Internal capacity hint for Rust-side reconstruction. C++ treats as opaque.
- pub capacity: usize,
-}
-
-impl PaimonTantivyBuffer {
- /// Build a buffer from owned bytes; consumes the Vec.
- pub(crate) fn from_vec(mut v: Vec<u8>) -> Self {
- if v.is_empty() {
- return Self::empty();
- }
- v.shrink_to_fit();
- let len = v.len();
- let capacity = v.capacity();
- let data = v.as_mut_ptr();
- std::mem::forget(v);
- Self { data, len, capacity }
- }
-
- pub(crate) fn empty() -> Self {
- Self {
- data: ptr::null_mut(),
- len: 0,
- capacity: 0,
- }
- }
-}
-
-/// Free a buffer returned by any Rust FFI function. Safe to call on an empty
-/// buffer (len=0 / data=null). Must only be called once per buffer.
-///
-/// SAFETY: `buf` must be either null, or point to a live `paimon_tantivy_buffer_t`
-/// produced by this crate and not yet freed.
-#[no_mangle]
-pub unsafe extern "C" fn paimon_tantivy_buffer_free(buf: *mut PaimonTantivyBuffer) {
- if buf.is_null() {
- return;
- }
- let b = unsafe { &mut *buf };
- if b.len != 0 && !b.data.is_null() {
- // Reconstruct the Vec<u8> and drop it
- let v = unsafe { Vec::from_raw_parts(b.data, b.len, b.capacity) };
- drop(v);
- }
- b.data = ptr::null_mut();
- b.len = 0;
- b.capacity = 0;
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- #[test]
- fn empty_has_null_data() {
- let b = PaimonTantivyBuffer::empty();
- assert!(b.data.is_null());
- assert_eq!(b.len, 0);
- }
-
- #[test]
- fn from_vec_roundtrip() {
- let src = vec![1u8, 2, 3, 4, 5];
- let src_clone = src.clone();
- let mut b = PaimonTantivyBuffer::from_vec(src);
- assert_eq!(b.len, 5);
- assert!(!b.data.is_null());
- let view: &[u8] = unsafe { std::slice::from_raw_parts(b.data, b.len) };
- assert_eq!(view, src_clone.as_slice());
- unsafe { paimon_tantivy_buffer_free(&mut b) };
- assert!(b.data.is_null());
- assert_eq!(b.len, 0);
- }
-
- #[test]
- fn free_null_is_noop() {
- unsafe { paimon_tantivy_buffer_free(std::ptr::null_mut()) };
- }
-
- #[test]
- fn free_empty_is_noop() {
- let mut b = PaimonTantivyBuffer::empty();
- unsafe { paimon_tantivy_buffer_free(&mut b) };
- }
-
- #[test]
- fn stress_alloc_free() {
- // LSAN would catch any leak
- for i in 0..5_000usize {
- let mut b = PaimonTantivyBuffer::from_vec(vec![42u8; i.min(256)]);
- unsafe { paimon_tantivy_buffer_free(&mut b) };
- }
- }
-}
diff --git a/crates/tantivy_ffi/src/callback_directory.rs b/crates/tantivy_ffi/src/callback_directory.rs
deleted file mode 100644
index 12768f32..00000000
--- a/crates/tantivy_ffi/src/callback_directory.rs
+++ /dev/null
@@ -1,531 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//! PaimonCallbackDirectory: streaming tantivy `Directory` backed by C FFI
-//! callbacks. Replaces the V1 `PaimonDirectory` (RamDirectory wrapper) with a
-//! callback-driven design that mirrors Java paimon-tantivy-jni's `JniDirectory`.
-//!
-//! ## Why callback-based?
-//!
-//! Loading the entire archive (100MB+) into `RamDirectory` at reader
-//! construction would give ~2x archive peak RAM and pay the whole download
-//! cost up front even for small queries. This directory keeps just the
-//! `HashMap<PathBuf, FileMeta>` layout and issues pread calls through the FFI
-//! callback whenever tantivy asks for bytes — peak RAM is ~KB, startup is
-//! ~header size.
-//!
-//! ## Concurrency
-//!
-//! `read_at` is serialized via `stream_mutex` (same as Java JniDir's
-//! `stream_lock`). pread-style callbacks in principle allow concurrent reads,
-//! but some `paimon::InputStream` subclasses (notably `JindoInputStream`)
-//! have shared-state races, so we play it safe.
-
-use std::collections::HashMap;
-use std::ffi::c_void;
-use std::fmt;
-use std::io;
-use std::ops::Range;
-use std::path::{Path, PathBuf};
-use std::sync::{Arc, Mutex};
-
-use tantivy::directory::error::{DeleteError, LockError, OpenReadError, OpenWriteError};
-use tantivy::directory::{
- AntiCallToken, Directory, DirectoryLock, FileHandle, Lock, OwnedBytes, TerminatingWrite,
- WatchCallback, WatchHandle, WritePtr,
-};
-use tantivy::HasLen;
-
-// =========================================================================
-// FFI types
-// =========================================================================
-
-/// pread-style callback table passed from C++ at reader construction.
-///
-/// `ctx` is an opaque pointer to C++'s `StreamCtx` (holding a
-/// `paimon::InputStream`). Rust never dereferences it — only forwards it
-/// into the callback functions. `release` is called exactly once when the
-/// last `Arc<CallbackCtx>` is dropped.
-#[repr(C)]
-pub struct PaimonStreamCallbacks {
- pub ctx: *mut c_void,
- pub read_at:
- extern "C" fn(ctx: *mut c_void, offset: u64, len: usize, out_buf: *mut u8) -> i32,
- pub release: extern "C" fn(ctx: *mut c_void),
-}
-
-// =========================================================================
-// Internal state
-// =========================================================================
-
-#[derive(Clone, Debug)]
-struct FileMeta {
- offset: u64,
- length: u64,
-}
-
-/// RAII wrapper owning the FFI callbacks. On drop, invokes `release(ctx)`.
-/// Shared across clones of `PaimonCallbackDirectory` via `Arc`.
-struct CallbackCtx {
- callbacks: PaimonStreamCallbacks,
-}
-
-impl Drop for CallbackCtx {
- fn drop(&mut self) {
- // Calling an extern "C" fn pointer from safe Rust is legal; the
- // contract safety relies on the C++ side providing a valid ctx.
- (self.callbacks.release)(self.callbacks.ctx);
- }
-}
-
-// Safety: callbacks.ctx is treated as opaque; C++ owner is responsible for
-// the ctx being usable across threads. Rust's stream_mutex serializes
-// read_at calls, and release is only invoked once (when Arc refcount hits 0).
-unsafe impl Send for CallbackCtx {}
-unsafe impl Sync for CallbackCtx {}
-
-// =========================================================================
-// PaimonCallbackDirectory
-// =========================================================================
-
-#[derive(Clone)]
-pub struct PaimonCallbackDirectory {
- /// name → (offset, length) in the stream. Immutable after construction.
- layout: Arc<HashMap<PathBuf, FileMeta>>,
- /// FFI callbacks + their ctx lifetime.
- ctx: Arc<CallbackCtx>,
- /// tantivy writes small atomic files (`.lock`, in some paths `meta.json`)
- /// via `atomic_write`; we keep them in memory instead of pushing back
- /// through C++ (read-only archive). Shared across clones.
- atomic_data: Arc<Mutex<HashMap<PathBuf, Vec<u8>>>>,
- /// Serialize seek+read (mirrors Java JniDir's `stream_lock`) to guard
- /// against shared-state races in some InputStream subclasses.
- stream_mutex: Arc<Mutex<()>>,
-}
-
-impl fmt::Debug for PaimonCallbackDirectory {
- fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
- f.debug_struct("PaimonCallbackDirectory")
- .field("files", &self.layout.keys().collect::<Vec<_>>())
- .finish()
- }
-}
-
-impl PaimonCallbackDirectory {
- /// Construct a new directory from the C++-parsed archive layout + callbacks.
- /// The ctx ownership transfers to this Directory; `release` is invoked on
- /// drop of the last clone.
- pub fn new(
- entries: Vec<(String, u64, u64)>,
- callbacks: PaimonStreamCallbacks,
- ) -> Self {
- let mut layout = HashMap::with_capacity(entries.len());
- for (name, offset, length) in entries {
- layout.insert(PathBuf::from(name), FileMeta { offset, length });
- }
- Self {
- layout: Arc::new(layout),
- ctx: Arc::new(CallbackCtx { callbacks }),
- atomic_data: Arc::new(Mutex::new(HashMap::new())),
- stream_mutex: Arc::new(Mutex::new(())),
- }
- }
-
- /// Perform an FFI pread. Serialized via `stream_mutex`.
- fn pread(&self, offset: u64, len: usize) -> io::Result<Vec<u8>> {
- let _guard = self.stream_mutex.lock().map_err(|e| {
- io::Error::new(io::ErrorKind::Other, format!("stream_mutex poisoned: {e}"))
- })?;
- let mut buf = vec![0u8; len];
- // Calling extern "C" fn pointer — safe from Rust's POV (ABI is C);
- // the contract safety (ctx validity, buffer ownership) is on the C++ side.
- let rc =
- (self.ctx.callbacks.read_at)(self.ctx.callbacks.ctx, offset, len, buf.as_mut_ptr());
- if rc != 0 {
- return Err(io::Error::new(
- io::ErrorKind::Other,
- format!("pread callback rc={rc} offset={offset} len={len}"),
- ));
- }
- Ok(buf)
- }
-
- /// Sorted file names, for diagnostic / test use.
- #[cfg(test)]
- pub(crate) fn file_names(&self) -> Vec<String> {
- let mut names: Vec<String> = self
- .layout
- .keys()
- .map(|p| p.to_string_lossy().into_owned())
- .collect();
- names.sort();
- names
- }
-}
-
-// =========================================================================
-// FileHandle
-// =========================================================================
-
-#[derive(Clone)]
-struct PaimonCallbackFileHandle {
- directory: PaimonCallbackDirectory,
- file_offset: u64,
- file_length: u64,
-}
-
-impl fmt::Debug for PaimonCallbackFileHandle {
- fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
- f.debug_struct("PaimonCallbackFileHandle")
- .field("offset", &self.file_offset)
- .field("length", &self.file_length)
- .finish()
- }
-}
-
-impl HasLen for PaimonCallbackFileHandle {
- fn len(&self) -> usize {
- self.file_length as usize
- }
-}
-
-impl FileHandle for PaimonCallbackFileHandle {
- fn read_bytes(&self, range: Range<usize>) -> io::Result<OwnedBytes> {
- let start = self.file_offset + range.start as u64;
- let len = range.end - range.start;
- let data = self.directory.pread(start, len)?;
- Ok(OwnedBytes::new(data))
- }
-}
-
-// =========================================================================
-// Directory trait (13 methods for tantivy 0.22)
-// =========================================================================
-
-impl Directory for PaimonCallbackDirectory {
- fn get_file_handle(&self, path: &Path) -> Result<Arc<dyn FileHandle>, OpenReadError> {
- let meta = self
- .layout
- .get(path)
- .ok_or_else(|| OpenReadError::FileDoesNotExist(path.to_path_buf()))?;
- Ok(Arc::new(PaimonCallbackFileHandle {
- directory: self.clone(),
- file_offset: meta.offset,
- file_length: meta.length,
- }))
- }
-
- fn exists(&self, path: &Path) -> Result<bool, OpenReadError> {
- let in_layout = self.layout.contains_key(path);
- let in_atomic = self.atomic_data.lock().unwrap().contains_key(path);
- Ok(in_layout || in_atomic)
- }
-
- fn atomic_read(&self, path: &Path) -> Result<Vec<u8>, OpenReadError> {
- if let Some(data) = self.atomic_data.lock().unwrap().get(path) {
- return Ok(data.clone());
- }
- let meta = self
- .layout
- .get(path)
- .ok_or_else(|| OpenReadError::FileDoesNotExist(path.to_path_buf()))?;
- self.pread(meta.offset, meta.length as usize)
- .map_err(|e| OpenReadError::wrap_io_error(e, path.to_path_buf()))
- }
-
- fn atomic_write(&self, path: &Path, data: &[u8]) -> io::Result<()> {
- self.atomic_data
- .lock()
- .unwrap()
- .insert(path.to_path_buf(), data.to_vec());
- Ok(())
- }
-
- fn delete(&self, _path: &Path) -> Result<(), DeleteError> {
- // read-only archive: ignore
- Ok(())
- }
-
- fn open_write(&self, _path: &Path) -> Result<WritePtr, OpenWriteError> {
- // tantivy needs this for lock files when opening an index; provide a
- // dummy in-memory writer (same trick as Java JniDirectory).
- let buf: Vec<u8> = Vec::new();
- Ok(io::BufWriter::new(Box::new(VecTerminatingWrite(buf))))
- }
-
- fn sync_directory(&self) -> io::Result<()> {
- Ok(())
- }
-
- fn acquire_lock(&self, _lock: &Lock) -> Result<DirectoryLock, LockError> {
- // Read-only: no actual locking.
- Ok(DirectoryLock::from(Box::new(())))
- }
-
- fn watch(&self, _watch_callback: WatchCallback) -> tantivy::Result<WatchHandle> {
- Ok(WatchHandle::empty())
- }
-}
-
-/// Throwaway writer for `open_write` — tantivy creates it for lock files but
-/// the bytes never matter in a read-only archive.
-struct VecTerminatingWrite(Vec<u8>);
-
-impl io::Write for VecTerminatingWrite {
- fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
- self.0.extend_from_slice(buf);
- Ok(buf.len())
- }
- fn flush(&mut self) -> io::Result<()> {
- Ok(())
- }
-}
-
-impl TerminatingWrite for VecTerminatingWrite {
- fn terminate_ref(&mut self, _token: AntiCallToken) -> io::Result<()> {
- Ok(())
- }
-}
-
-// =========================================================================
-// Test support (pub(crate) — used by reader.rs tests too)
-// =========================================================================
-
-#[cfg(test)]
-pub(crate) mod test_support {
- use super::*;
- use std::sync::atomic::{AtomicUsize, Ordering};
-
- /// Mock backend: an in-memory buffer serving pread requests. Counters
- /// expose behavior for test assertions (read count / release count).
- pub(crate) struct MockBackend {
- pub data: Vec<u8>,
- pub read_count: AtomicUsize,
- pub release_count: AtomicUsize,
- }
-
- extern "C" fn mock_read_at(
- ctx: *mut c_void,
- offset: u64,
- len: usize,
- out_buf: *mut u8,
- ) -> i32 {
- let backend = unsafe { &*(ctx as *const MockBackend) };
- backend.read_count.fetch_add(1, Ordering::SeqCst);
- let data = &backend.data;
- let end = (offset as usize).saturating_add(len);
- if end > data.len() {
- return 1; // out of range
- }
- unsafe {
- std::ptr::copy_nonoverlapping(data.as_ptr().add(offset as usize), out_buf, len);
- }
- 0
- }
-
- extern "C" fn mock_release(ctx: *mut c_void) {
- // Reclaim the strong ref that `Arc::into_raw` leaked at construction.
- let backend = unsafe { Arc::from_raw(ctx as *const MockBackend) };
- backend.release_count.fetch_add(1, Ordering::SeqCst);
- // `arc` drops here → decrement; test still holds its own clone.
- }
-
- /// Build a mock-backed directory for tests. Returns (dir, backend clone).
- /// The backend Arc is shared — drop the directory to trigger release.
- pub(crate) fn build_mock_directory(
- data: Vec<u8>,
- entries: Vec<(String, u64, u64)>,
- ) -> (PaimonCallbackDirectory, Arc<MockBackend>) {
- let backend = Arc::new(MockBackend {
- data,
- read_count: AtomicUsize::new(0),
- release_count: AtomicUsize::new(0),
- });
- let ctx_ptr = Arc::into_raw(backend.clone()) as *mut c_void;
- let cb = PaimonStreamCallbacks {
- ctx: ctx_ptr,
- read_at: mock_read_at,
- release: mock_release,
- };
- let dir = PaimonCallbackDirectory::new(entries, cb);
- (dir, backend)
- }
-
- /// Build mock callbacks (+ a backend clone) without wrapping them in a
- /// directory, for tests that drive the FFI entry points directly.
- pub(crate) fn make_mock_callbacks(data: Vec<u8>) -> (PaimonStreamCallbacks, Arc<MockBackend>) {
- let backend = Arc::new(MockBackend {
- data,
- read_count: AtomicUsize::new(0),
- release_count: AtomicUsize::new(0),
- });
- let ctx_ptr = Arc::into_raw(backend.clone()) as *mut c_void;
- let cb = PaimonStreamCallbacks {
- ctx: ctx_ptr,
- read_at: mock_read_at,
- release: mock_release,
- };
- (cb, backend)
- }
-
- /// Parse a packed archive blob (BE, no version header, matching
- /// `writer::pack_index_dir`) and build a mock-backed directory. Used by
- /// `reader.rs::tests` since writer.finish currently still returns a Vec<u8>.
- pub(crate) fn build_directory_from_archive(
- packed: Vec<u8>,
- ) -> (PaimonCallbackDirectory, Arc<MockBackend>) {
- let entries = parse_archive_header(&packed);
- build_mock_directory(packed, entries)
- }
-
- /// Parse the archive header — mirrors the layout that
- /// C++ `ArchiveLayout::Parse` produces in production.
- fn parse_archive_header(bytes: &[u8]) -> Vec<(String, u64, u64)> {
- let mut off = 0usize;
- let file_count = i32::from_be_bytes(bytes[off..off + 4].try_into().unwrap()) as usize;
- off += 4;
- let mut entries = Vec::with_capacity(file_count);
- for _ in 0..file_count {
- let nlen = i32::from_be_bytes(bytes[off..off + 4].try_into().unwrap()) as usize;
- off += 4;
- let name =
- std::str::from_utf8(&bytes[off..off + nlen]).unwrap().to_owned();
- off += nlen;
- let flen = i64::from_be_bytes(bytes[off..off + 8].try_into().unwrap()) as u64;
- off += 8;
- let data_offset = off as u64;
- entries.push((name, data_offset, flen));
- off += flen as usize;
- }
- entries
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::test_support::*;
- use super::*;
-
- #[test]
- fn file_handle_reads_correct_bytes() {
- let data = b"hello world".to_vec();
- let entries = vec![("foo.txt".to_string(), 0, 11)];
- let (dir, _backend) = build_mock_directory(data, entries);
-
- let handle = dir.get_file_handle(Path::new("foo.txt")).unwrap();
- let bytes = handle.read_bytes(0..5).unwrap();
- assert_eq!(&bytes[..], b"hello");
- let bytes = handle.read_bytes(6..11).unwrap();
- assert_eq!(&bytes[..], b"world");
- }
-
- #[test]
- fn missing_file_returns_error() {
- let (dir, _backend) = build_mock_directory(vec![], vec![]);
- let err = dir.get_file_handle(Path::new("nonexistent")).unwrap_err();
- match err {
- OpenReadError::FileDoesNotExist(p) => {
- assert_eq!(p.to_string_lossy(), "nonexistent")
- }
- other => panic!("expected FileDoesNotExist, got {other:?}"),
- }
- }
-
- #[test]
- fn pread_out_of_range_propagates_error() {
- let data = b"short".to_vec();
- let entries = vec![("bad.txt".to_string(), 0, 100)]; // length exceeds data
- let (dir, _backend) = build_mock_directory(data, entries);
- let handle = dir.get_file_handle(Path::new("bad.txt")).unwrap();
- let err = handle.read_bytes(0..100).unwrap_err();
- assert_eq!(err.kind(), io::ErrorKind::Other);
- }
-
- #[test]
- fn atomic_write_read_roundtrip_and_exists() {
- let (dir, _backend) = build_mock_directory(vec![], vec![]);
- dir.atomic_write(Path::new(".lock"), b"locked").unwrap();
- let data = dir.atomic_read(Path::new(".lock")).unwrap();
- assert_eq!(data, b"locked");
- assert!(dir.exists(Path::new(".lock")).unwrap());
- assert!(!dir.exists(Path::new("gone")).unwrap());
- }
-
- #[test]
- fn release_called_exactly_once_on_last_drop() {
- let entries = vec![("a".to_string(), 0, 5)];
- let (dir, backend) = build_mock_directory(b"hello".to_vec(), entries);
- assert_eq!(backend.release_count.load(std::sync::atomic::Ordering::SeqCst), 0);
- drop(dir);
- assert_eq!(backend.release_count.load(std::sync::atomic::Ordering::SeqCst), 1);
- }
-
- #[test]
- fn cloned_directory_shares_ctx_and_atomic_data() {
- let (dir, backend) = build_mock_directory(vec![], vec![]);
- let dir2 = dir.clone();
- dir.atomic_write(Path::new("x"), b"hello").unwrap();
- assert!(dir2.exists(Path::new("x")).unwrap()); // shared atomic_data
- drop(dir);
- assert_eq!(backend.release_count.load(std::sync::atomic::Ordering::SeqCst), 0); // ctx still held by dir2
- drop(dir2);
- assert_eq!(backend.release_count.load(std::sync::atomic::Ordering::SeqCst), 1);
- }
-
- #[test]
- fn concurrent_pread_results_correct_under_stream_mutex() {
- use std::thread;
-
- let data: Vec<u8> = (0..1000).map(|i| (i % 256) as u8).collect();
- let entries = vec![("data".to_string(), 0, 1000)];
- let (dir, backend) = build_mock_directory(data.clone(), entries);
- let handle: Arc<dyn FileHandle> =
- dir.get_file_handle(Path::new("data")).unwrap();
-
- let threads: Vec<_> = (0..8)
- .map(|_| {
- let h = handle.clone();
- let expected = data.clone();
- thread::spawn(move || {
- for _ in 0..20 {
- let bytes = h.read_bytes(100..200).unwrap();
- assert_eq!(&bytes[..], &expected[100..200]);
- }
- })
- })
- .collect();
-
- for t in threads {
- t.join().unwrap();
- }
- assert_eq!(
- backend.read_count.load(std::sync::atomic::Ordering::SeqCst),
- 8 * 20
- );
- }
-
- #[test]
- fn file_names_sorted() {
- let entries = vec![
- ("z.idx".to_string(), 0, 10),
- ("a.meta".to_string(), 10, 20),
- ("m.term".to_string(), 30, 5),
- ];
- let (dir, _backend) = build_mock_directory(vec![0u8; 100], entries);
- let names = dir.file_names();
- assert_eq!(names, vec!["a.meta", "m.term", "z.idx"]);
- }
-}
diff --git a/crates/tantivy_ffi/src/error.rs b/crates/tantivy_ffi/src/error.rs
deleted file mode 100644
index 36c36f48..00000000
--- a/crates/tantivy_ffi/src/error.rs
+++ /dev/null
@@ -1,153 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//! Error model for paimon_tantivy_ffi.
-//!
-//! Contract:
-//! - Every fallible FFI function returns `paimon_tantivy_status_t`
-//! - Failure sets `last_error` (thread-local) with human-readable text
-//! - C++ calls `paimon_tantivy_last_error()` after a non-OK status to fetch text
-//! - Pointer returned by `last_error()` is thread-local and valid until the
-//! next failing FFI call on the same thread. C++ must NOT free it.
-
-use std::cell::RefCell;
-use std::ffi::c_char;
-use std::ffi::CString;
-
-/// Status codes. Values are stable ABI; append-only.
-#[repr(i32)]
-#[derive(Copy, Clone, Debug, Eq, PartialEq)]
-pub enum PaimonTantivyStatus {
- Ok = 0,
- InvalidArgument = 1,
- NotFound = 2,
- IoError = 3,
- Unsupported = 4,
- TokenizerError = 5,
- QueryParseError = 6,
- IndexFormatError = 7,
- InternalError = 99,
-}
-
-thread_local! {
- /// Pre-allocated empty string so `paimon_tantivy_last_error()` can always
- /// return a valid non-null pointer.
- static LAST_ERROR: RefCell<CString> = RefCell::new(CString::new("").unwrap());
-}
-
-/// Record an error message for the current thread. Called by fallible FFI
-/// functions right before returning a non-OK status.
-pub(crate) fn set_last_error(msg: impl Into<String>) {
- // Interior nul bytes would make CString::new fail; strip them as a safety net.
- let s: String = msg.into().replace('\0', "\u{FFFD}");
- LAST_ERROR.with(|cell| {
- // CString::new clones the bytes and appends a nul terminator.
- *cell.borrow_mut() = CString::new(s).unwrap_or_else(|_| CString::new("").unwrap());
- });
-}
-
-/// Clear the current thread's error slot. Called at the top of fallible APIs
-/// so a subsequent successful call doesn't return stale text.
-#[allow(dead_code)]
-pub(crate) fn clear_last_error() {
- LAST_ERROR.with(|cell| {
- *cell.borrow_mut() = CString::new("").unwrap();
- });
-}
-
-/// Macro that wraps a `Result<T, String>`-returning block: sets last_error on
-/// Err and returns the given status code; returns Ok value on success.
-#[macro_export]
-macro_rules! ffi_try {
- ($expr:expr, $err_status:expr) => {{
- match $expr {
- Ok(v) => v,
- Err(e) => {
- $crate::error::set_last_error(format!("{e}"));
- return $err_status;
- }
- }
- }};
-}
-
-/// Return the last error text for the calling thread. Always non-null; returns
-/// pointer to "" when there is no error recorded yet. Pointer is thread-local;
-/// C++ must NOT free it; treat as valid until the next failing FFI call on
-/// the same thread.
-#[no_mangle]
-pub extern "C" fn paimon_tantivy_last_error() -> *const c_char {
- LAST_ERROR.with(|cell| cell.borrow().as_ptr())
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
- use std::ffi::CStr;
-
- #[test]
- fn initial_last_error_is_empty() {
- let ptr = paimon_tantivy_last_error();
- assert!(!ptr.is_null());
- let s = unsafe { CStr::from_ptr(ptr) }.to_str().unwrap();
- assert_eq!(s, "");
- }
-
- #[test]
- fn set_then_retrieve() {
- set_last_error("boom");
- let s = unsafe { CStr::from_ptr(paimon_tantivy_last_error()) }
- .to_str()
- .unwrap();
- assert_eq!(s, "boom");
- }
-
- #[test]
- fn clear_resets_to_empty() {
- set_last_error("x");
- clear_last_error();
- let s = unsafe { CStr::from_ptr(paimon_tantivy_last_error()) }
- .to_str()
- .unwrap();
- assert_eq!(s, "");
- }
-
- #[test]
- fn embedded_nul_is_stripped() {
- set_last_error("a\0b");
- let s = unsafe { CStr::from_ptr(paimon_tantivy_last_error()) }
- .to_str()
- .unwrap();
- assert_eq!(s, "a\u{FFFD}b");
- }
-
- #[test]
- fn thread_local_isolation() {
- set_last_error("main");
- let t = std::thread::spawn(|| {
- let s = unsafe { CStr::from_ptr(paimon_tantivy_last_error()) }
- .to_str()
- .unwrap();
- s.to_owned()
- })
- .join()
- .unwrap();
- assert_eq!(t, "");
- let s = unsafe { CStr::from_ptr(paimon_tantivy_last_error()) }
- .to_str()
- .unwrap();
- assert_eq!(s, "main");
- }
-}
diff --git a/crates/tantivy_ffi/src/handle.rs b/crates/tantivy_ffi/src/handle.rs
deleted file mode 100644
index 7bd98e73..00000000
--- a/crates/tantivy_ffi/src/handle.rs
+++ /dev/null
@@ -1,122 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//! Opaque handle helpers.
-//!
-//! Contract:
-//! - Rust creates handles with `Box::into_raw(Box::new(T))`
-//! - C++ must free with the matching `xxx_free(*mut T)` function, once
-//! - Functions accepting handles treat null as invalid argument
-
-use std::ffi::c_void;
-
-/// Consume `T`, return a raw opaque pointer suitable for C++.
-#[inline]
-pub(crate) fn into_handle<T>(value: T) -> *mut T {
- Box::into_raw(Box::new(value))
-}
-
-/// Reconstitute a `Box<T>` from an FFI-provided pointer and drop it.
-/// SAFETY: caller must pass a pointer previously returned by `into_handle::<T>`,
-/// and must not use it again after this call.
-#[inline]
-pub(crate) unsafe fn free_handle<T>(handle: *mut T) {
- if handle.is_null() {
- return;
- }
- drop(unsafe { Box::from_raw(handle) });
-}
-
-/// Borrow an `&T` from an FFI-provided pointer. Returns None on null.
-/// SAFETY: caller must ensure the pointer was previously returned by
-/// `into_handle::<T>` and is still alive (not freed).
-#[inline]
-pub(crate) unsafe fn borrow_handle<'a, T>(handle: *const T) -> Option<&'a T> {
- if handle.is_null() {
- None
- } else {
- Some(unsafe { &*handle })
- }
-}
-
-/// Borrow `&mut T` from an FFI-provided pointer. Returns None on null.
-/// SAFETY: same as `borrow_handle`, plus caller must ensure there is no
-/// concurrent access via another pointer (writer/reader handles are
-/// documented as thread-unsafe).
-#[inline]
-pub(crate) unsafe fn borrow_handle_mut<'a, T>(handle: *mut T) -> Option<&'a mut T> {
- if handle.is_null() {
- None
- } else {
- Some(unsafe { &mut *handle })
- }
-}
-
-/// Opaque ctx pointer from C++ (passed through to Rust Directory callbacks).
-/// Type-erased on purpose: only C++ side knows the concrete type.
-pub(crate) type OpaqueCtx = *mut c_void;
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- #[test]
- fn into_then_free() {
- struct X(i32);
- let h: *mut X = into_handle(X(42));
- assert!(!h.is_null());
- unsafe { free_handle(h) };
- // no leak (LSAN would catch if compiled with sanitizers)
- }
-
- #[test]
- fn free_null_is_noop() {
- let h: *mut i32 = std::ptr::null_mut();
- unsafe { free_handle(h) };
- }
-
- #[test]
- fn borrow_roundtrip() {
- let h = into_handle(42i32);
- unsafe {
- assert_eq!(*borrow_handle(h as *const i32).unwrap(), 42);
- *borrow_handle_mut(h).unwrap() = 7;
- assert_eq!(*borrow_handle(h as *const i32).unwrap(), 7);
- free_handle(h);
- }
- }
-
- #[test]
- fn borrow_null_is_none() {
- unsafe {
- assert!(borrow_handle::<i32>(std::ptr::null()).is_none());
- assert!(borrow_handle_mut::<i32>(std::ptr::null_mut()).is_none());
- }
- }
-
- #[test]
- fn stress_many_create_destroy() {
- // smoke stress: many allocations, no leak
- for i in 0..10_000 {
- let h = into_handle(vec![i; 8]);
- unsafe {
- let v = borrow_handle(h as *const Vec<i32>).unwrap();
- assert_eq!(v.len(), 8);
- free_handle(h);
- }
- }
- }
-}
diff --git a/crates/tantivy_ffi/src/lib.rs b/crates/tantivy_ffi/src/lib.rs
deleted file mode 100644
index df4b80a5..00000000
--- a/crates/tantivy_ffi/src/lib.rs
+++ /dev/null
@@ -1,95 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//! paimon_tantivy_ffi: C ABI layer for tantivy + jieba-rs,
-//! consumed by paimon-cpp's `tantivy-fulltext` global index.
-//!
-//! Modules: error / handle / buffer / log (common FFI layer), tokenizer,
-//! writer, callback directory, and reader (query).
-
-#![deny(unsafe_op_in_unsafe_fn)]
-
-use std::ffi::c_char;
-
-pub mod error;
-pub mod handle;
-pub mod buffer;
-pub mod log_bridge;
-pub mod tokenizer;
-pub mod writer;
-pub mod callback_directory;
-pub mod reader;
-
-// Re-export public FFI symbols at crate root so cbindgen picks them up.
-pub use buffer::{paimon_tantivy_buffer_free, PaimonTantivyBuffer};
-pub use error::{paimon_tantivy_last_error, PaimonTantivyStatus};
-pub use log_bridge::{
- paimon_tantivy_clear_log_callback, paimon_tantivy_set_log_callback, PaimonTantivyLogFn,
-};
-pub use tokenizer::{
- paimon_tantivy_tokenizer_free, paimon_tantivy_tokenizer_new,
- paimon_tantivy_tokenizer_tokenize, PaimonJiebaTokenizer,
-};
-pub use writer::{
- paimon_tantivy_writer_add, paimon_tantivy_writer_finish_streaming,
- paimon_tantivy_writer_free, paimon_tantivy_writer_new, PaimonTantivyWriter,
- PaimonWriteCallbacks,
-};
-pub use callback_directory::{PaimonCallbackDirectory, PaimonStreamCallbacks};
-pub use reader::{
- paimon_tantivy_reader_free, paimon_tantivy_reader_new_streaming,
- paimon_tantivy_reader_search, PaimonTantivyReader,
-};
-
-/// Semantic version of this crate, **'static lifetime**; C++ must NOT free.
-/// Format: `"<semver>"` (git sha postfix can be added later via build.rs).
-/// Returned as a NUL-terminated UTF-8 C string.
-#[no_mangle]
-pub extern "C" fn paimon_tantivy_version() -> *const c_char {
- concat!(env!("CARGO_PKG_VERSION"), "\0").as_ptr() as *const c_char
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
- use std::ffi::CStr;
-
- #[test]
- fn version_is_non_empty() {
- let ptr = paimon_tantivy_version();
- assert!(!ptr.is_null());
- let s = unsafe { CStr::from_ptr(ptr) }.to_str().unwrap();
- assert!(!s.is_empty(), "version must be non-empty");
- assert!(s.contains('.'), "version must look like semver, got {s:?}");
- }
-
- #[test]
- fn tantivy_and_jieba_are_linked() {
- let _ = tantivy::schema::Schema::builder();
- let _ = jieba_rs::Jieba::new();
- }
-
- #[test]
- fn croaring_serialize_roundtrip() {
- use croaring::Bitmap;
- let mut b = Bitmap::new();
- b.add(42);
- b.add(100);
- let bytes = b.serialize::<croaring::Portable>();
- let b2 = Bitmap::deserialize::<croaring::Portable>(&bytes);
- assert_eq!(b.cardinality(), b2.cardinality());
- }
-}
diff --git a/crates/tantivy_ffi/src/log_bridge.rs b/crates/tantivy_ffi/src/log_bridge.rs
deleted file mode 100644
index 6b6b286f..00000000
--- a/crates/tantivy_ffi/src/log_bridge.rs
+++ /dev/null
@@ -1,119 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//! Log bridge: tantivy internally emits log records via the `log` crate
-//! (via `tantivy::debug` / `info` etc.). This module registers a global
-//! `log::Log` implementation that forwards records to a C callback.
-//!
-//! Contract:
-//! - C++ calls `paimon_tantivy_set_log_callback(cb)` once at process startup
-//! - Passing null unregisters (reverts to stderr)
-//! - Callback receives (level, msg_ptr, msg_len); pointer is non-null,
-//! UTF-8, NOT null-terminated, valid only for the duration of the call
-//! - Level mapping: 0=trace 1=debug 2=info 3=warn 4=error
-//! - Callback must be thread-safe: tantivy writes from worker threads
-//!
-//! NOTE: tantivy uses `tracing` in newer versions and `log` in others.
-//! Our current `tantivy = "0.22"` uses `log`.
-//! If a future upgrade switches to `tracing`, install a `tracing-log`
-//! bridge here.
-
-use std::ffi::c_char;
-use std::sync::atomic::{AtomicPtr, Ordering};
-
-pub type PaimonTantivyLogFn = extern "C" fn(level: i32, msg: *const c_char, len: usize);
-
-static CALLBACK: AtomicPtr<()> = AtomicPtr::new(std::ptr::null_mut());
-
-struct LogBridge;
-
-impl log::Log for LogBridge {
- fn enabled(&self, _: &log::Metadata) -> bool {
- true
- }
-
- fn log(&self, record: &log::Record) {
- let level = match record.level() {
- log::Level::Trace => 0,
- log::Level::Debug => 1,
- log::Level::Info => 2,
- log::Level::Warn => 3,
- log::Level::Error => 4,
- };
- let msg = format!("[{}] {}", record.target(), record.args());
- let ptr = CALLBACK.load(Ordering::Acquire);
- if ptr.is_null() {
- // Fallback: stderr
- eprintln!("{msg}");
- return;
- }
- // SAFETY: ptr was installed as PaimonTantivyLogFn via transmute below
- let cb: PaimonTantivyLogFn = unsafe { std::mem::transmute(ptr) };
- cb(level, msg.as_ptr() as *const c_char, msg.len());
- }
-
- fn flush(&self) {}
-}
-
-static LOGGER: LogBridge = LogBridge;
-
-/// Install a non-null callback. First call also registers `LogBridge` as
-/// the global `log` crate sink. Subsequent calls swap the callback atomically.
-/// Thread-safety: safe to call from any thread.
-///
-/// Note: we use separate `set`/`clear` functions instead of `Option<fn>`
-/// because cbindgen translates `Option<extern "C" fn>` into an opaque struct
-/// rather than a nullable C function pointer.
-#[no_mangle]
-pub extern "C" fn paimon_tantivy_set_log_callback(cb: PaimonTantivyLogFn) {
- let ptr = cb as *mut ();
- CALLBACK.store(ptr, Ordering::Release);
- let _ = log::set_logger(&LOGGER);
- log::set_max_level(log::LevelFilter::Info);
-}
-
-/// Clear the installed callback (revert to Rust-side stderr fallback).
-#[no_mangle]
-pub extern "C" fn paimon_tantivy_clear_log_callback() {
- CALLBACK.store(std::ptr::null_mut(), Ordering::Release);
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
- use std::sync::atomic::{AtomicUsize, Ordering};
-
- // Simple test callback that counts invocations
- static COUNT: AtomicUsize = AtomicUsize::new(0);
- extern "C" fn counting_cb(_: i32, _: *const c_char, _: usize) {
- COUNT.fetch_add(1, Ordering::SeqCst);
- }
-
- #[test]
- fn install_then_log() {
- COUNT.store(0, Ordering::SeqCst);
- paimon_tantivy_set_log_callback(counting_cb);
- log::info!("hello");
- assert!(COUNT.load(Ordering::SeqCst) >= 1);
- }
-
- #[test]
- fn clear_reverts_to_stderr() {
- paimon_tantivy_set_log_callback(counting_cb);
- paimon_tantivy_clear_log_callback();
- log::warn!("goes to stderr");
- }
-}
diff --git a/crates/tantivy_ffi/src/reader.rs b/crates/tantivy_ffi/src/reader.rs
deleted file mode 100644
index 033b7f25..00000000
--- a/crates/tantivy_ffi/src/reader.rs
+++ /dev/null
@@ -1,1398 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//! PaimonTantivyReader: query side of tantivy-fulltext.
-//!
-//! Constructs a tantivy Index from a packed-blob produced by writer.rs (via
-//! PaimonDirectory), registers the same `paimon_jieba` tokenizer, and runs
-//! one of 5 search types (mirrors `paimon::FullTextSearch::SearchType`):
-//!
-//! 1 MATCH_ALL — tokenize query, BooleanQuery (Must)
-//! 2 MATCH_ANY — tokenize query, BooleanQuery (Should)
-//! 3 PHRASE — tokenize query, PhraseQuery
-//! 4 PREFIX — original + case-normalized RegexQuery `<escaped>.*` (no tokenization)
-//! 5 WILDCARD — original + case-normalized RegexQuery from glob pattern
-//! (`*` → `.*`, `?` → `.`, others escaped)
-//!
-//! For paimon-java compatibility, row_id is stored as an explicit u64 field
-//! (`fast` for O(1) retrieval). Reader translates tantivy DocAddress → row_id
-//! via `fast_fields().u64("row_id").first(doc_id)` per segment.
-//!
-//! FFI return format (little-endian, **doc identifiers are u64 row_ids**):
-//! `[u8 has_scores | u64 count | u64 row_id[count] | optional f32 score[count]]`
-
-use std::ffi::{c_char, CStr};
-use std::path::Path;
-
-use croaring::{Portable, Treemap};
-use tantivy::collector::{Collector, SegmentCollector};
-use tantivy::columnar::Column;
-use tantivy::query::{
- BooleanQuery, DisjunctionMaxQuery, Occur, PhraseQuery, Query, RegexQuery, TermQuery,
-};
-use tantivy::schema::{Field, IndexRecordOption};
-use tantivy::{DocAddress, DocId, Index, IndexReader, ReloadPolicy, Score, SegmentOrdinal,
- SegmentReader, Term};
-
-use crate::buffer::PaimonTantivyBuffer;
-use crate::callback_directory::{PaimonCallbackDirectory, PaimonStreamCallbacks};
-use crate::error::{set_last_error, PaimonTantivyStatus};
-use crate::handle::{borrow_handle_mut, free_handle, into_handle};
-use crate::tokenizer::{normalize_case, PaimonJiebaTokenizer, TokenizeMode};
-use crate::writer::{PAIMON_ROW_ID_FIELD_NAME, PAIMON_TEXT_FIELD_NAME, PAIMON_TOKENIZER_NAME};
-
-/// Numeric encoding of `paimon::FullTextSearch::SearchType`. Kept in sync
-/// with include/paimon/predicate/full_text_search.h.
-#[repr(i32)]
-#[derive(Clone, Copy, Debug)]
-pub enum SearchType {
- MatchAll = 1,
- MatchAny = 2,
- Phrase = 3,
- Prefix = 4,
- Wildcard = 5,
-}
-
-impl SearchType {
- fn from_i32(v: i32) -> Option<Self> {
- match v {
- 1 => Some(Self::MatchAll),
- 2 => Some(Self::MatchAny),
- 3 => Some(Self::Phrase),
- 4 => Some(Self::Prefix),
- 5 => Some(Self::Wildcard),
- _ => None,
- }
- }
-}
-
-pub struct PaimonTantivyReader {
- /// Held alive so `IndexReader::searcher()` + `index.tokenizers()` stay
- /// usable for the reader's lifetime.
- index: Index,
- reader: IndexReader,
- text_field: Field,
- /// Name of the tokenizer the `text` field is actually bound to in the open
- /// index's schema (read from `meta.json` at construction time). Query-side
- /// tokenization looks this up in `index.tokenizers()` every time
- tokenizer_name: String,
-}
-
-impl PaimonTantivyReader {
- /// Construct a reader from a pre-built callback-backed Directory.
- /// Layout (file names + offsets + lengths) must come from the caller
- /// (C++ side `ArchiveLayout::Parse`); Rust does not re-parse the archive.
- pub fn new(
- directory: PaimonCallbackDirectory,
- mode: TokenizeMode,
- with_position: bool,
- dict_dir: &Path,
- ) -> Result<Self, String> {
- let index = Index::open(directory)
- .map_err(|e| format!("tantivy::Index::open: {e}"))?;
-
- // Resolve fields by their fixed names (schema is `row_id` + `text`).
- let schema = index.schema();
- let text_field = schema.get_field(PAIMON_TEXT_FIELD_NAME).map_err(|e| {
- format!("tantivy index missing '{PAIMON_TEXT_FIELD_NAME}' field: {e}")
- })?;
-
- // Read the tokenizer name the `text` field was actually written with
- // (lives in meta.json's schema). Auto-aligns cpp query-side tokenizer
- // with whatever the writer side used.
- let tokenizer_name = match schema.get_field_entry(text_field).field_type() {
- tantivy::schema::FieldType::Str(text_options) => text_options
- .get_indexing_options()
- .map(|io| io.tokenizer().to_string())
- .unwrap_or_else(|| "default".to_string()),
- other => {
- return Err(format!(
- "text field has non-TEXT type: {other:?} (schema corrupted?)"
- ));
- }
- };
-
- // Only register paimon_jieba if the index actually uses it. The
- // tantivy-builtin "default" / "raw" / "en_stem" etc. are pre-registered
- // by the TokenizerManager — no setup needed for those.
- if tokenizer_name == PAIMON_TOKENIZER_NAME {
- // `Path::is_empty` is unstable; check via OsStr.
- if dict_dir.as_os_str().is_empty() {
- return Err(format!(
- "paimon_jieba tokenizer required by archive schema but dict dir \
- is empty — set the PAIMON_JIEBA_DICT_DIR env var to a directory \
- containing jieba.dict.utf8 / hmm_model.utf8 / user.dict.utf8 / \
- idf.utf8 / stop_words.utf8"
- ));
- }
- let jieba = PaimonJiebaTokenizer::new(dict_dir, mode, with_position)
- .map_err(|e| format!("create paimon_jieba tokenizer: {e}"))?;
- index.tokenizers().register(PAIMON_TOKENIZER_NAME, jieba);
- } else {
- // For other known-safe names we trust tantivy's builtin registry.
- // `mode` / `dict_dir` are unused in this branch — no-op; we still
- // require them in the ABI for backward-compat with the jieba case.
- let _ = (mode, dict_dir);
- }
-
- // Sanity: the tokenizer MUST be resolvable now; otherwise query-time
- // lookup fails mid-flight.
- if index.tokenizers().get(&tokenizer_name).is_none() {
- return Err(format!(
- "tokenizer {tokenizer_name:?} referenced by text field is not \
- registered; add it to TokenizerManager before opening the reader"
- ));
- }
-
- let reader = index
- .reader_builder()
- .reload_policy(ReloadPolicy::Manual)
- .try_into()
- .map_err(|e| format!("build IndexReader: {e}"))?;
-
- Ok(Self {
- index,
- reader,
- text_field,
- tokenizer_name,
- })
- }
-
- /// Tokenize the query string using the *same* tokenizer the index's text
- /// field was built with. Looks up `self.tokenizer_name` in the index's
- /// `TokenizerManager` — which was populated by `new()` with either
- /// `paimon_jieba` (if cpp wrote the index) or a tantivy builtin like
- /// `default` (if paimon-java wrote it).
- fn tokenize_query(&self, query: &str) -> Vec<String> {
- // `TokenizerManager::get` returns a fresh clone per call — safe to use
- // across threads / calls. If the tokenizer was missing we'd have
- // failed in `new()`; we still defend with `unwrap_or_default`.
- let mut analyzer = match self.index.tokenizers().get(&self.tokenizer_name) {
- Some(a) => a,
- None => return Vec::new(),
- };
- let mut stream = analyzer.token_stream(query);
- let mut out = Vec::new();
- while stream.advance() {
- out.push(stream.token().text.clone());
- }
- out
- }
-
- fn build_match_query(&self, query: &str, occur: Occur) -> Result<Box<dyn Query>, String> {
- let terms = self.tokenize_query(query);
- if terms.is_empty() {
- return Err(format!("query {query:?} produced no tokens after analysis"));
- }
- if terms.len() == 1 {
- let term = Term::from_field_text(self.text_field, &terms[0]);
- return Ok(Box::new(TermQuery::new(term, IndexRecordOption::WithFreqs)));
- }
- let clauses: Vec<(Occur, Box<dyn Query>)> = terms
- .iter()
- .map(|t| {
- let term = Term::from_field_text(self.text_field, t);
- let q: Box<dyn Query> =
- Box::new(TermQuery::new(term, IndexRecordOption::WithFreqs));
- (occur, q)
- })
- .collect();
- Ok(Box::new(BooleanQuery::new(clauses)))
- }
-
- fn build_phrase_query(&self, query: &str) -> Result<Box<dyn Query>, String> {
- let terms = self.tokenize_query(query);
- if terms.is_empty() {
- return Err(format!("phrase query {query:?} produced no tokens"));
- }
- if terms.len() == 1 {
- // PhraseQuery requires >=2 terms in tantivy; degrade to TermQuery.
- let term = Term::from_field_text(self.text_field, &terms[0]);
- return Ok(Box::new(TermQuery::new(term, IndexRecordOption::WithFreqsAndPositions)));
- }
- let tantivy_terms: Vec<Term> = terms
- .iter()
- .map(|t| Term::from_field_text(self.text_field, t))
- .collect();
- Ok(Box::new(PhraseQuery::new(tantivy_terms)))
- }
-
- fn build_prefix_query(&self, query: &str) -> Result<Box<dyn Query>, String> {
- if query.is_empty() {
- return Err("prefix query is empty".into());
- }
- // Mirror lucene-fts: don't tokenize the prefix. Retain the original form for mixed
- // ASCII/CJK indexed terms, and add the normalized form for pure ASCII terms.
- let normalized_query = normalize_case(query);
- let create_query = |value: &str| -> Result<Box<dyn Query>, String> {
- let pattern = format!("{}.*", regex_escape(value));
- RegexQuery::from_pattern(&pattern, self.text_field)
- .map(|q| Box::new(q) as Box<dyn Query>)
- .map_err(|e| format!("RegexQuery from prefix {value:?}: {e}"))
- };
- let original = create_query(query)?;
- if normalized_query == query {
- return Ok(original);
- }
- let normalized = create_query(&normalized_query)?;
- Ok(Box::new(DisjunctionMaxQuery::new(vec![
- original, normalized,
- ])))
- }
-
- fn build_wildcard_query(&self, query: &str) -> Result<Box<dyn Query>, String> {
- if query.is_empty() {
- return Err("wildcard query is empty".into());
- }
- let normalized_query = normalize_wildcard_query(query);
- let create_query = |value: &str| -> Result<Box<dyn Query>, String> {
- let pattern = wildcard_to_regex(value);
- RegexQuery::from_pattern(&pattern, self.text_field)
- .map(|q| Box::new(q) as Box<dyn Query>)
- .map_err(|e| format!("RegexQuery from wildcard {value:?} (pattern {pattern}): {e}"))
- };
- let original = create_query(query)?;
- if normalized_query == query {
- return Ok(original);
- }
- let normalized = create_query(&normalized_query)?;
- Ok(Box::new(DisjunctionMaxQuery::new(vec![
- original, normalized,
- ])))
- }
-
- fn build_query(&self, search_type: SearchType, query: &str) -> Result<Box<dyn Query>, String> {
- match search_type {
- SearchType::MatchAll => self.build_match_query(query, Occur::Must),
- SearchType::MatchAny => self.build_match_query(query, Occur::Should),
- SearchType::Phrase => self.build_phrase_query(query),
- SearchType::Prefix => self.build_prefix_query(query),
- SearchType::Wildcard => self.build_wildcard_query(query),
- }
- }
-
- /// Return all matching row_ids (no scoring, no limit, no pre_filter).
- /// row_ids come from the explicit `row_id` u64 fast field, supporting
- /// multi-segment indexes (e.g. produced by paimon-java without force-merge).
- pub fn search_all(&self, search_type: SearchType, query: &str) -> Result<Vec<u64>, String> {
- let q = self.build_query(search_type, query)?;
- let searcher = self.reader.searcher();
- let mut ids: Vec<u64> = searcher
- .search(&*q, &RowIdCollector)
- .map_err(|e| format!("tantivy search: {e}"))?;
- ids.sort_unstable();
- ids.dedup();
- Ok(ids)
- }
-
- /// Four-path dispatch on `(with_score, limit)`.
- ///
- /// | with_score | limit | path | collector | sort | truncate | output score |
- /// |------------|--------|------|------------------------|----------------|----------|--------------|
- /// | false | None | A | RowIdCollector | row_id asc | — | ❌ |
- /// | false | Some(n)| B | AllScoredCollector | score desc | top n | ❌ (dropped) |
- /// | true | None | C | AllScoredCollector | row_id asc | — | ✅ |
- /// | true | Some(n)| D | AllScoredCollector | score desc | top n | ✅ |
- ///
- /// Pre-filter is a `Treemap` of paimon row_ids (not tantivy doc_ids), applied BEFORE
- /// truncation so high-score matches outside the filter don't crowd out valid ones.
- ///
- /// **v0.2 contract change**: previously `limit.is_some()` implicitly triggered scoring; now
- /// scoring is gated solely by `with_score`.
- pub fn search_with_limit_and_filter(
- &self,
- search_type: SearchType,
- query: &str,
- with_score: bool,
- limit: Option<usize>,
- pre_filter: Option<&Treemap>,
- min_score: Option<f32>,
- ) -> Result<Vec<(u64, Option<f32>)>, String> {
- let q = self.build_query(search_type, query)?;
- let searcher = self.reader.searcher();
- match (with_score, limit) {
- // Path A: all rows, no score. RowIdCollector reads the `row_id` fast
- // field inline per segment (opened once), avoiding a DocSetCollector
- // HashSet and per-doc handle — hot path for high-cardinality counts.
- (false, None) => {
- let mut row_ids: Vec<u64> = searcher
- .search(&*q, &RowIdCollector)
- .map_err(|e| format!("tantivy search: {e}"))?;
- if let Some(filter) = pre_filter {
- row_ids.retain(|id| filter.contains(*id));
- }
- row_ids.sort_unstable();
- row_ids.dedup();
- Ok(row_ids.into_iter().map(|id| (id, None)).collect())
- }
- // Path B: any N matches, unscored. Used by SR's `WHERE MATCH ... LIMIT N` (no
- // ORDER BY): pushes the limit down so each shard stops collecting once N hits
- // are gathered per segment instead of materialising the full posting list.
- // If the caller wants top-N by BM25 they should set `with_score=true` (Path D)
- // and ignore the score values.
- (false, Some(n)) => {
- if n == 0 {
- return Ok(Vec::new());
- }
- if min_score.is_some() {
- // min_score requires scoring — fall back to collect_scored path
- let mut filtered = self.collect_scored(&*q, &searcher, pre_filter)?;
- if let Some(threshold) = min_score {
- filtered.retain(|(s, _)| *s > threshold);
- }
- let truncated = Self::sort_by_score_desc_truncate(filtered, n);
- Ok(truncated.into_iter().map(|(_, id)| (id, None)).collect())
- } else if let Some(filter) = pre_filter {
- // pre_filter present: it MUST be applied to the full match set
- // before truncation. LimitedDocSetCollector stops after the
- // first N raw matches, which could all be filtered out while
- // valid matches exist further down the posting list — that
- // would under-return (fewer than N, or even empty). So collect
- // every matching row_id (filter-aware), then truncate to N.
- let mut row_ids: Vec<u64> = searcher
- .search(&*q, &RowIdCollector)
- .map_err(|e| format!("tantivy search: {e}"))?;
- row_ids.retain(|id| filter.contains(*id));
- row_ids.sort_unstable();
- row_ids.dedup();
- row_ids.truncate(n);
- Ok(row_ids.into_iter().map(|id| (id, None)).collect())
- } else {
- // No pre_filter: fast path — stop collecting once N matches are
- // gathered per segment instead of materialising the full posting list.
- let collector = LimitedDocSetCollector::new(n);
- let mut docset = searcher
- .search(&*q, &collector)
- .map_err(|e| format!("tantivy search: {e}"))?;
- let mut by_segment: std::collections::HashMap<SegmentOrdinal, Vec<DocId>> =
- std::collections::HashMap::new();
- for addr in docset.drain(..) {
- by_segment.entry(addr.segment_ord).or_default().push(addr.doc_id);
- }
- let mut row_ids: Vec<u64> = Vec::new();
- for (segment_ord, doc_ids) in by_segment.iter() {
- let segment_reader = searcher.segment_reader(*segment_ord);
- let fast = segment_reader
- .fast_fields()
- .u64(PAIMON_ROW_ID_FIELD_NAME)
- .map_err(|e| format!("fast_fields().u64('row_id') on segment {}: {e}",
- segment_ord))?;
- for &doc_id in doc_ids {
- row_ids.push(fast.first(doc_id).unwrap_or(0));
- }
- }
- row_ids.sort_unstable();
- row_ids.dedup();
- row_ids.truncate(n);
- Ok(row_ids.into_iter().map(|id| (id, None)).collect())
- }
- }
- // Path C: all rows + all scores, sorted by row_id asc to match the
- // BitmapScoredGlobalIndexResult contract (bitmap iter order == score order).
- (true, None) => {
- let mut filtered = self.collect_scored(&*q, &searcher, pre_filter)?;
- if let Some(threshold) = min_score {
- filtered.retain(|(s, _)| *s > threshold);
- }
- filtered.sort_unstable_by(|a, b| a.1.cmp(&b.1));
- Ok(filtered.into_iter().map(|(s, id)| (id, Some(s))).collect())
- }
- // Path D: top-N by BM25 with scores.
- (true, Some(n)) => {
- if n == 0 {
- return Ok(Vec::new());
- }
- let mut filtered = self.collect_scored(&*q, &searcher, pre_filter)?;
- if let Some(threshold) = min_score {
- filtered.retain(|(s, _)| *s > threshold);
- }
- let truncated = Self::sort_by_score_desc_truncate(filtered, n);
- Ok(truncated.into_iter().map(|(s, id)| (id, Some(s))).collect())
- }
- }
- }
-
- /// Helper for paths B/C/D: run AllScoredCollector, translate doc_id → row_id, apply pre_filter.
- /// Groups results by segment so the fast field column handle is opened once per segment
- /// (same rationale as Path A — avoids per-match Column<u64> allocation).
- fn collect_scored(
- &self,
- q: &dyn Query,
- searcher: &tantivy::Searcher,
- pre_filter: Option<&Treemap>,
- ) -> Result<Vec<(Score, u64)>, String> {
- let scored = searcher
- .search(q, &AllScoredCollector)
- .map_err(|e| format!("tantivy search: {e}"))?;
- let mut by_segment: std::collections::HashMap<SegmentOrdinal, Vec<(Score, DocId)>> =
- std::collections::HashMap::new();
- for (s, addr) in scored.into_iter() {
- by_segment.entry(addr.segment_ord).or_default().push((s, addr.doc_id));
- }
- let mut result: Vec<(Score, u64)> = Vec::new();
- for (segment_ord, entries) in by_segment.iter() {
- let segment_reader = searcher.segment_reader(*segment_ord);
- let fast = segment_reader
- .fast_fields()
- .u64(PAIMON_ROW_ID_FIELD_NAME)
- .map_err(|e| format!("fast_fields().u64('row_id') on segment {}: {e}",
- segment_ord))?;
- for &(score, doc_id) in entries {
- let rid = fast.first(doc_id).unwrap_or(0);
- if pre_filter.map_or(true, |t| t.contains(rid)) {
- result.push((score, rid));
- }
- }
- }
- Ok(result)
- }
-
- /// Helper for paths B/D: sort (score, row_id) by score desc with row_id asc tie-break,
- /// then truncate to `n` items.
- fn sort_by_score_desc_truncate(mut v: Vec<(Score, u64)>, n: usize) -> Vec<(Score, u64)> {
- v.sort_unstable_by(|a, b| {
- b.0.partial_cmp(&a.0)
- .unwrap_or(std::cmp::Ordering::Equal)
- .then(a.1.cmp(&b.1))
- });
- v.truncate(n);
- v
- }
-
- #[cfg(test)]
- pub(crate) fn tokenizer_name(&self) -> &str {
- &self.tokenizer_name
- }
-
- #[cfg(test)]
- pub(crate) fn debug_index(&self) -> &Index {
- &self.index
- }
-}
-
-/// Escape regex metacharacters, but leave the input as a verbatim literal.
-fn regex_escape(input: &str) -> String {
- let mut out = String::with_capacity(input.len() + 4);
- for ch in input.chars() {
- match ch {
- '.' | '+' | '*' | '?' | '(' | ')' | '[' | ']' | '{' | '}' | '|' | '^' | '$' | '\\' => {
- out.push('\\');
- out.push(ch);
- }
- _ => out.push(ch),
- }
- }
- out
-}
-
-/// Normalize each literal segment independently while preserving wildcard
-/// operators. This matches lucene-fts, where `*` and `?` are not analyzed.
-fn normalize_wildcard_query(input: &str) -> String {
- let mut out = String::with_capacity(input.len());
- let mut segment_start = 0;
- for (index, ch) in input.char_indices() {
- if ch != '*' && ch != '?' {
- continue;
- }
- out.push_str(&normalize_case(&input[segment_start..index]));
- out.push(ch);
- segment_start = index + ch.len_utf8();
- }
- out.push_str(&normalize_case(&input[segment_start..]));
- out
-}
-
-/// Translate a glob-style wildcard ('*' = any, '?' = single char) into a
-/// regex pattern, escaping all other regex metacharacters.
-fn wildcard_to_regex(input: &str) -> String {
- let mut out = String::with_capacity(input.len() + 4);
- for ch in input.chars() {
- match ch {
- '*' => out.push_str(".*"),
- '?' => out.push('.'),
- '.' | '+' | '(' | ')' | '[' | ']' | '{' | '}' | '|' | '^' | '$' | '\\' => {
- out.push('\\');
- out.push(ch);
- }
- _ => out.push(ch),
- }
- }
- out
-}
-
-/// Collector that reads the explicit `row_id` u64 fast field directly into a
-/// `Vec<u64>`, opening the column once per segment in `for_segment`. Replaces
-/// the DocSetCollector → HashSet → per-doc translate path for unscored queries.
-struct RowIdCollector;
-
-struct RowIdSegmentCollector {
- row_id: Column<u64>,
- ids: Vec<u64>,
-}
-
-impl SegmentCollector for RowIdSegmentCollector {
- type Fruit = Vec<u64>;
-
- fn collect(&mut self, doc: DocId, _score: Score) {
- self.ids.push(self.row_id.first(doc).unwrap_or(0));
- }
-
- fn harvest(self) -> Vec<u64> {
- self.ids
- }
-}
-
-impl Collector for RowIdCollector {
- type Fruit = Vec<u64>;
- type Child = RowIdSegmentCollector;
-
- fn for_segment(
- &self, _ord: SegmentOrdinal, segment: &SegmentReader,
- ) -> tantivy::Result<RowIdSegmentCollector> {
- let row_id = segment.fast_fields().u64(PAIMON_ROW_ID_FIELD_NAME)?;
- Ok(RowIdSegmentCollector { row_id, ids: Vec::new() })
- }
-
- fn requires_scoring(&self) -> bool {
- false
- }
-
- fn merge_fruits(&self, segs: Vec<Vec<u64>>) -> tantivy::Result<Vec<u64>> {
- Ok(segs.into_iter().flatten().collect())
- }
-}
-
-/// Collector that returns at most `limit` DocAddresses across all segments,
-/// no scoring. Shared atomic counter caps the global total so per-shard
-/// transfer stays bounded for plain `LIMIT N` queries (no ORDER BY).
-struct LimitedDocSetCollector {
- limit: usize,
- counter: std::sync::Arc<std::sync::atomic::AtomicU64>,
-}
-
-impl LimitedDocSetCollector {
- fn new(limit: usize) -> Self {
- Self { limit, counter: std::sync::Arc::new(std::sync::atomic::AtomicU64::new(0)) }
- }
-}
-
-struct LimitedDocSetSegmentCollector {
- segment_ord: SegmentOrdinal,
- docs: Vec<DocId>,
- counter: std::sync::Arc<std::sync::atomic::AtomicU64>,
- limit: u64,
-}
-
-impl SegmentCollector for LimitedDocSetSegmentCollector {
- type Fruit = Vec<DocAddress>;
-
- fn collect(&mut self, doc: DocId, _score: Score) {
- // Best-effort cap: if multiple segments are scanned concurrently the
- // atomic ensures we never accept more than `limit` rows total.
- let prev = self.counter.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
- if prev < self.limit {
- self.docs.push(doc);
- }
- }
-
- fn harvest(self) -> Self::Fruit {
- let segment_ord = self.segment_ord;
- self.docs.into_iter().map(|d| DocAddress::new(segment_ord, d)).collect()
- }
-}
-
-impl Collector for LimitedDocSetCollector {
- type Fruit = Vec<DocAddress>;
- type Child = LimitedDocSetSegmentCollector;
-
- fn for_segment(
- &self, segment_ord: SegmentOrdinal, _segment: &SegmentReader,
- ) -> tantivy::Result<Self::Child> {
- Ok(LimitedDocSetSegmentCollector {
- segment_ord,
- docs: Vec::new(),
- counter: self.counter.clone(),
- limit: self.limit as u64,
- })
- }
-
- fn requires_scoring(&self) -> bool { false }
-
- fn merge_fruits(
- &self, segment_fruits: Vec<Vec<DocAddress>>,
- ) -> tantivy::Result<Vec<DocAddress>> {
- let mut result: Vec<DocAddress> = segment_fruits.into_iter().flatten().collect();
- result.truncate(self.limit);
- Ok(result)
- }
-}
-
-/// Custom Collector that returns ALL matching (score, DocAddress) tuples,
-/// without truncation. tantivy's stock `TopDocs::with_limit(N)` would force
-/// us to either pick N upfront (wrong when pre_filter rejects high-score
-/// docs) or pass `usize::MAX` (which still enforces a binary heap on every
-/// push). Our collector is just a plain Vec append, then merge.
-struct AllScoredCollector;
-
-struct AllScoredSegmentCollector {
- segment_ord: SegmentOrdinal,
- docs: Vec<(Score, DocId)>,
-}
-
-impl SegmentCollector for AllScoredSegmentCollector {
- type Fruit = Vec<(Score, DocAddress)>;
-
- fn collect(&mut self, doc: DocId, score: Score) {
- self.docs.push((score, doc));
- }
-
- fn harvest(self) -> Self::Fruit {
- let segment_ord = self.segment_ord;
- self.docs
- .into_iter()
- .map(|(s, d)| (s, DocAddress::new(segment_ord, d)))
- .collect()
- }
-}
-
-impl Collector for AllScoredCollector {
- type Fruit = Vec<(Score, DocAddress)>;
- type Child = AllScoredSegmentCollector;
-
- fn for_segment(
- &self,
- segment_ord: SegmentOrdinal,
- _segment: &SegmentReader,
- ) -> tantivy::Result<Self::Child> {
- Ok(AllScoredSegmentCollector {
- segment_ord,
- docs: Vec::new(),
- })
- }
-
- fn requires_scoring(&self) -> bool {
- true
- }
-
- fn merge_fruits(
- &self,
- segment_fruits: Vec<Vec<(Score, DocAddress)>>,
- ) -> tantivy::Result<Vec<(Score, DocAddress)>> {
- Ok(segment_fruits.into_iter().flatten().collect())
- }
-}
-
-// ============================ FFI surface ============================
-
-/// Construct a streaming reader from a layout table + pread callbacks.
-///
-/// The layout arrays (names / offsets / lengths) are produced by C++-side
-/// `ArchiveLayout::Parse` after reading only the archive header bytes. Payload
-/// bytes are fetched lazily through `callbacks.read_at` as tantivy reads.
-///
-/// # Arguments
-/// * `file_names` — array of `file_count` UTF-8 NUL-terminated C strings
-/// * `file_offsets` / `file_lengths` — u64 arrays (archive-absolute offsets and lengths)
-/// * `file_count` — number of entries in each of the three arrays
-/// * `callbacks` — pread + release callbacks; `ctx` ownership transfers to Rust
-/// * `mode_cstr` — tokenize mode ("mp"/"mix"/"full"/"query"; "hmm" → Unsupported)
-/// * `with_position` — whether text field was indexed with positions
-/// * `dict_dir_cstr` — paimon_jieba dictionary directory
-/// * `out` — receives the reader handle on success
-///
-/// Releases an FFI stream ctx on drop unless disarmed. Gives
-/// `paimon_tantivy_reader_new_streaming` a single ownership rule: Rust owns ctx
-/// from entry and releases it on any error before the directory takes over.
-struct StreamCtxReleaseGuard {
- release: extern "C" fn(*mut std::os::raw::c_void),
- ctx: *mut std::os::raw::c_void,
- armed: bool,
-}
-
-impl Drop for StreamCtxReleaseGuard {
- fn drop(&mut self) {
- if self.armed {
- (self.release)(self.ctx);
- }
- }
-}
-
-/// # Safety
-/// All pointer args must be valid for the duration of the call. Ownership of
-/// `callbacks.ctx` transfers to Rust on entry: on any error this function
-/// releases it, and on success it lives until the reader handle is freed. The
-/// caller must NOT release ctx itself after calling this function.
-#[no_mangle]
-pub unsafe extern "C" fn paimon_tantivy_reader_new_streaming(
- file_names: *const *const c_char,
- file_offsets: *const u64,
- file_lengths: *const u64,
- file_count: usize,
- callbacks: PaimonStreamCallbacks,
- mode_cstr: *const c_char,
- with_position: bool,
- dict_dir_cstr: *const c_char,
- out: *mut *mut PaimonTantivyReader,
-) -> PaimonTantivyStatus {
- // Single, uniform ownership rule: Rust owns ctx from entry. This guard
- // releases it on every error path until ownership moves into the directory
- // (which then releases on its own drop). Prevents the C++ caller and Rust
- // from both releasing the same ctx on the post-directory failure path.
- let mut ctx_guard = StreamCtxReleaseGuard {
- release: callbacks.release,
- ctx: callbacks.ctx,
- armed: true,
- };
-
- if mode_cstr.is_null() || dict_dir_cstr.is_null() || out.is_null() {
- set_last_error("paimon_tantivy_reader_new_streaming: null mandatory argument");
- return PaimonTantivyStatus::InvalidArgument;
- }
- if file_count > 0
- && (file_names.is_null() || file_offsets.is_null() || file_lengths.is_null())
- {
- set_last_error("file_names/offsets/lengths must be non-null when file_count > 0");
- return PaimonTantivyStatus::InvalidArgument;
- }
-
- let mode_str = match unsafe { CStr::from_ptr(mode_cstr) }.to_str() {
- Ok(s) => s,
- Err(e) => {
- set_last_error(format!("mode not utf-8: {e}"));
- return PaimonTantivyStatus::InvalidArgument;
- }
- };
- let dict_dir = match unsafe { CStr::from_ptr(dict_dir_cstr) }.to_str() {
- Ok(s) => s,
- Err(e) => {
- set_last_error(format!("dict_dir not utf-8: {e}"));
- return PaimonTantivyStatus::InvalidArgument;
- }
- };
- let mode = match TokenizeMode::parse(mode_str) {
- Some(m) => m,
- None => {
- set_last_error(format!(
- "unknown tokenize mode {mode_str:?}; expected mp/mix/full/query"
- ));
- return PaimonTantivyStatus::InvalidArgument;
- }
- };
-
- // Copy the C string array into owned Rust entries so the directory doesn't
- // depend on caller-supplied lifetime.
- let mut entries: Vec<(String, u64, u64)> = Vec::with_capacity(file_count);
- for i in 0..file_count {
- let name_ptr = unsafe { *file_names.add(i) };
- if name_ptr.is_null() {
- set_last_error(format!("file_names[{i}] is null"));
- return PaimonTantivyStatus::InvalidArgument;
- }
- let name = match unsafe { CStr::from_ptr(name_ptr) }.to_str() {
- Ok(s) => s.to_owned(),
- Err(e) => {
- set_last_error(format!("file_names[{i}] not utf-8: {e}"));
- return PaimonTantivyStatus::InvalidArgument;
- }
- };
- let offset = unsafe { *file_offsets.add(i) };
- let length = unsafe { *file_lengths.add(i) };
- entries.push((name, offset, length));
- }
-
- // Ownership of ctx transfers to the directory from here on (it releases on
- // its own drop, whether it fails below or lives inside the returned reader).
- ctx_guard.armed = false;
- let directory = PaimonCallbackDirectory::new(entries, callbacks);
-
- match PaimonTantivyReader::new(directory, mode, with_position, Path::new(dict_dir)) {
- Ok(r) => {
- unsafe { *out = into_handle(r) };
- PaimonTantivyStatus::Ok
- }
- Err(e) => {
- let unsupported = e.contains("'hmm' is not supported");
- let bad_format = e.contains("tantivy::Index::open")
- || e.contains("missing 'text' field");
- set_last_error(e);
- if unsupported {
- PaimonTantivyStatus::Unsupported
- } else if bad_format {
- PaimonTantivyStatus::IndexFormatError
- } else {
- PaimonTantivyStatus::InternalError
- }
- }
- }
-}
-
-/// Run a query and emit results into `out`.
-///
-/// Output bytes (little-endian):
-/// `[u8 has_scores | u64 count | u64 row_ids[count] | optional f32 scores[count]]`
-///
-/// `has_scores=1` iff `limit >= 0` (caller asked for scoring + limit).
-///
-/// `limit < 0` ⇒ no limit, no scoring; sorted ascending by row_id.
-/// `limit >= 0` ⇒ top-N by descending score (pre_filter applied first).
-/// `pre_filter_bytes`: serialized croaring `Roaring64Map::write` (portable),
-/// containing paimon **row_ids** (not tantivy doc_ids); null+0 = no filter.
-///
-/// SAFETY: `reader` must be a live handle; `query` and `pre_filter_bytes`
-/// may be null+0 or readable slices; `out` non-null.
-#[no_mangle]
-pub unsafe extern "C" fn paimon_tantivy_reader_search(
- reader: *mut PaimonTantivyReader,
- search_type: i32,
- query: *const c_char,
- query_len: usize,
- with_score: bool,
- limit: i32,
- pre_filter_bytes: *const c_char,
- pre_filter_len: usize,
- min_score: f32,
- out: *mut PaimonTantivyBuffer,
-) -> PaimonTantivyStatus {
- if out.is_null() {
- set_last_error("reader_search: out is null");
- return PaimonTantivyStatus::InvalidArgument;
- }
- let Some(r) = (unsafe { borrow_handle_mut::<PaimonTantivyReader>(reader) }) else {
- set_last_error("reader_search: null reader handle");
- return PaimonTantivyStatus::InvalidArgument;
- };
- let st = match SearchType::from_i32(search_type) {
- Some(s) => s,
- None => {
- set_last_error(format!("unknown search_type {search_type}"));
- return PaimonTantivyStatus::InvalidArgument;
- }
- };
- if query.is_null() && query_len != 0 {
- set_last_error("query is null but len > 0");
- return PaimonTantivyStatus::InvalidArgument;
- }
- let query_str = if query_len == 0 {
- ""
- } else {
- let slice = unsafe { std::slice::from_raw_parts(query as *const u8, query_len) };
- match std::str::from_utf8(slice) {
- Ok(s) => s,
- Err(e) => {
- set_last_error(format!("query not utf-8: {e}"));
- return PaimonTantivyStatus::InvalidArgument;
- }
- }
- };
-
- let pre_filter: Option<Treemap> = if pre_filter_bytes.is_null() && pre_filter_len == 0 {
- None
- } else if pre_filter_bytes.is_null() {
- set_last_error("pre_filter_bytes is null but len > 0");
- return PaimonTantivyStatus::InvalidArgument;
- } else {
- let slice = unsafe {
- std::slice::from_raw_parts(pre_filter_bytes as *const u8, pre_filter_len)
- };
- match Treemap::try_deserialize::<Portable>(slice) {
- Some(t) => Some(t),
- None => {
- set_last_error(format!(
- "pre_filter not a valid Roaring64Map portable serialization ({} bytes)",
- pre_filter_len
- ));
- return PaimonTantivyStatus::InvalidArgument;
- }
- }
- };
-
- let limit_opt: Option<usize> = if limit < 0 { None } else { Some(limit as usize) };
- let min_score_opt: Option<f32> = if min_score > 0.0 { Some(min_score) } else { None };
-
- match r.search_with_limit_and_filter(st, query_str, with_score, limit_opt, pre_filter.as_ref(), min_score_opt)
- {
- Ok(rows) => {
- // v0.2: has_scores is decoupled from limit — it equals with_score directly.
- let has_scores = with_score;
- let count = rows.len() as u64;
- // 1 byte has_scores + 8 bytes count + 8 bytes per row_id + optional 4 bytes per score
- let mut buf = Vec::with_capacity(
- 1 + 8 + rows.len() * 8 + if has_scores { rows.len() * 4 } else { 0 },
- );
- buf.push(if has_scores { 1u8 } else { 0u8 });
- buf.extend_from_slice(&count.to_le_bytes());
- for (id, _) in &rows {
- buf.extend_from_slice(&id.to_le_bytes()); // u64 row_id LE
- }
- if has_scores {
- for (_, score) in &rows {
- let s = score.unwrap_or(0.0);
- buf.extend_from_slice(&s.to_le_bytes());
- }
- }
- unsafe { *out = PaimonTantivyBuffer::from_vec(buf) };
- PaimonTantivyStatus::Ok
- }
- Err(e) => {
- let parse_err = e.contains("RegexQuery from")
- || e.contains("phrase query")
- || e.contains("produced no tokens");
- set_last_error(e);
- if parse_err {
- PaimonTantivyStatus::QueryParseError
- } else {
- PaimonTantivyStatus::InternalError
- }
- }
- }
-}
-
-/// Destroy a reader handle. Safe on null.
-#[no_mangle]
-pub unsafe extern "C" fn paimon_tantivy_reader_free(reader: *mut PaimonTantivyReader) {
- unsafe { free_handle(reader) };
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
- use crate::callback_directory::test_support::{build_directory_from_archive, make_mock_callbacks};
- use crate::writer::PaimonTantivyWriter;
- use std::path::PathBuf;
-
- // When PaimonTantivyReader::new fails *after* the callback directory (and
- // thus ctx) is constructed, Rust must release ctx exactly once — never zero
- // (leak) and never twice (the C++ caller no longer releases on failure, so a
- // second release would be a double-free). file_count=0 yields an index with
- // no meta.json, which fails to open after the directory is built.
- #[test]
- fn reader_new_streaming_releases_ctx_once_on_failure() {
- let (cb, backend) = make_mock_callbacks(Vec::new());
- let mode = std::ffi::CString::new("mix").unwrap();
- let dict = std::ffi::CString::new("").unwrap();
- let mut out: *mut PaimonTantivyReader = std::ptr::null_mut();
- let st = unsafe {
- paimon_tantivy_reader_new_streaming(
- std::ptr::null(),
- std::ptr::null(),
- std::ptr::null(),
- 0,
- cb,
- mode.as_ptr(),
- true,
- dict.as_ptr(),
- &mut out,
- )
- };
- assert!(!matches!(st, PaimonTantivyStatus::Ok));
- assert!(out.is_null());
- assert_eq!(
- backend.release_count.load(std::sync::atomic::Ordering::SeqCst),
- 1
- );
- }
-
- fn dict_dir() -> PathBuf {
- std::env::var("PAIMON_JIEBA_DICT_DIR")
- .map(PathBuf::from)
- .unwrap_or_else(|_| PathBuf::from("/tmp/nonexistent-dict"))
- }
-
- fn build(docs: &[&str]) -> Vec<u8> {
- let mut w = PaimonTantivyWriter::new("f0", TokenizeMode::Mix, true, &dict_dir(), "paimon_jieba").unwrap();
- for (i, d) in docs.iter().enumerate() {
- w.add(i as u64, d).unwrap();
- }
- w.finish().unwrap().1
- }
-
- fn open(packed: &[u8]) -> PaimonTantivyReader {
- // Simulate production flow: parse archive header → build layout →
- // back PaimonCallbackDirectory with a mock pread that reads from the
- // packed Vec. Once C++ `ArchiveLayout::Parse` is in place, prod
- // uses the same PaimonCallbackDirectory path.
- let (dir, _backend) = build_directory_from_archive(packed.to_vec());
- PaimonTantivyReader::new(dir, TokenizeMode::Mix, true, &dict_dir()).unwrap()
- }
-
- #[test]
- fn match_all_single_term() {
- let bytes = build(&["hello world", "hello there", "world peace"]);
- let r = open(&bytes);
- let ids = r.search_all(SearchType::MatchAll, "hello").unwrap();
- assert_eq!(ids, vec![0u64, 1]);
- }
-
- #[test]
- fn match_all_two_terms_intersection() {
- let bytes = build(&["hello world", "hello there", "world peace"]);
- let r = open(&bytes);
- let ids = r.search_all(SearchType::MatchAll, "hello world").unwrap();
- assert_eq!(ids, vec![0u64]);
- }
-
- #[test]
- fn match_any_two_terms_union() {
- let bytes = build(&["hello world", "hello there", "world peace"]);
- let r = open(&bytes);
- let ids = r.search_all(SearchType::MatchAny, "hello peace").unwrap();
- assert_eq!(ids, vec![0u64, 1, 2]);
- }
-
- #[test]
- fn phrase_only_consecutive() {
- let bytes = build(&["hello world there", "world hello there"]);
- let r = open(&bytes);
- let ids = r.search_all(SearchType::Phrase, "hello world").unwrap();
- assert_eq!(ids, vec![0u64]);
- }
-
- #[test]
- fn prefix_matches_indexed_terms() {
- let bytes = build(&["unordered user-defined doc id"]);
- let r = open(&bytes);
- let ids = r.search_all(SearchType::Prefix, "unorder").unwrap();
- assert_eq!(ids, vec![0u64]);
- }
-
- #[test]
- fn prefix_normalizes_ascii_alphanumeric_case() {
- let bytes = build(&["This is a test document", "another document"]);
- let r = open(&bytes);
- let ids = r.search_all(SearchType::Prefix, "THIS").unwrap();
- assert_eq!(ids, vec![0u64]);
- }
-
- #[test]
- fn wildcard_with_star() {
- let bytes = build(&["unordered", "ordered", "border"]);
- let r = open(&bytes);
- let ids = r.search_all(SearchType::Wildcard, "*order*").unwrap();
- assert_eq!(ids, vec![0u64, 1, 2]);
- }
-
- #[test]
- fn wildcard_normalizes_ascii_alphanumeric_segments() {
- let bytes = build(&["This is a test document", "another document"]);
- let r = open(&bytes);
- assert_eq!(
- r.search_all(SearchType::Wildcard, "*THIS*").unwrap(),
- vec![0u64]
- );
- assert_eq!(
- r.search_all(SearchType::Wildcard, "*?HIS*").unwrap(),
- vec![0u64]
- );
- }
-
- #[test]
- fn prefix_and_wildcard_preserve_mixed_ascii_cjk_terms() {
- let bytes = build(&["B超检查", "T恤"]);
- let r = open(&bytes);
- assert_eq!(
- r.search_all(SearchType::Prefix, "B").unwrap(),
- vec![0u64]
- );
- assert_eq!(
- r.search_all(SearchType::Wildcard, "*T*").unwrap(),
- vec![1u64]
- );
- }
-
- #[test]
- fn empty_query_for_match_returns_query_parse_error() {
- let bytes = build(&["hello"]);
- let r = open(&bytes);
- let err = r.search_all(SearchType::MatchAll, "").unwrap_err();
- assert!(err.contains("no tokens"), "got: {err}");
- }
-
- #[test]
- fn wildcard_helper_escapes_dots() {
- assert_eq!(wildcard_to_regex("a*b"), "a.*b");
- assert_eq!(wildcard_to_regex("a?b"), "a.b");
- assert_eq!(wildcard_to_regex("a.b"), r"a\.b");
- assert_eq!(wildcard_to_regex("*a*"), ".*a.*");
- }
-
- #[test]
- fn wildcard_normalization_preserves_non_alphanumeric_segments() {
- assert_eq!(normalize_wildcard_query("*?HIS*"), "*?his*");
- assert_eq!(normalize_wildcard_query("*THIS_IS*"), "*THIS_IS*");
- assert_eq!(normalize_wildcard_query("*机器*"), "*机器*");
- }
-
- // ----- limit + pre_filter + scoring (row_id-based) -----
-
- #[test]
- fn limit_returns_top_n_with_scores() {
- let bytes = build(&[
- "doc", // 0: low score (1 occurrence)
- "doc doc doc doc doc", // 1: high score (5 occurrences)
- "doc doc", // 2: medium score
- ]);
- let r = open(&bytes);
- let rows = r
- .search_with_limit_and_filter(SearchType::MatchAll, "doc", true, Some(2), None, None)
- .unwrap();
- assert_eq!(rows.len(), 2);
- // doc 1 has highest TF, expect first
- assert_eq!(rows[0].0, 1u64);
- assert!(rows[0].1.is_some());
- assert!(rows[1].1.is_some());
- // Scores monotonically decreasing
- assert!(rows[0].1.unwrap() >= rows[1].1.unwrap());
- }
-
- #[test]
- fn no_limit_returns_all_unscored() {
- let bytes = build(&["hello world", "world hello", "world peace"]);
- let r = open(&bytes);
- let rows = r
- .search_with_limit_and_filter(SearchType::MatchAll, "world", false, None, None, None)
- .unwrap();
- let ids: Vec<u64> = rows.iter().map(|(id, _)| *id).collect();
- assert_eq!(ids, vec![0u64, 1, 2]);
- assert!(rows.iter().all(|(_, s)| s.is_none()));
- }
-
- #[test]
- fn pre_filter_no_limit_intersects() {
- let bytes = build(&["alpha beta", "alpha gamma", "beta gamma"]);
- let r = open(&bytes);
- // pre_filter = {0, 2}; query "alpha" matches {0, 1}; expect intersection {0}
- let mut tm = Treemap::new();
- tm.add(0);
- tm.add(2);
- let rows = r
- .search_with_limit_and_filter(SearchType::MatchAll, "alpha", false, None, Some(&tm), None)
- .unwrap();
- let ids: Vec<u64> = rows.iter().map(|(id, _)| *id).collect();
- assert_eq!(ids, vec![0u64]);
- }
-
- #[test]
- fn pre_filter_with_limit_filters_before_topn() {
- // doc 0 has highest TF for "doc" but is NOT in pre_filter → must NOT
- // be in result, even with limit=1.
- let bytes = build(&[
- "doc doc doc doc doc", // 0: highest TF, but excluded
- "doc doc", // 1: medium TF, included
- "doc", // 2: low TF, excluded
- ]);
- let r = open(&bytes);
- let mut tm = Treemap::new();
- tm.add(1); // only doc 1 passes pre_filter
- let rows = r
- .search_with_limit_and_filter(SearchType::MatchAll, "doc", true, Some(10), Some(&tm), None)
- .unwrap();
- assert_eq!(rows.len(), 1);
- assert_eq!(rows[0].0, 1u64);
- }
-
- #[test]
- fn unscored_limit_with_pre_filter_applies_filter_before_truncate() {
- // Regression (review finding #1): with_score=false + limit=N + pre_filter
- // must apply the filter to the FULL match set before truncating to N.
- // All three docs match "doc" but only row_id 2 (the LAST one) passes the
- // pre_filter; a truncate-before-filter impl (LimitedDocSetCollector that
- // stops at N raw matches, then filters) would collect doc 0, filter it
- // out, and wrongly return empty instead of {2}.
- let bytes = build(&["doc", "doc", "doc"]);
- let r = open(&bytes);
- let mut tm = Treemap::new();
- tm.add(2); // only row_id 2 passes the pre_filter
- let rows = r
- .search_with_limit_and_filter(SearchType::MatchAll, "doc", false, Some(1), Some(&tm), None)
- .unwrap();
- let ids: Vec<u64> = rows.iter().map(|(id, _)| *id).collect();
- assert_eq!(ids, vec![2u64], "pre_filter must be applied before LIMIT truncation");
- assert!(rows.iter().all(|(_, s)| s.is_none()));
- }
-
- #[test]
- fn empty_pre_filter_returns_empty() {
- let bytes = build(&["alpha", "beta"]);
- let r = open(&bytes);
- let tm = Treemap::new(); // empty
- let rows = r
- .search_with_limit_and_filter(SearchType::MatchAll, "alpha", false, None, Some(&tm), None)
- .unwrap();
- assert!(rows.is_empty());
- }
-
- #[test]
- fn limit_zero_returns_empty_without_running_query() {
- let bytes = build(&["alpha", "beta"]);
- let r = open(&bytes);
- let rows = r
- .search_with_limit_and_filter(SearchType::MatchAll, "alpha", true, Some(0), None, None)
- .unwrap();
- assert!(rows.is_empty());
- }
-
- // ----- row_id is independent of doc_id -----
-
- #[test]
- fn pre_filter_uses_row_id_not_doc_id() {
- // Build with non-contiguous row_ids so doc_id ≠ row_id. Then verify
- // pre_filter operates on row_id values, not internal tantivy doc_ids.
- let mut w = PaimonTantivyWriter::new("f0", TokenizeMode::Mix, true, &dict_dir(), "paimon_jieba").unwrap();
- w.add(100, "alpha").unwrap();
- w.add(200, "alpha").unwrap();
- w.add(300, "alpha").unwrap();
- let bytes = w.finish().unwrap().1;
- let r = open(&bytes);
-
- // pre_filter = {200} as row_id (doc_id would be 1)
- let mut tm = Treemap::new();
- tm.add(200);
- let rows = r
- .search_with_limit_and_filter(SearchType::MatchAll, "alpha", false, None, Some(&tm), None)
- .unwrap();
- let ids: Vec<u64> = rows.iter().map(|(id, _)| *id).collect();
- assert_eq!(ids, vec![200u64], "pre_filter must operate on row_id, not doc_id");
- }
-
- #[test]
- fn search_returns_caller_supplied_row_ids() {
- // Same setup: row_ids 100/200/300, verify search_all returns those values.
- let mut w = PaimonTantivyWriter::new("f0", TokenizeMode::Mix, true, &dict_dir(), "paimon_jieba").unwrap();
- w.add(100, "doc").unwrap();
- w.add(200, "doc").unwrap();
- w.add(300, "doc").unwrap();
- let bytes = w.finish().unwrap().1;
- let r = open(&bytes);
- let ids = r.search_all(SearchType::MatchAll, "doc").unwrap();
- assert_eq!(ids, vec![100u64, 200, 300]);
- }
-
- #[test]
- fn tokenizer_name_reflects_paimon_jieba_schema_for_cpp_written_index() {
- // cpp-written index: PaimonTantivyWriter binds the text field to
- // `paimon_jieba`. Reader must pick that up from meta.json (not hardcode).
- let bytes = build(&["hello world"]);
- let r = open(&bytes);
- assert_eq!(r.tokenizer_name(), PAIMON_TOKENIZER_NAME);
-
- // tokenize sanity: jieba mode="mix" picks `hello` + `world` from ASCII.
- let q = r.tokenize_query("hello world");
- assert_eq!(q, vec!["hello".to_string(), "world".to_string()]);
- }
-
- #[test]
- fn tokenizer_name_reflects_default_schema_for_externally_written_index() {
- // Simulate a paimon-java-shaped index: text field bound to the
- // builtin `default` tokenizer (SimpleTokenizer + LowerCaser), not jieba.
- // Build it directly via tantivy (bypassing PaimonTantivyWriter's jieba
- // schema) so we can prove the reader auto-switches to the builtin.
- use crate::callback_directory::test_support::build_mock_directory;
- use tantivy::directory::Directory;
- use tantivy::schema::{IndexRecordOption, NumericOptions, Schema, TextFieldIndexing, TextOptions};
- use tantivy::{doc, Index};
-
- // Build a minimal index with field "text" bound to "default".
- let mut sb = Schema::builder();
- let row_id_f = sb.add_u64_field(
- "row_id",
- NumericOptions::default().set_stored().set_indexed().set_fast(),
- );
- let text_opts = TextOptions::default().set_indexing_options(
- TextFieldIndexing::default()
- .set_tokenizer("default") // ← key: match paimon-java's TEXT default
- .set_index_option(IndexRecordOption::WithFreqsAndPositions),
- );
- let text_f = sb.add_text_field("text", text_opts);
- let schema = sb.build();
- let tmp = tempfile::Builder::new()
- .prefix("paimon-tantivy-dyn-tk-")
- .tempdir()
- .unwrap();
- let index = Index::create_in_dir(tmp.path(), schema).unwrap();
- let mut writer = index.writer(15_000_000).unwrap();
- writer
- .add_document(doc!(row_id_f => 0u64, text_f => "Hello World"))
- .unwrap();
- writer
- .add_document(doc!(row_id_f => 1u64, text_f => "Apple.Banana"))
- .unwrap();
- writer.commit().unwrap();
- writer.wait_merging_threads().unwrap();
-
- // Pack the index dir into our archive format so the callback directory
- // can serve it. Reuse writer.rs's format by streaming entries manually.
- let mut data = Vec::new();
- let mut entries = Vec::<(String, u64, u64)>::new();
- let dir_iter = std::fs::read_dir(tmp.path()).unwrap();
- let mut files: Vec<_> = dir_iter
- .filter_map(|e| e.ok())
- .filter(|e| e.file_type().ok().map_or(false, |t| t.is_file()))
- .filter(|e| !e.file_name().to_string_lossy().starts_with('.'))
- .collect();
- files.sort_by_key(|e| e.file_name());
- data.extend_from_slice(&(files.len() as i32).to_be_bytes());
- for e in &files {
- let name = e.file_name().to_string_lossy().into_owned();
- let bytes = std::fs::read(e.path()).unwrap();
- data.extend_from_slice(&(name.len() as i32).to_be_bytes());
- data.extend_from_slice(name.as_bytes());
- data.extend_from_slice(&(bytes.len() as i64).to_be_bytes());
- let off = data.len() as u64;
- data.extend_from_slice(&bytes);
- entries.push((name, off, bytes.len() as u64));
- }
-
- let (dir, _backend) = build_mock_directory(data, entries);
- let r = PaimonTantivyReader::new(dir, TokenizeMode::Mix, true, &dict_dir()).unwrap();
-
- // Reader must pick up `default` from schema, not hardcode `paimon_jieba`.
- assert_eq!(r.tokenizer_name(), "default");
-
- // Query tokenization now goes through tantivy's builtin default
- // (SimpleTokenizer + LowerCaser):
- // "Apple.Banana" → ["apple", "banana"] (dot is non-alnum, split)
- // "Hello World" → ["hello", "world"] (space split + lowercase)
- let q1 = r.tokenize_query("Hello World");
- assert_eq!(q1, vec!["hello".to_string(), "world".to_string()]);
- let q2 = r.tokenize_query("Apple.Banana");
- assert_eq!(q2, vec!["apple".to_string(), "banana".to_string()]);
-
- // And the search path works across tokenizer:
- let ids = r.search_all(SearchType::MatchAll, "hello").unwrap();
- assert_eq!(ids, vec![0u64]);
- let ids = r.search_all(SearchType::MatchAll, "apple").unwrap();
- assert_eq!(ids, vec![1u64]);
- }
-
- #[test]
- fn reader_aggregates_row_ids_across_segments() {
- // Multi-thread default writer + many docs => may produce multiple
- // segments before force-merge. After finish(), force-merge collapses
- // to one segment, but this test still validates the row_id retrieval
- // path works for ≥1 segment.
- let mut w = PaimonTantivyWriter::new("f0", TokenizeMode::Mix, true, &dict_dir(), "paimon_jieba").unwrap();
- for i in 0..200u64 {
- w.add(i * 7, &format!("docmark_{i} apple")).unwrap();
- }
- let bytes = w.finish().unwrap().1;
- let r = open(&bytes);
- let ids = r.search_all(SearchType::MatchAll, "apple").unwrap();
- assert_eq!(ids.len(), 200);
- for i in 0..200u64 {
- assert!(ids.contains(&(i * 7)), "missing row_id={}", i * 7);
- }
- }
-}
diff --git a/crates/tantivy_ffi/src/tokenizer.rs b/crates/tantivy_ffi/src/tokenizer.rs
deleted file mode 100644
index 19139fcb..00000000
--- a/crates/tantivy_ffi/src/tokenizer.rs
+++ /dev/null
@@ -1,467 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//! PaimonJiebaTokenizer: tantivy Tokenizer impl wrapping jieba-rs.
-//!
-//! Contract:
-//! - Behavior-equivalent with `JiebaAnalyzer` in src/paimon/global_index/lucene/
-//! - 5 modes: mp / hmm / mix / full / query
-//! - `hmm` is Unsupported (jieba-rs has no standalone HMM entry point)
-//! - `mp` accepts cut(hmm=false) but does not replicate cppjieba's
-//! max_word_len truncation
-//! - Normalize: skip pure whitespace, skip stop_words, lowercase ASCII-only tokens
-//! - Token offsets: byte offsets into the original UTF-8 string
-//! - `with_position=false`: all tokens emitted at `position=0` (disables PhraseQuery)
-//! - Custom dict dir: loads `jieba.dict.utf8` (+optional `user.dict.utf8`) from
-//! `$PAIMON_JIEBA_DICT_DIR`; stop_words.utf8 loaded if present
-
-use std::collections::HashSet;
-use std::ffi::{c_char, CStr};
-use std::fs::File;
-use std::io::{BufRead, BufReader};
-use std::path::Path;
-use std::sync::Arc;
-
-use jieba_rs::Jieba;
-use tantivy::tokenizer::{Token, TokenStream, Tokenizer};
-
-use crate::buffer::PaimonTantivyBuffer;
-use crate::error::{set_last_error, PaimonTantivyStatus};
-use crate::handle::{borrow_handle, free_handle, into_handle};
-
-#[derive(Clone, Copy, Debug, Eq, PartialEq)]
-pub enum TokenizeMode {
- Mp,
- Hmm,
- Mix,
- Full,
- Query,
-}
-
-impl TokenizeMode {
- pub(crate) fn parse(s: &str) -> Option<Self> {
- match s {
- "mp" => Some(Self::Mp),
- "hmm" => Some(Self::Hmm),
- "mix" => Some(Self::Mix),
- "full" => Some(Self::Full),
- "query" => Some(Self::Query),
- _ => None,
- }
- }
-}
-
-#[derive(Clone)]
-pub struct PaimonJiebaTokenizer {
- jieba: Arc<Jieba>,
- mode: TokenizeMode,
- with_position: bool,
- stop_words: Arc<HashSet<String>>,
-}
-
-impl PaimonJiebaTokenizer {
- pub fn new(
- dict_dir: &Path,
- mode: TokenizeMode,
- with_position: bool,
- ) -> Result<Self, String> {
- if mode == TokenizeMode::Hmm {
- return Err(
- "tokenize mode 'hmm' is not supported (jieba-rs does not expose standalone HMM)"
- .into(),
- );
- }
- let jieba = load_jieba(dict_dir)?;
- let stop_words = load_stop_words(dict_dir);
- Ok(Self {
- jieba: Arc::new(jieba),
- mode,
- with_position,
- stop_words: Arc::new(stop_words),
- })
- }
-
- /// Directly tokenize, returning a Vec of (offset_start, offset_end, text) tuples.
- /// Used both by the tantivy Tokenizer impl and the standalone `tokenize` FFI.
- pub fn tokenize_raw(&self, text: &str) -> Vec<(usize, usize, String)> {
- // Use jieba-rs's cut variants which return Vec<&'a str>; compute byte offsets
- // via pointer arithmetic (each &str is a slice of the original).
- let cuts: Vec<&str> = match self.mode {
- TokenizeMode::Mp => self.jieba.cut(text, false),
- TokenizeMode::Hmm => Vec::new(), // unreachable (caught in new())
- TokenizeMode::Mix => self.jieba.cut(text, true),
- TokenizeMode::Full => self.jieba.cut_all(text),
- TokenizeMode::Query => self.jieba.cut_for_search(text, true),
- };
-
- let text_start = text.as_ptr() as usize;
- let mut out = Vec::with_capacity(cuts.len());
- for piece in cuts {
- // skip pure whitespace
- if piece.chars().all(char::is_whitespace) {
- continue;
- }
- // skip stop words (compare original case)
- if self.stop_words.contains(piece) {
- continue;
- }
- // offset calc
- let start = piece.as_ptr() as usize - text_start;
- let end = start + piece.len();
- // lowercase only if pure ASCII alphanumeric (match cppjieba Normalize behavior)
- let token_text = normalize_case(piece);
- out.push((start, end, token_text));
- }
- out
- }
-}
-
-fn is_ascii_alnum(s: &str) -> bool {
- !s.is_empty() && s.bytes().all(|b| b.is_ascii_alphanumeric())
-}
-
-pub(crate) fn normalize_case(s: &str) -> String {
- if is_ascii_alnum(s) {
- s.to_ascii_lowercase()
- } else {
- s.to_string()
- }
-}
-
-fn load_jieba(dict_dir: &Path) -> Result<Jieba, String> {
- let main_dict = dict_dir.join("jieba.dict.utf8");
- let mut jieba = if main_dict.exists() {
- let file = File::open(&main_dict)
- .map_err(|e| format!("open {}: {e}", main_dict.display()))?;
- let mut rdr = BufReader::new(file);
- Jieba::with_dict(&mut rdr).map_err(|e| format!("load jieba dict: {e:?}"))?
- } else {
- // No custom dict; use jieba-rs builtin
- Jieba::new()
- };
- // Optional user dict. cppjieba's user.dict.utf8 is lenient: lines are
- // `word [freq] [tag]` where freq can be omitted (e.g. `<word> nz`), but
- // jieba-rs's load_dict strictly requires `word freq [tag]` and fails if
- // freq is not an integer. We parse line-by-line with `add_word` to stay
- // compatible.
- let user_dict = dict_dir.join("user.dict.utf8");
- if user_dict.exists() {
- let file = File::open(&user_dict)
- .map_err(|e| format!("open {}: {e}", user_dict.display()))?;
- for (n, line_res) in BufReader::new(file).lines().enumerate() {
- let line = match line_res {
- Ok(l) => l,
- Err(_) => continue, // skip unreadable lines
- };
- let trimmed = line.trim();
- if trimmed.is_empty() || trimmed.starts_with('#') {
- continue;
- }
- let mut it = trimmed.split_whitespace();
- let word = it.next().unwrap(); // non-empty guaranteed
- let next = it.next();
- let freq = next.and_then(|s| s.parse::<usize>().ok());
- let tag = match (freq, next) {
- (Some(_), _) => it.next(), // <word> <freq> [tag]
- (None, tok) => tok, // <word> <tag> (no freq)
- };
- // `add_word` returns the assigned frequency; ignore it. For lines
- // with bogus content we silently keep going, matching cppjieba's
- // tolerant behavior.
- let _ = jieba.add_word(word, freq, tag);
- let _ = n; // keep for potential debug
- }
- }
- Ok(jieba)
-}
-
-fn load_stop_words(dict_dir: &Path) -> HashSet<String> {
- let path = dict_dir.join("stop_words.utf8");
- let mut out = HashSet::new();
- if let Ok(f) = File::open(&path) {
- for line in BufReader::new(f).lines().map_while(Result::ok) {
- let w = line.trim();
- if !w.is_empty() {
- out.insert(w.to_owned());
- }
- }
- }
- out
-}
-
-// ----------------- tantivy Tokenizer integration -----------------
-
-pub struct PaimonJiebaTokenStream {
- tokens: Vec<Token>,
- index: usize,
-}
-
-impl TokenStream for PaimonJiebaTokenStream {
- fn advance(&mut self) -> bool {
- self.index += 1;
- self.index <= self.tokens.len()
- }
-
- fn token(&self) -> &Token {
- &self.tokens[self.index - 1]
- }
-
- fn token_mut(&mut self) -> &mut Token {
- &mut self.tokens[self.index - 1]
- }
-}
-
-impl Tokenizer for PaimonJiebaTokenizer {
- type TokenStream<'a> = PaimonJiebaTokenStream;
-
- fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {
- let raw = self.tokenize_raw(text);
- let tokens: Vec<Token> = raw
- .into_iter()
- .enumerate()
- .map(|(i, (s, e, t))| Token {
- offset_from: s,
- offset_to: e,
- position: if self.with_position { i } else { 0 },
- text: t,
- position_length: 1,
- })
- .collect();
- PaimonJiebaTokenStream { tokens, index: 0 }
- }
-}
-
-// ----------------- FFI surface -----------------
-
-/// Create a tokenizer handle. Returns OK and writes *out on success; returns
-/// status and sets last_error on failure.
-///
-/// SAFETY: `mode_cstr` and `dict_dir_cstr` must be NUL-terminated UTF-8;
-/// `out` must be a valid non-null pointer.
-#[no_mangle]
-pub unsafe extern "C" fn paimon_tantivy_tokenizer_new(
- mode_cstr: *const c_char,
- with_position: bool,
- dict_dir_cstr: *const c_char,
- out: *mut *mut PaimonJiebaTokenizer,
-) -> PaimonTantivyStatus {
- if mode_cstr.is_null() || dict_dir_cstr.is_null() || out.is_null() {
- set_last_error("paimon_tantivy_tokenizer_new: null argument");
- return PaimonTantivyStatus::InvalidArgument;
- }
- let mode_s = match unsafe { CStr::from_ptr(mode_cstr) }.to_str() {
- Ok(s) => s,
- Err(e) => {
- set_last_error(format!("mode not utf-8: {e}"));
- return PaimonTantivyStatus::InvalidArgument;
- }
- };
- let dict_s = match unsafe { CStr::from_ptr(dict_dir_cstr) }.to_str() {
- Ok(s) => s,
- Err(e) => {
- set_last_error(format!("dict_dir not utf-8: {e}"));
- return PaimonTantivyStatus::InvalidArgument;
- }
- };
- let mode = match TokenizeMode::parse(mode_s) {
- Some(m) => m,
- None => {
- set_last_error(format!(
- "unknown tokenize mode {mode_s:?}; expected one of mp/hmm/mix/full/query"
- ));
- return PaimonTantivyStatus::InvalidArgument;
- }
- };
- match PaimonJiebaTokenizer::new(Path::new(dict_s), mode, with_position) {
- Ok(t) => {
- unsafe { *out = into_handle(t) };
- PaimonTantivyStatus::Ok
- }
- Err(e) => {
- let is_hmm_unsupported = e.contains("'hmm' is not supported");
- set_last_error(e);
- if is_hmm_unsupported {
- PaimonTantivyStatus::Unsupported
- } else {
- PaimonTantivyStatus::TokenizerError
- }
- }
- }
-}
-
-/// Free a tokenizer handle. Safe on null.
-#[no_mangle]
-pub unsafe extern "C" fn paimon_tantivy_tokenizer_free(tok: *mut PaimonJiebaTokenizer) {
- unsafe { free_handle(tok) };
-}
-
-/// Tokenize a string and return a newline-delimited list of tokens as bytes.
-/// Used for golden-sample tests (easy to diff from C++).
-///
-/// Output format:
-/// `<offset_from>\t<offset_to>\t<position>\t<text>\n` for each token.
-///
-/// SAFETY: `tok` must be a valid handle; `text` must point to `text_len` UTF-8 bytes;
-/// `out` must be non-null.
-#[no_mangle]
-pub unsafe extern "C" fn paimon_tantivy_tokenizer_tokenize(
- tok: *const PaimonJiebaTokenizer,
- text: *const c_char,
- text_len: usize,
- out: *mut PaimonTantivyBuffer,
-) -> PaimonTantivyStatus {
- if out.is_null() {
- set_last_error("paimon_tantivy_tokenizer_tokenize: out is null");
- return PaimonTantivyStatus::InvalidArgument;
- }
- let Some(tokenizer) = (unsafe { borrow_handle::<PaimonJiebaTokenizer>(tok) }) else {
- set_last_error("paimon_tantivy_tokenizer_tokenize: null tokenizer handle");
- return PaimonTantivyStatus::InvalidArgument;
- };
- if text.is_null() && text_len != 0 {
- set_last_error("text is null but len > 0");
- return PaimonTantivyStatus::InvalidArgument;
- }
- let text_str = if text_len == 0 {
- ""
- } else {
- let slice = unsafe { std::slice::from_raw_parts(text as *const u8, text_len) };
- match std::str::from_utf8(slice) {
- Ok(s) => s,
- Err(e) => {
- set_last_error(format!("text not utf-8: {e}"));
- return PaimonTantivyStatus::InvalidArgument;
- }
- }
- };
- let raw = tokenizer.tokenize_raw(text_str);
- let mut buf = String::new();
- for (i, (s, e, t)) in raw.iter().enumerate() {
- let pos = if tokenizer.with_position { i } else { 0 };
- buf.push_str(&format!("{s}\t{e}\t{pos}\t{t}\n"));
- }
- let bytes = buf.into_bytes();
- unsafe { *out = PaimonTantivyBuffer::from_vec(bytes) };
- PaimonTantivyStatus::Ok
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
- use std::ffi::CString;
-
- fn dict_dir_from_env() -> std::path::PathBuf {
- std::env::var("PAIMON_JIEBA_DICT_DIR")
- .map(std::path::PathBuf::from)
- .unwrap_or_else(|_| std::path::PathBuf::from("/tmp/nonexistent-dict"))
- }
-
- #[test]
- fn mode_parse() {
- for (s, m) in [
- ("mp", TokenizeMode::Mp),
- ("hmm", TokenizeMode::Hmm),
- ("mix", TokenizeMode::Mix),
- ("full", TokenizeMode::Full),
- ("query", TokenizeMode::Query),
- ] {
- assert_eq!(TokenizeMode::parse(s), Some(m));
- }
- assert!(TokenizeMode::parse("bogus").is_none());
- }
-
- #[test]
- fn hmm_mode_returns_unsupported() {
- let tok = PaimonJiebaTokenizer::new(
- &dict_dir_from_env(),
- TokenizeMode::Hmm,
- true,
- );
- match tok {
- Err(e) => assert!(e.contains("'hmm' is not supported"), "got: {e}"),
- Ok(_) => panic!("expected Err"),
- }
- }
-
- #[test]
- fn tokenize_mix_default_dict_smoke() {
- // If no custom dict dir, jieba-rs builtin is used.
- let t = PaimonJiebaTokenizer::new(Path::new("/tmp/nonexistent-dict"), TokenizeMode::Mix, true)
- .unwrap();
- let raw = t.tokenize_raw("他来到了网易杭研大厦");
- let texts: Vec<&str> = raw.iter().map(|(_, _, s)| s.as_str()).collect();
- assert!(texts.contains(&"网易"));
- assert!(texts.contains(&"大厦"));
- }
-
- #[test]
- fn ascii_alnum_is_lowercased() {
- let t = PaimonJiebaTokenizer::new(Path::new("/tmp/nx"), TokenizeMode::Mix, true).unwrap();
- let raw = t.tokenize_raw("Hello World 中国");
- let texts: Vec<&str> = raw.iter().map(|(_, _, s)| s.as_str()).collect();
- assert!(texts.contains(&"hello"));
- assert!(texts.contains(&"world"));
- assert!(texts.contains(&"中国"));
- }
-
- #[test]
- fn with_position_false_emits_zero_position() {
- let t = PaimonJiebaTokenizer::new(Path::new("/tmp/nx"), TokenizeMode::Mix, false).unwrap();
- let raw = t.tokenize_raw("中国人");
- // Can't check position on raw tuples; check via tantivy Token stream:
- let mut t2 = t.clone();
- let mut stream = <PaimonJiebaTokenizer as Tokenizer>::token_stream(&mut t2, "中国人");
- let mut positions = Vec::new();
- while stream.advance() {
- positions.push(stream.token().position);
- }
- assert!(!raw.is_empty());
- assert!(positions.iter().all(|&p| p == 0));
- }
-
- #[test]
- fn ffi_roundtrip() {
- let dict = dict_dir_from_env();
- let dict_str = dict.to_str().unwrap();
- let mode = CString::new("mix").unwrap();
- let dict_c = CString::new(dict_str).unwrap();
- let mut handle: *mut PaimonJiebaTokenizer = std::ptr::null_mut();
- unsafe {
- let st = paimon_tantivy_tokenizer_new(
- mode.as_ptr(),
- true,
- dict_c.as_ptr(),
- &mut handle,
- );
- assert_eq!(st, PaimonTantivyStatus::Ok);
- assert!(!handle.is_null());
-
- let input = "Hello 中国";
- let input_c = CString::new(input).unwrap();
- let mut buf = PaimonTantivyBuffer::empty();
- let st2 = paimon_tantivy_tokenizer_tokenize(
- handle,
- input_c.as_ptr(),
- input.len(),
- &mut buf,
- );
- assert_eq!(st2, PaimonTantivyStatus::Ok);
- assert!(buf.len > 0);
- crate::buffer::paimon_tantivy_buffer_free(&mut buf);
- paimon_tantivy_tokenizer_free(handle);
- }
- }
-}
diff --git a/crates/tantivy_ffi/src/writer.rs b/crates/tantivy_ffi/src/writer.rs
deleted file mode 100644
index 1a068e7b..00000000
--- a/crates/tantivy_ffi/src/writer.rs
+++ /dev/null
@@ -1,788 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//! PaimonTantivyWriter: Writer for tantivy-fulltext global index.
-//!
-//! Contract:
-//! - `writer_new(field_name, mode, with_position, dict_dir, out)` — create on a
-//! private tmp dir backed by MmapDirectory + PaimonJiebaTokenizer.
-//! `field_name` is **ignored** by the Rust schema (kept for FFI ABI
-//! compatibility); schema field names are fixed (`row_id`, `text`) to match
-//! paimon-java `paimon-tantivy-jni/rust/src/lib.rs:55-66`.
-//! - `writer_add(writer, row_id, text, len)` — add a single document with the
-//! caller-supplied `row_id` (u64) and a TEXT field
-//! - `writer_finish(writer, out_row_count, out_buf)` — commit + force-merge to
-//! single segment + pack all on-disk index files into a Rust-allocated buffer
-//! - `writer_free(writer)` — destroy (RAII removes tmp dir)
-//!
-//! Packing format (big-endian, **cross-readable with paimon-java archives**):
-//! `[i32 BE file_count |
-//! (i32 BE name_len | name_bytes | i64 BE file_len | file_bytes)*]`
-
-use std::ffi::{c_char, c_void, CStr};
-use std::fs::File;
-use std::io::Read;
-use std::path::{Path, PathBuf};
-
-use tantivy::schema::{
- Field, IndexRecordOption, NumericOptions, Schema, TextFieldIndexing, TextOptions,
-};
-use tantivy::{doc, Index, IndexWriter, TantivyDocument};
-use tempfile::TempDir;
-
-use crate::error::{set_last_error, PaimonTantivyStatus};
-use crate::handle::{borrow_handle_mut, free_handle, into_handle};
-use crate::tokenizer::{PaimonJiebaTokenizer, TokenizeMode};
-
-/// Schema field names. Fixed to match paimon-java's tantivy schema so that
-/// indexes are cross-readable. Both fields are required.
-pub const PAIMON_ROW_ID_FIELD_NAME: &str = "row_id";
-pub const PAIMON_TEXT_FIELD_NAME: &str = "text";
-
-/// Name registered with the tantivy `TokenizerManager`. Reader must register
-/// the same name to make stored term dictionaries readable.
-pub const PAIMON_TOKENIZER_NAME: &str = "paimon_jieba";
-
-/// Heap budget for the in-process IndexWriter (50 MB; tantivy minimum is ~3 MB).
-/// Default multi-threaded writer (`Index::writer(heap)`) splits this budget
-/// across `min(num_cpus, MAX_NUM_THREAD=8)` worker threads.
-const WRITER_HEAP_SIZE: usize = 50_000_000;
-
-pub struct PaimonTantivyWriter {
- /// Owned tmp dir; cleaned up when this struct drops.
- tmpdir: TempDir,
- /// `row_id` u64 field (stored + indexed + fast). Reader retrieves the
- /// caller-supplied row_id via `fast_fields().u64("row_id").first(doc_id)`.
- row_id_field: Field,
- /// `text` TEXT field tokenized via the registered jieba tokenizer.
- text_field: Field,
- /// tantivy index instance, file-backed in `tmpdir`.
- index: Index,
- /// Active writer; consumed by `wait_merging_threads()` in `finish`.
- writer: Option<IndexWriter>,
- /// Documents added since construction.
- row_count: i64,
-}
-
-impl PaimonTantivyWriter {
- pub fn new(
- field_name: &str,
- mode: TokenizeMode,
- with_position: bool,
- dict_dir: &Path,
- tokenizer_name: &str,
- ) -> Result<Self, String> {
- if field_name.is_empty() {
- return Err("field_name must be non-empty".into());
- }
- // Schema is fixed to match paimon-java: row_id (u64
- // stored+indexed+fast) + text (TEXT). The caller-supplied `field_name`
- // parameter is currently ignored by the Rust schema (kept for FFI
- // backward-compatibility); the C++ side still uses it to extract the
- // right column from arrow batches.
- let _ = field_name; // intentionally unused on the Rust side
- let mut schema_builder = Schema::builder();
- let row_id_field = schema_builder.add_u64_field(
- PAIMON_ROW_ID_FIELD_NAME,
- NumericOptions::default()
- .set_stored()
- .set_indexed()
- .set_fast(),
- );
- let index_option = if with_position {
- IndexRecordOption::WithFreqsAndPositions
- } else {
- IndexRecordOption::Basic
- };
- // Empty input falls back to tantivy's built-in "default" (SimpleTokenizer),
- // matching the cpp-side default in `tantivy_defs.h::kDefaultTantivyWriteTokenizer`.
- // Cross-read with paimon-java works out of the box; CJK callers must
- // pass "paimon_jieba" explicitly.
- let effective_tokenizer = if tokenizer_name.is_empty() {
- "default"
- } else {
- tokenizer_name
- };
- let text_options = TextOptions::default().set_indexing_options(
- TextFieldIndexing::default()
- .set_tokenizer(effective_tokenizer)
- .set_index_option(index_option),
- );
- let text_field = schema_builder.add_text_field(PAIMON_TEXT_FIELD_NAME, text_options);
- let schema = schema_builder.build();
-
- let tmpdir = tempfile::Builder::new()
- .prefix("paimon-tantivy-")
- .tempdir()
- .map_err(|e| format!("create tmp dir: {e}"))?;
-
- let index = Index::create_in_dir(tmpdir.path(), schema)
- .map_err(|e| format!("create tantivy index: {e}"))?;
- // When caller picks "paimon_jieba" we construct + register the jieba
- // tokenizer. For any tantivy built-in name ("default", "whitespace",
- // "raw", "en_stem", ...) tantivy's TokenizerManager already has it
- // registered via `TokenizerManager::default()`; no-op here. This lets
- // paimon-cpp emit archives cross-readable by paimon-java's default
- // TEXT tokenizer path.
- if effective_tokenizer == PAIMON_TOKENIZER_NAME {
- let tokenizer = PaimonJiebaTokenizer::new(dict_dir, mode, with_position)
- .map_err(|e| format!("create tokenizer: {e}"))?;
- index
- .tokenizers()
- .register(PAIMON_TOKENIZER_NAME, tokenizer);
- }
-
- // Default multi-threaded writer (schema stores row_id explicitly so
- // we no longer need single-threaded ordering invariants). tantivy will
- // use min(num_cpus, MAX_NUM_THREAD=8) workers, splitting heap budget.
- let writer: IndexWriter = index
- .writer(WRITER_HEAP_SIZE)
- .map_err(|e| format!("create index writer: {e}"))?;
-
- Ok(Self {
- tmpdir,
- row_id_field,
- text_field,
- index,
- writer: Some(writer),
- row_count: 0,
- })
- }
-
- pub fn add(&mut self, row_id: u64, text: &str) -> Result<(), String> {
- let writer = self
- .writer
- .as_mut()
- .ok_or_else(|| "writer already finished".to_string())?;
- let document: TantivyDocument = doc!(
- self.row_id_field => row_id,
- self.text_field => text,
- );
- writer
- .add_document(document)
- .map_err(|e| format!("add document: {e}"))?;
- self.row_count += 1;
- Ok(())
- }
-
- /// Commit + force-merge + GC on-disk index. Extracted from `finish_*`
- /// so both streaming and test paths can share it.
- fn commit_and_merge(&mut self) -> Result<(), String> {
- let mut writer = self
- .writer
- .take()
- .ok_or_else(|| "writer already finished".to_string())?;
- writer.commit().map_err(|e| format!("commit: {e}"))?;
-
- let segment_metas = self
- .index
- .searchable_segment_metas()
- .map_err(|e| format!("list segments: {e}"))?;
- if segment_metas.len() > 1 {
- let segment_ids: Vec<_> = segment_metas.iter().map(|m| m.id()).collect();
- writer
- .merge(&segment_ids)
- .wait()
- .map_err(|e| format!("merge: {e}"))?;
- }
- writer
- .garbage_collect_files()
- .wait()
- .map_err(|e| format!("garbage_collect_files: {e}"))?;
- writer
- .wait_merging_threads()
- .map_err(|e| format!("wait_merging_threads: {e}"))?;
- Ok(())
- }
-
- /// Streaming finish (production path): commit + force-merge + push archive
- /// bytes through the FFI callback in fixed-size chunks (see
- /// WRITER_STREAM_BUFFER_SIZE). Peak RAM independent of archive size — one
- /// heap buffer + a few KB metadata.
- pub fn finish_streaming(
- &mut self,
- cb: &PaimonWriteCallbacks,
- ) -> Result<i64, String> {
- self.commit_and_merge()?;
- let ctx = cb.ctx;
- let write_fn = cb.write;
- pack_index_dir_stream(self.tmpdir.path(), |bytes| {
- // Calling extern "C" fn pointer is safe; C++ side owns ctx validity.
- let rc = (write_fn)(ctx, bytes.as_ptr(), bytes.len());
- if rc != 0 {
- return Err(format!("write callback rc={rc} len={}", bytes.len()));
- }
- Ok(())
- })?;
- Ok(self.row_count)
- }
-
- /// Test-only convenience: collect streaming output into a `Vec<u8>`.
- /// Rust unit tests / integration tests use this; production path is
- /// `finish_streaming`.
- #[cfg(test)]
- pub(crate) fn finish(&mut self) -> Result<(i64, Vec<u8>), String> {
- self.commit_and_merge()?;
- let mut out: Vec<u8> = Vec::new();
- pack_index_dir_stream(self.tmpdir.path(), |bytes| {
- out.extend_from_slice(bytes);
- Ok(())
- })?;
- Ok((self.row_count, out))
- }
-
- #[cfg(test)]
- pub(crate) fn tmpdir_path(&self) -> &Path {
- self.tmpdir.path()
- }
-}
-
-// =========================================================================
-// Streaming pack
-// =========================================================================
-
-/// Streaming pack buffer size. 1MB matches the buffer size data-lake storage
-/// backends (e.g. Pangu) use for good throughput, still far below any archive
-/// size we care about. Heap-allocated (see pack_index_dir_stream), so the size
-/// does not affect stack usage.
-const WRITER_STREAM_BUFFER_SIZE: usize = 1024 * 1024;
-
-/// Callback table passed from C++ for streaming writer output.
-///
-/// `ctx` is an opaque pointer to C++'s `WriteCtx` (holding a `paimon::OutputStream`).
-/// `write` is called in-order by Rust (not concurrently) to push bytes.
-#[repr(C)]
-pub struct PaimonWriteCallbacks {
- pub ctx: *mut c_void,
- /// Returns 0 on success, non-zero to signal C++ side error (Rust aborts pack).
- pub write: extern "C" fn(ctx: *mut c_void, data: *const u8, len: usize) -> i32,
-}
-
-/// Walk tempdir + pack into the Java-compatible archive format, pushing each
-/// chunk through `write_fn`. Peak RAM = one WRITER_STREAM_BUFFER_SIZE heap
-/// buffer + a few KB of entry metadata (name + PathBuf + u64 length). Mirrors
-/// Java `TantivyFullTextGlobalIndexWriter.packIndex` but with a bigger buffer.
-///
-/// Archive format (BE, no version): `[i32 file_count | (i32 name_len, name,
-/// i64 file_len, file_bytes)*]`. Files sorted alphabetically for deterministic
-/// output; `.`-prefixed (lock) files and non-regular entries skipped.
-fn pack_index_dir_stream<F>(dir: &Path, mut write_fn: F) -> Result<(), String>
-where
- F: FnMut(&[u8]) -> Result<(), String>,
-{
- let entries = collect_dir_entries(dir)?;
-
- // Header: BE i32 file_count
- write_fn(&(entries.len() as i32).to_be_bytes())?;
-
- let mut buf = vec![0u8; WRITER_STREAM_BUFFER_SIZE];
- for (name, path, file_len) in &entries {
- // Per-entry header: name_len, name, data_len
- write_fn(&(name.len() as i32).to_be_bytes())?;
- write_fn(name.as_bytes())?;
- write_fn(&(*file_len as i64).to_be_bytes())?;
-
- // Payload: fixed-size buffer loop
- let mut f = File::open(path)
- .map_err(|e| format!("open {}: {e}", path.display()))?;
- let mut pushed: u64 = 0;
- loop {
- let n = f
- .read(&mut buf)
- .map_err(|e| format!("read {}: {e}", path.display()))?;
- if n == 0 {
- break;
- }
- write_fn(&buf[..n])?;
- pushed += n as u64;
- }
- if pushed != *file_len {
- return Err(format!(
- "file {} changed size during packing: header said {}, streamed {}",
- name, file_len, pushed
- ));
- }
- }
- Ok(())
-}
-
-/// Enumerate the tempdir: sorted (name, path, len) for regular non-`.lock` files.
-fn collect_dir_entries(dir: &Path) -> Result<Vec<(String, PathBuf, u64)>, String> {
- let mut entries: Vec<(String, PathBuf, u64)> = Vec::new();
- let read_dir =
- std::fs::read_dir(dir).map_err(|e| format!("read tmp dir {}: {e}", dir.display()))?;
- for entry_res in read_dir {
- let entry = entry_res.map_err(|e| format!("read entry: {e}"))?;
- let name = match entry.file_name().into_string() {
- Ok(n) => n,
- Err(_) => continue,
- };
- if name.starts_with('.') {
- continue;
- }
- let ft = entry
- .file_type()
- .map_err(|e| format!("file_type for {}: {e}", entry.path().display()))?;
- if !ft.is_file() {
- continue;
- }
- let len = entry
- .metadata()
- .map_err(|e| format!("metadata for {}: {e}", entry.path().display()))?
- .len();
- entries.push((name, entry.path(), len));
- }
- entries.sort_by(|a, b| a.0.cmp(&b.0));
- Ok(entries)
-}
-
-// ============================ FFI surface ============================
-
-/// Create a writer handle on a private tmp dir.
-///
-/// SAFETY: all C-string args must be NUL-terminated UTF-8; `out` non-null.
-#[no_mangle]
-pub unsafe extern "C" fn paimon_tantivy_writer_new(
- field_name_cstr: *const c_char,
- mode_cstr: *const c_char,
- with_position: bool,
- dict_dir_cstr: *const c_char,
- tokenizer_cstr: *const c_char,
- out: *mut *mut PaimonTantivyWriter,
-) -> PaimonTantivyStatus {
- if field_name_cstr.is_null()
- || mode_cstr.is_null()
- || dict_dir_cstr.is_null()
- || tokenizer_cstr.is_null()
- || out.is_null()
- {
- set_last_error("paimon_tantivy_writer_new: null argument");
- return PaimonTantivyStatus::InvalidArgument;
- }
- let field_name = match unsafe { CStr::from_ptr(field_name_cstr) }.to_str() {
- Ok(s) => s,
- Err(e) => {
- set_last_error(format!("field_name not utf-8: {e}"));
- return PaimonTantivyStatus::InvalidArgument;
- }
- };
- let mode_str = match unsafe { CStr::from_ptr(mode_cstr) }.to_str() {
- Ok(s) => s,
- Err(e) => {
- set_last_error(format!("mode not utf-8: {e}"));
- return PaimonTantivyStatus::InvalidArgument;
- }
- };
- let dict_dir = match unsafe { CStr::from_ptr(dict_dir_cstr) }.to_str() {
- Ok(s) => s,
- Err(e) => {
- set_last_error(format!("dict_dir not utf-8: {e}"));
- return PaimonTantivyStatus::InvalidArgument;
- }
- };
- let tokenizer_name = match unsafe { CStr::from_ptr(tokenizer_cstr) }.to_str() {
- Ok(s) => s,
- Err(e) => {
- set_last_error(format!("tokenizer not utf-8: {e}"));
- return PaimonTantivyStatus::InvalidArgument;
- }
- };
- let mode = match TokenizeMode::parse(mode_str) {
- Some(m) => m,
- None => {
- set_last_error(format!(
- "unknown tokenize mode {mode_str:?}; expected one of mp/hmm/mix/full/query"
- ));
- return PaimonTantivyStatus::InvalidArgument;
- }
- };
- match PaimonTantivyWriter::new(
- field_name,
- mode,
- with_position,
- Path::new(dict_dir),
- tokenizer_name,
- ) {
- Ok(w) => {
- unsafe { *out = into_handle(w) };
- PaimonTantivyStatus::Ok
- }
- Err(e) => {
- // hmm-mode rejection bubbles through tokenizer construction.
- let unsupported = e.contains("'hmm' is not supported");
- set_last_error(e);
- if unsupported {
- PaimonTantivyStatus::Unsupported
- } else {
- PaimonTantivyStatus::InternalError
- }
- }
- }
-}
-
-/// Add a single document. `text` need not be NUL-terminated; treat as a slice
-/// of `text_len` UTF-8 bytes. Empty text (len=0) inserts an empty-text doc.
-/// `row_id` is the caller-supplied paimon row id (u64), stored in a fast field
-/// for retrieval by the reader.
-///
-/// SAFETY: `writer` must be a live handle from `writer_new`.
-#[no_mangle]
-pub unsafe extern "C" fn paimon_tantivy_writer_add(
- writer: *mut PaimonTantivyWriter,
- row_id: u64,
- text: *const c_char,
- text_len: usize,
-) -> PaimonTantivyStatus {
- let Some(w) = (unsafe { borrow_handle_mut::<PaimonTantivyWriter>(writer) }) else {
- set_last_error("paimon_tantivy_writer_add: null writer handle");
- return PaimonTantivyStatus::InvalidArgument;
- };
- if text.is_null() && text_len != 0 {
- set_last_error("text is null but len > 0");
- return PaimonTantivyStatus::InvalidArgument;
- }
- let text_str = if text_len == 0 {
- ""
- } else {
- let slice = unsafe { std::slice::from_raw_parts(text as *const u8, text_len) };
- match std::str::from_utf8(slice) {
- Ok(s) => s,
- Err(e) => {
- set_last_error(format!("text not utf-8: {e}"));
- return PaimonTantivyStatus::InvalidArgument;
- }
- }
- };
- match w.add(row_id, text_str) {
- Ok(()) => PaimonTantivyStatus::Ok,
- Err(e) => {
- set_last_error(e);
- PaimonTantivyStatus::InternalError
- }
- }
-}
-
-/// Commit + force-merge + stream archive bytes through `callbacks.write` in
-/// fixed-size chunks (see WRITER_STREAM_BUFFER_SIZE). May only be called once
-/// per writer; subsequent calls return InvalidArgument with
-/// last_error="writer already finished". Peak Rust RAM ≈ one buffer + entry
-/// metadata (independent of archive size).
-///
-/// The callback is invoked **serially** (not concurrently) within this call;
-/// C++ side can write directly to paimon OutputStream without locking.
-///
-/// SAFETY: `writer` must be a live handle; `out_row_count` non-null.
-/// `callbacks.write` / `callbacks.ctx` must remain valid for the duration of
-/// the call (callback is consumed in-place, not retained).
-#[no_mangle]
-pub unsafe extern "C" fn paimon_tantivy_writer_finish_streaming(
- writer: *mut PaimonTantivyWriter,
- callbacks: PaimonWriteCallbacks,
- out_row_count: *mut i64,
-) -> PaimonTantivyStatus {
- if out_row_count.is_null() {
- set_last_error("paimon_tantivy_writer_finish_streaming: null out_row_count");
- return PaimonTantivyStatus::InvalidArgument;
- }
- let Some(w) = (unsafe { borrow_handle_mut::<PaimonTantivyWriter>(writer) }) else {
- set_last_error("paimon_tantivy_writer_finish_streaming: null writer handle");
- return PaimonTantivyStatus::InvalidArgument;
- };
- match w.finish_streaming(&callbacks) {
- Ok(rows) => {
- unsafe { *out_row_count = rows };
- PaimonTantivyStatus::Ok
- }
- Err(e) => {
- let already_finished = e == "writer already finished";
- let io_err = e.starts_with("write callback rc=")
- || e.starts_with("open ")
- || e.starts_with("read ");
- set_last_error(e);
- if already_finished {
- PaimonTantivyStatus::InvalidArgument
- } else if io_err {
- PaimonTantivyStatus::IoError
- } else {
- PaimonTantivyStatus::InternalError
- }
- }
- }
-}
-
-/// Destroy a writer handle. Safe on null. Tmp dir is removed via Drop.
-#[no_mangle]
-pub unsafe extern "C" fn paimon_tantivy_writer_free(writer: *mut PaimonTantivyWriter) {
- unsafe { free_handle(writer) };
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
- use std::ffi::CString;
-
- /// Test dict dir for jieba; defaults to a non-existent path so jieba-rs uses
- /// its built-in dict (which is enough for these smoke tests).
- fn dict_dir_from_env() -> std::path::PathBuf {
- std::env::var("PAIMON_JIEBA_DICT_DIR")
- .map(std::path::PathBuf::from)
- .unwrap_or_else(|_| std::path::PathBuf::from("/tmp/nonexistent-dict"))
- }
-
- #[test]
- fn empty_field_name_rejected() {
- let err = PaimonTantivyWriter::new("", TokenizeMode::Mix, true, Path::new("/tmp/nx"), "paimon_jieba")
- .err()
- .unwrap();
- assert!(err.contains("field_name"), "got: {err}");
- }
-
- #[test]
- fn hmm_mode_rejected() {
- let err =
- PaimonTantivyWriter::new("f0", TokenizeMode::Hmm, true, Path::new("/tmp/nx"), "paimon_jieba")
- .err()
- .unwrap();
- assert!(err.contains("'hmm' is not supported"), "got: {err}");
- }
-
- #[test]
- fn create_add_finish_roundtrip() {
- let mut w =
- PaimonTantivyWriter::new("f0", TokenizeMode::Mix, true, &dict_dir_from_env(), "paimon_jieba").unwrap();
- w.add(0, "hello world").unwrap();
- w.add(1, "中国人民").unwrap();
- w.add(2, "").unwrap(); // empty doc
- let (rows, bytes) = w.finish().unwrap();
- assert_eq!(rows, 3);
- assert!(bytes.len() > 4);
-
- // Validate header (Java-compatible: BE int32 file_count, no version)
- let file_count = i32::from_be_bytes(bytes[0..4].try_into().unwrap());
- assert!(file_count > 0, "expected >0 packed files");
-
- // Walk entries (BE)
- let mut off: usize = 4;
- let mut names = Vec::new();
- for _ in 0..file_count {
- let nlen = i32::from_be_bytes(bytes[off..off + 4].try_into().unwrap()) as usize;
- off += 4;
- let name = std::str::from_utf8(&bytes[off..off + nlen]).unwrap().to_owned();
- off += nlen;
- let flen = i64::from_be_bytes(bytes[off..off + 8].try_into().unwrap()) as usize;
- off += 8;
- assert!(off + flen <= bytes.len(), "file {name} extends past buffer");
- off += flen;
- names.push(name);
- }
- assert_eq!(off, bytes.len(), "trailing bytes after pack");
- // tantivy must produce at least meta.json
- assert!(names.iter().any(|n| n == "meta.json"), "names={names:?}");
- }
-
- #[test]
- fn schema_field_names_are_fixed() {
- // Schema must be `row_id` (u64) + `text` (TEXT) regardless of caller's
- // field_name argument — matches paimon-java for cross-readability.
- let w =
- PaimonTantivyWriter::new("ignored_name", TokenizeMode::Mix, true, &dict_dir_from_env(), "paimon_jieba")
- .unwrap();
- let schema = w.index.schema();
- assert!(schema.get_field(PAIMON_ROW_ID_FIELD_NAME).is_ok(),
- "schema must have row_id field");
- assert!(schema.get_field(PAIMON_TEXT_FIELD_NAME).is_ok(),
- "schema must have text field");
- // Caller-supplied name must NOT appear
- assert!(schema.get_field("ignored_name").is_err(),
- "caller-supplied field_name must be ignored");
- }
-
- #[test]
- fn archive_uses_big_endian_no_version_header() {
- // Strong guard: header must be BE int32 file_count, NOT LE int32
- // version=1 + LE int32 file_count. Any regression to LE/version-header
- // would silently break paimon-java cross-read.
- let mut w =
- PaimonTantivyWriter::new("f0", TokenizeMode::Mix, true, &dict_dir_from_env(), "paimon_jieba").unwrap();
- w.add(0, "hello").unwrap();
- let (_, bytes) = w.finish().unwrap();
- let header_be = i32::from_be_bytes(bytes[0..4].try_into().unwrap());
- let header_le = i32::from_le_bytes(bytes[0..4].try_into().unwrap());
- // BE file_count is small (single-segment force-merge: ~6-7 files)
- assert!(header_be > 0 && header_be < 100,
- "expected sensible BE file_count, got BE={header_be} LE={header_le}");
- // LE-decoded header would be a huge number (e.g. 0x06000000), ensuring
- // we did NOT regress to the old LE+version layout.
- assert_ne!(header_be, header_le, "buffer must be BE-encoded");
- }
-
- #[test]
- fn multi_thread_writer_default() {
- // Schema stores row_id explicitly so we no longer enforce
- // single-threaded writer. Just verify many docs across threads land
- // correctly and force-merge collapses to a single segment.
- let mut w =
- PaimonTantivyWriter::new("f0", TokenizeMode::Mix, true, &dict_dir_from_env(), "paimon_jieba").unwrap();
- for i in 0..200u64 {
- w.add(i, &format!("row {i} apple banana")).unwrap();
- }
- let (rows, bytes) = w.finish().unwrap();
- assert_eq!(rows, 200);
- assert!(bytes.len() > 4);
- // After force-merge there must be exactly one meta.json + segment files.
- let file_count = i32::from_be_bytes(bytes[0..4].try_into().unwrap());
- assert!(file_count >= 2, "force-merged single segment needs ≥ 2 files (meta + segment), got {file_count}");
- }
-
- #[test]
- fn finish_twice_errors() {
- let mut w =
- PaimonTantivyWriter::new("f0", TokenizeMode::Mix, true, &dict_dir_from_env(), "paimon_jieba").unwrap();
- w.add(0, "hi").unwrap();
- let _ = w.finish().unwrap();
- let err = w.finish().err().unwrap();
- assert!(err.contains("already finished"), "got: {err}");
- }
-
- /// Mock collector for FFI streaming tests: push bytes into a Box<Vec<u8>>
- /// pointed to by `ctx`. (No Arc / atomic needed — test is single-threaded.)
- extern "C" fn mock_write_collect(ctx: *mut c_void, data: *const u8, len: usize) -> i32 {
- let vec = unsafe { &mut *(ctx as *mut Vec<u8>) };
- let slice = unsafe { std::slice::from_raw_parts(data, len) };
- vec.extend_from_slice(slice);
- 0
- }
-
- /// Mock that counts the largest single `write` call — sanity check that
- /// Rust streams in chunks bounded by the buffer (+ small header fields).
- extern "C" fn mock_write_max_chunk(
- ctx: *mut c_void,
- _data: *const u8,
- len: usize,
- ) -> i32 {
- let max = unsafe { &mut *(ctx as *mut usize) };
- if len > *max {
- *max = len;
- }
- 0
- }
-
- #[test]
- fn ffi_full_path_streaming() {
- unsafe {
- let field = CString::new("f0").unwrap();
- let mode = CString::new("mix").unwrap();
- let dict = CString::new(dict_dir_from_env().to_str().unwrap()).unwrap();
- let tokenizer = CString::new("paimon_jieba").unwrap();
- let mut handle: *mut PaimonTantivyWriter = std::ptr::null_mut();
- let st = paimon_tantivy_writer_new(
- field.as_ptr(),
- mode.as_ptr(),
- true,
- dict.as_ptr(),
- tokenizer.as_ptr(),
- &mut handle,
- );
- assert_eq!(st, PaimonTantivyStatus::Ok);
- assert!(!handle.is_null());
-
- let txt = "hello world";
- let st =
- paimon_tantivy_writer_add(handle, 42u64, txt.as_ptr() as *const c_char, txt.len());
- assert_eq!(st, PaimonTantivyStatus::Ok);
-
- // Streaming finish: collect bytes into a Vec<u8> via FFI callback
- let mut out: Vec<u8> = Vec::new();
- let cb = PaimonWriteCallbacks {
- ctx: &mut out as *mut _ as *mut c_void,
- write: mock_write_collect,
- };
- let mut rows: i64 = 0;
- let st = paimon_tantivy_writer_finish_streaming(handle, cb, &mut rows);
- assert_eq!(st, PaimonTantivyStatus::Ok);
- assert_eq!(rows, 1);
- // BE file_count at byte 0,> 0
- let file_count = i32::from_be_bytes(out[0..4].try_into().unwrap());
- assert!(file_count > 0);
-
- // double finish must error
- let mut out2: Vec<u8> = Vec::new();
- let cb2 = PaimonWriteCallbacks {
- ctx: &mut out2 as *mut _ as *mut c_void,
- write: mock_write_collect,
- };
- let mut rows2: i64 = 0;
- let st = paimon_tantivy_writer_finish_streaming(handle, cb2, &mut rows2);
- assert_eq!(st, PaimonTantivyStatus::InvalidArgument);
-
- paimon_tantivy_writer_free(handle);
- }
- }
-
- #[test]
- fn streaming_chunk_size_bounded_by_buffer() {
- // After force-merge, a 200-doc index still streams in chunks bounded by
- // WRITER_STREAM_BUFFER_SIZE (payload) or small header-field chunks.
- let mut w =
- PaimonTantivyWriter::new("f0", TokenizeMode::Mix, true, &dict_dir_from_env(), "paimon_jieba").unwrap();
- for i in 0..200u64 {
- w.add(i, &format!("row {i} apple banana")).unwrap();
- }
- let mut max_chunk: usize = 0;
- let cb = PaimonWriteCallbacks {
- ctx: &mut max_chunk as *mut _ as *mut c_void,
- write: mock_write_max_chunk,
- };
- let rows = w.finish_streaming(&cb).unwrap();
- assert_eq!(rows, 200);
- assert!(
- max_chunk <= WRITER_STREAM_BUFFER_SIZE,
- "streaming chunk size {} exceeded buffer {}",
- max_chunk,
- WRITER_STREAM_BUFFER_SIZE
- );
- }
-
- #[test]
- fn streaming_write_callback_error_propagates() {
- extern "C" fn always_fail(_ctx: *mut c_void, _data: *const u8, _len: usize) -> i32 {
- 7
- }
- let mut w =
- PaimonTantivyWriter::new("f0", TokenizeMode::Mix, true, &dict_dir_from_env(), "paimon_jieba").unwrap();
- w.add(0, "hello").unwrap();
- let cb = PaimonWriteCallbacks {
- ctx: std::ptr::null_mut(),
- write: always_fail,
- };
- let err = w.finish_streaming(&cb).unwrap_err();
- assert!(err.contains("write callback rc=7"), "got: {err}");
- }
-
- #[test]
- fn ffi_null_writer_invalid() {
- unsafe {
- let txt = "x";
- let st = paimon_tantivy_writer_add(
- std::ptr::null_mut(),
- 0u64,
- txt.as_ptr() as *const c_char,
- txt.len(),
- );
- assert_eq!(st, PaimonTantivyStatus::InvalidArgument);
- }
- }
-}
diff --git a/docs/source/building.rst b/docs/source/building.rst
index 5ca6ff45..93a4890b 100644
--- a/docs/source/building.rst
+++ b/docs/source/building.rst
@@ -185,7 +185,11 @@ boolean flags to ``cmake``.
* ``-DPAIMON_ENABLE_LUMINA=ON``: Support for the Lumina vector index. Requires
Linux ``x86_64``; see :ref:`cpp-building-platforms`.
* ``-DPAIMON_ENABLE_LUCENE=ON``: Support for Lucene full-text search indexes
-* ``-DPAIMON_ENABLE_TANTIVY=ON``: Enable the experimental Tantivy full-text index Rust FFI.
+* ``-DPAIMON_ENABLE_FULL_TEXT=ON``: Experimental full-text global index (index type
+ ``full-text``) backed by the ``paimon-full-text-index`` 0.1.0 Rust engine. It uses the same index
+ file format as Java Paimon. Requires Rust 1.90 or newer.
+ The engine is built from the Apache Paimon Full Text source release, which can be provided
+ through ``PAIMON_FULL_TEXT_URL``.
* ``-DPAIMON_ENABLE_REST=ON``: Support for the REST catalog
(``metastore=rest``), requires the libcurl and OpenSSL development packages.
diff --git a/docs/source/user_guide/global_index.rst b/docs/source/user_guide/global_index.rst
index 0cb853a0..040d483e 100644
--- a/docs/source/user_guide/global_index.rst
+++ b/docs/source/user_guide/global_index.rst
@@ -25,7 +25,9 @@ Paimon C++ supports the following global index types:
- **BTree Index**: An efficient index based on multi-level SST files for scalar column lookups.
- **Range Bitmap Index**: A range bitmap index optimized for range predicates on ordered scalar columns. Extends the bitmap approach by encoding value ordering, enabling efficient less-than, greater-than, and range conditions.
- **Lucene Index**: A full-text search index powered by Lucene++. Supports tokenized text search with multiple modes including match-all, match-any, phrase, prefix, and wildcard queries.
-- **Tantivy Full-Text Index (experimental)**: A Rust FFI-based full-text search index.
+- **Full-Text Index (experimental)**: A full-text search index powered by the native
+ ``paimon-full-text-index`` engine. It uses the same index file format as Java Paimon's
+ ``full-text`` index.
- **Vector Index (Lumina)**: An approximate nearest neighbor (ANN) index powered by Lumina for vector similarity search with configurable distance metrics.
Global indexes work on top of Data Evolution tables. To use global indexes, your table must have:
@@ -92,13 +94,106 @@ search modes including match-all, match-any, phrase, prefix, and wildcard querie
- **Description**: Specifies the directory containing Jieba dictionary files for Chinese text tokenization. At runtime, the system first checks this environment variable; if not set, it falls back to the compile-time ``JIEBA_TEST_DICT_DIR`` macro (only available in test builds). If neither is available, will fail with an error.
-Tantivy Full-Text Index (Experimental)
----------------------------------------
-
-The Tantivy full-text index is an experimental Rust FFI-based backend. Enable it at build time
-with ``-DPAIMON_ENABLE_TANTIVY=ON``. Its global index identifier is ``tantivy-fulltext``.
-Because this backend is experimental, validate its behavior and compatibility for your workload
-before production use.
+Full-Text Index (Experimental)
+------------------------------
+
+A full-text search index powered by the native
+`paimon-full-text-index <https://github.com/apache/paimon-full-text>`_ engine (0.1.0, built on
+Tantivy 0.26). It uses the same index type ``full-text``, option prefix and index file format as
+Java Paimon, so index files written by either side are meant to be readable by the other when both
+use the same engine version: a reader rejects index files written with another Tantivy version.
+Cross-reading is currently verified with index files written by the engine's Python binding
+(``paimon-ftindex``), which wraps the same engine as Java Paimon, but not yet with index files
+written by Java Paimon itself. Enable it at build time with ``-DPAIMON_ENABLE_FULL_TEXT=ON``; see
+:doc:`../building`.
+
+The indexed field must be a ``STRING``, ``CHAR`` or ``VARCHAR`` column. Null values are not
+indexed. Unlike in Java Paimon, values that contain NUL characters are rejected, because the native
+C API takes NUL-terminated strings; index option keys and values and queries must not contain NUL
+characters either. A writer that receives no rows writes no index file.
+
+This index replaces the former experimental ``tantivy-fulltext`` index, whose files cannot be read
+by it. Rebuild such indexes with the ``full-text`` index type.
+
+**Queries:**
+
+``FullTextSearch::query`` is passed to the engine unchanged and must be a JSON query, for example:
+
+- ``{"match": {"query": "paimon lake"}}``: rows that contain any of the analyzed terms. Add
+ ``"operator": "And"`` to require all terms.
+- ``{"match_phrase": {"query": "data lake"}}``: rows that contain the terms as a phrase. An
+ optional ``"slop"`` allows other terms between them.
+- ``{"boolean": {"must": [...], "should": [...], "must_not": [...]}}``: combines other queries.
+
+The query text is analyzed with the analyzer stored in the index file. ``search_type`` is ignored.
+A positive ``limit`` is required. The engine always ranks the matching rows by BM25 score and
+returns the ``limit`` rows with the highest scores; ``pre_filter`` restricts the rows that are
+ranked. ``min_score`` is applied to those rows afterwards and drops rows whose score is not greater
+than it, so a search can return fewer than ``limit`` rows. ``with_score`` only selects whether the
+scores are returned; scores are computed either way.
+
+**Configuration:**
+
+Table options with the ``full-text.`` prefix are passed to the engine with the prefix removed.
+They are only used when writing an index: the analyzer configuration is stored in every index file
+and readers use that copy.
+
+.. list-table::
+ :header-rows: 1
+ :widths: 30 10 60
+
+ * - Option
+ - Default
+ - Description
+ * - ``full-text.tokenizer``
+ - ``default``
+ - Tokenizer: ``default`` or ``simple`` (split on non-alphanumeric characters),
+ ``whitespace``, ``raw`` (no splitting), ``ngram`` (n-grams of the whole text) or ``jieba``
+ (Chinese segmentation with a built-in dictionary).
+ * - ``full-text.ngram.min-gram``
+ - 3
+ - Minimum n-gram length of the ``ngram`` tokenizer.
+ * - ``full-text.ngram.max-gram``
+ - 3
+ - Maximum n-gram length of the ``ngram`` tokenizer.
+ * - ``full-text.ngram.prefix-only``
+ - false
+ - Whether the ``ngram`` tokenizer only emits the n-grams that start at the beginning of the
+ text.
+ * - ``full-text.jieba.search-mode``
+ - true
+ - Whether the ``jieba`` tokenizer also emits the shorter words contained in long words.
+ * - ``full-text.jieba.ordinal-position``
+ - true
+ - Whether the ``jieba`` tokenizer assigns consecutive token positions.
+ * - ``full-text.lower-case``
+ - true
+ - Whether tokens are lower-cased.
+ * - ``full-text.max-token-length``
+ - 40
+ - Tokens of this many bytes or more are dropped.
+ * - ``full-text.ascii-folding``
+ - true
+ - Whether non-ASCII characters are folded to their ASCII equivalents.
+ * - ``full-text.stem``
+ - true
+ - Whether tokens are stemmed, so that ``run`` matches ``running``.
+ * - ``full-text.language``
+ - ``english``
+ - Language used for stemming and stop words.
+ * - ``full-text.remove-stop-words``
+ - true
+ - Whether the built-in stop words of ``full-text.language`` and ``full-text.stop-words`` are
+ removed.
+ * - ``full-text.stop-words``
+ - (empty)
+ - Additional stop words, separated by ``;``. Requires ``full-text.remove-stop-words=true``.
+ * - ``full-text.with-position``
+ - true
+ - Whether token positions are indexed. Phrase queries require positions.
+
+The prefix-stripped options are also stored as a flat JSON object in the metadata of each index
+file, as Java Paimon does.
Vector Index (Lumina)
---------------------
diff --git a/include/paimon/predicate/full_text_search.h b/include/paimon/predicate/full_text_search.h
index 3e0676c5..ba3af6d8 100644
--- a/include/paimon/predicate/full_text_search.h
+++ b/include/paimon/predicate/full_text_search.h
@@ -69,6 +69,9 @@ struct PAIMON_EXPORT FullTextSearch {
/// Maximum number of documents to return. Purely a truncation switch,
/// orthogonal to `with_score`: set `with_score = true` to get relevance
/// scores; a non-empty `limit` does not by itself imply scoring.
+ ///
+ /// The `full-text` index requires a positive `limit` and always returns at most `limit` rows
+ /// with the highest scores.
std::optional<int32_t> limit;
/// The query string to search for. The interpretation depends on search_type:
///
@@ -88,6 +91,10 @@ struct PAIMON_EXPORT FullTextSearch {
/// preserved. Both the original pattern and an alternative with each ASCII alphanumeric
/// fragment lowercased are matched, covering pure ASCII and mixed ASCII/non-ASCII terms.
///
+ /// - For the `full-text` index, `search_type` is ignored and the query is passed to the
+ /// native engine unchanged, so it must be a JSON DSL query such as
+ /// `{"match":{"query":"paimon lake"}}`.
+ ///
/// @note Analyzer consistency between indexing and querying is critical for correctness.
std::string query;
/// Type of search to perform.
@@ -105,16 +112,25 @@ struct PAIMON_EXPORT FullTextSearch {
///
/// For plain `LIMIT N` without ORDER BY (the common case when an online
/// engine, e.g. StarRocks, pushes down a predicate) set `with_score=false,
- /// limit=N` — the unscored fast path. For top-N by relevance use
- /// `with_score=true, limit=N` and drop the scores in the caller if unneeded.
+ /// limit=N` — the unscored fast path, where the backend provides one. For top-N by relevance
+ /// use `with_score=true, limit=N` and drop the scores in the caller if unneeded.
///
- /// Default is `false` to avoid score computation overhead for callers that don't need it.
+ /// The built-in indexes deviate from this matrix:
+ /// - `lucene` ignores `with_score`: a search with `limit` returns the top `limit` rows with
+ /// their scores, and a search without `limit` returns all matching rows without scores.
+ /// - `full-text` requires `limit` and always ranks the matching rows by BM25 score, keeping the
+ /// top `limit` rows. `with_score` only selects whether their scores are returned.
+ ///
+ /// Default is `false`, which avoids score computation on backends with an unscored path.
bool with_score = false;
/// Minimum relevance-score threshold (exclusive); results with score ≤ this value are
/// excluded. The score is whatever the backend's similarity produces (e.g. BM25 for
- /// tantivy, classic TF-IDF for lucene), so a threshold is not directly comparable across
- /// backends. Only meaningful when scoring is active (`with_score = true` or `limit` set);
- /// applied before truncation so low-score documents never occupy limit slots.
+ /// full-text, classic TF-IDF for lucene), so a threshold is not directly comparable across
+ /// backends.
+ ///
+ /// Only the `full-text` index supports it; the `lucene` index rejects a search that sets it.
+ /// The `full-text` index applies it to the top `limit` rows returned by the engine, so a
+ /// search can return fewer than `limit` rows.
/// Default is nullopt (no threshold filtering).
std::optional<float> min_score;
};
diff --git a/src/paimon/common/utils/file_type_test.cpp b/src/paimon/common/utils/file_type_test.cpp
index 9f18a70c..9d61af72 100644
--- a/src/paimon/common/utils/file_type_test.cpp
+++ b/src/paimon/common/utils/file_type_test.cpp
@@ -59,9 +59,8 @@ TEST(FileTypeTest, TestIndexTypes) {
ASSERT_EQ(
FileTypeUtils::Classify("dfs://cluster/db/index/lumina-vector-ann-global-index-a1b2.index"),
FileType::kGlobalIndex);
- ASSERT_EQ(
- FileTypeUtils::Classify("dfs://cluster/db/index/tantivy-fulltext-global-index-a1b2.index"),
- FileType::kGlobalIndex);
+ ASSERT_EQ(FileTypeUtils::Classify("dfs://cluster/db/index/full-text-global-index-a1b2.index"),
+ FileType::kGlobalIndex);
ASSERT_EQ(FileTypeUtils::Classify("dfs://cluster/db/p=1/bucket-0/index-abcdef-1"),
FileType::kBucketIndex);
}
diff --git a/src/paimon/global_index/full_text/CMakeLists.txt b/src/paimon/global_index/full_text/CMakeLists.txt
new file mode 100644
index 00000000..3fbf5369
--- /dev/null
+++ b/src/paimon/global_index/full_text/CMakeLists.txt
@@ -0,0 +1,56 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if(PAIMON_ENABLE_FULL_TEXT)
+ set(PAIMON_FULL_TEXT_INDEX
+ full_text_ffi_utils.cpp
+ full_text_global_index.cpp
+ full_text_global_index_factory.cpp
+ full_text_global_index_reader.cpp
+ full_text_global_index_writer.cpp)
+
+ add_paimon_lib(paimon_full_text_index
+ SOURCES
+ ${PAIMON_FULL_TEXT_INDEX}
+ DEPENDENCIES
+ paimon_shared
+ paimon_ftindex_ffi
+ STATIC_LINK_LIBS
+ arrow
+ fmt
+ paimon_ftindex_ffi
+ Threads::Threads
+ SHARED_LINK_LIBS
+ paimon_shared
+ paimon_ftindex_ffi
+ SHARED_LINK_FLAGS
+ ${PAIMON_VERSION_SCRIPT_FLAGS})
+
+ target_link_libraries(paimon_full_text_index_objlib PUBLIC paimon_ftindex_ffi)
+
+ if(PAIMON_BUILD_TESTS)
+ add_paimon_test(full_text_index_test
+ SOURCES
+ full_text_global_index_test.cpp
+ STATIC_LINK_LIBS
+ paimon_shared
+ test_utils_static
+ ${PAIMON_FULL_TEXT_INDEX_STATIC_LINK_LIBS}
+ ${PAIMON_LOCAL_FILE_SYSTEM_STATIC_LINK_LIBS}
+ paimon_ftindex_ffi
+ ${GTEST_LINK_TOOLCHAIN})
+ endif()
+endif()
diff --git a/src/paimon/global_index/tantivy/tantivy_ffi_log.h b/src/paimon/global_index/full_text/full_text_defs.h
similarity index 62%
rename from src/paimon/global_index/tantivy/tantivy_ffi_log.h
rename to src/paimon/global_index/full_text/full_text_defs.h
index 7fad3299..ebd8ea75 100644
--- a/src/paimon/global_index/tantivy/tantivy_ffi_log.h
+++ b/src/paimon/global_index/full_text/full_text_defs.h
@@ -16,16 +16,19 @@
* specific language governing permissions and limitations
* under the License.
*/
+
#pragma once
-namespace paimon::tantivy {
+namespace paimon::full_text {
+
+/// Index type of the full-text global index, shared with Java Paimon
+/// (`NativeFullTextGlobalIndexerFactory#IDENTIFIER`).
+static inline const char kIdentifier[] = "full-text";
-/// Install the Rust -> C++ log callback. Idempotent; only the last caller's
-/// callback is active. Threading: C callback runs on tantivy worker threads;
-/// our adapter must be thread-safe (it routes to glog which is).
-void InstallTantivyLogBridge();
+/// Prefix of the index file names, matching Java `NativeFullTextGlobalIndexWriter`.
+static inline const char kFileNamePrefix[] = "full-text";
-/// Uninstall (revert to Rust stderr). Mostly useful for tests.
-void UninstallTantivyLogBridge();
+/// Prefix of the table options of the full-text index.
+static inline const char kOptionKeyPrefix[] = "full-text.";
-} // namespace paimon::tantivy
+} // namespace paimon::full_text
diff --git a/src/paimon/global_index/full_text/full_text_ffi_utils.cpp b/src/paimon/global_index/full_text/full_text_ffi_utils.cpp
new file mode 100644
index 00000000..eafe35cb
--- /dev/null
+++ b/src/paimon/global_index/full_text/full_text_ffi_utils.cpp
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/global_index/full_text/full_text_ffi_utils.h"
+
+#include "fmt/format.h"
+#include "paimon/common/utils/string_utils.h"
+
+namespace paimon::full_text {
+
+namespace {
+constexpr char kNativeIoErrorPrefix[] = "io error:";
+} // namespace
+
+Status LastFtindexError(const std::string& action, const Status& callback_error) {
+ if (!callback_error.ok()) {
+ return callback_error.WithMessage("failed to ", action, ": ", callback_error.message());
+ }
+ const char* native_error = paimon_ftindex_last_error();
+ std::string message = native_error != nullptr ? native_error : "unknown native error";
+ std::string full_message = fmt::format("failed to {}: {}", action, message);
+ if (StringUtils::StartsWith(message, kNativeIoErrorPrefix)) {
+ return Status::IOError(full_message);
+ }
+ return Status::Invalid(full_message);
+}
+
+} // namespace paimon::full_text
diff --git a/src/paimon/global_index/full_text/full_text_ffi_utils.h b/src/paimon/global_index/full_text/full_text_ffi_utils.h
new file mode 100644
index 00000000..e8dacb29
--- /dev/null
+++ b/src/paimon/global_index/full_text/full_text_ffi_utils.h
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <memory>
+#include <string>
+
+#include "paimon/status.h"
+#include "paimon_ftindex.h" // NOLINT(build/include_subdir)
+
+namespace paimon::full_text {
+
+struct FtindexWriterDeleter {
+ void operator()(PaimonFtindexWriterHandle* writer) const {
+ paimon_ftindex_writer_free(writer);
+ }
+};
+
+struct FtindexReaderDeleter {
+ void operator()(PaimonFtindexReaderHandle* reader) const {
+ paimon_ftindex_reader_free(reader);
+ }
+};
+
+using FtindexWriterPtr = std::unique_ptr<PaimonFtindexWriterHandle, FtindexWriterDeleter>;
+using FtindexReaderPtr = std::unique_ptr<PaimonFtindexReaderHandle, FtindexReaderDeleter>;
+
+/// Converts the error of the last failed native call into a `Status`, preferring `callback_error`,
+/// the error raised by a file system callback during that call. The native library keeps its last
+/// error per thread, so this must be called on the failing thread before any other native call.
+Status LastFtindexError(const std::string& action, const Status& callback_error = Status::OK());
+
+} // namespace paimon::full_text
diff --git a/src/paimon/global_index/tantivy/tantivy_global_index.cpp b/src/paimon/global_index/full_text/full_text_global_index.cpp
similarity index 57%
rename from src/paimon/global_index/tantivy/tantivy_global_index.cpp
rename to src/paimon/global_index/full_text/full_text_global_index.cpp
index f281c035..93c3dd80 100644
--- a/src/paimon/global_index/tantivy/tantivy_global_index.cpp
+++ b/src/paimon/global_index/full_text/full_text_global_index.cpp
@@ -17,15 +17,18 @@
* under the License.
*/
-#include "paimon/global_index/tantivy/tantivy_global_index.h"
+#include "paimon/global_index/full_text/full_text_global_index.h"
#include "arrow/c/bridge.h"
+#include "arrow/type.h"
#include "fmt/format.h"
+#include "paimon/common/utils/arrow/status_utils.h"
#include "paimon/common/utils/options_utils.h"
-#include "paimon/global_index/tantivy/tantivy_global_index_reader.h"
-#include "paimon/global_index/tantivy/tantivy_global_index_writer.h"
+#include "paimon/global_index/full_text/full_text_defs.h"
+#include "paimon/global_index/full_text/full_text_global_index_reader.h"
+#include "paimon/global_index/full_text/full_text_global_index_writer.h"
-namespace paimon::tantivy {
+namespace paimon::full_text {
#define CHECK_NOT_NULL(pointer, error_msg) \
do { \
@@ -34,14 +37,14 @@ namespace paimon::tantivy {
} \
} while (0)
-TantivyGlobalIndex::TantivyGlobalIndex(const std::map<std::string, std::string>& options)
+FullTextGlobalIndex::FullTextGlobalIndex(const std::map<std::string, std::string>& options)
: options_(OptionsUtils::FetchOptionsWithPrefix(kOptionKeyPrefix, options)) {}
-Result<std::optional<std::vector<std::string>>> TantivyGlobalIndex::GetExtraFieldNames() const {
+Result<std::optional<std::vector<std::string>>> FullTextGlobalIndex::GetExtraFieldNames() const {
return std::optional<std::vector<std::string>>(std::nullopt);
}
-Result<std::shared_ptr<GlobalIndexWriter>> TantivyGlobalIndex::CreateWriter(
+Result<std::shared_ptr<GlobalIndexWriter>> FullTextGlobalIndex::CreateWriter(
const std::string& field_name, ::ArrowSchema* arrow_schema,
const std::shared_ptr<GlobalIndexFileWriter>& file_writer,
const std::shared_ptr<MemoryPool>& pool) const {
@@ -49,36 +52,47 @@ Result<std::shared_ptr<GlobalIndexWriter>> TantivyGlobalIndex::CreateWriter(
arrow::ImportType(arrow_schema));
auto struct_type = std::dynamic_pointer_cast<arrow::StructType>(arrow_type);
CHECK_NOT_NULL(struct_type,
- "arrow schema must be struct type when create TantivyGlobalIndexWriter");
+ "arrow schema must be struct type when create FullTextGlobalIndexWriter");
auto index_field = struct_type->GetFieldByName(field_name);
CHECK_NOT_NULL(
index_field,
- fmt::format("field {} not exist in arrow schema when create TantivyGlobalIndexWriter",
+ fmt::format("field {} not exist in arrow schema when create FullTextGlobalIndexWriter",
field_name));
+ // STRING, CHAR and VARCHAR are all mapped to the arrow utf8 type.
if (index_field->type()->id() != arrow::Type::type::STRING) {
- return Status::Invalid("field type must be string");
+ return Status::Invalid(
+ fmt::format("full-text index only supports string fields, field {} is {}", field_name,
+ index_field->type()->ToString()));
}
- return TantivyGlobalIndexWriter::Create(field_name, arrow_type, file_writer, options_, pool);
+ if (!file_writer) {
+ return Status::Invalid(
+ "file writer must not be null when create FullTextGlobalIndexWriter");
+ }
+ return FullTextGlobalIndexWriter::Create(field_name, arrow_type, file_writer, options_, pool);
}
-Result<std::shared_ptr<GlobalIndexReader>> TantivyGlobalIndex::CreateReader(
+Result<std::shared_ptr<GlobalIndexReader>> FullTextGlobalIndex::CreateReader(
::ArrowSchema* c_arrow_schema, const std::shared_ptr<GlobalIndexFileReader>& file_reader,
const std::vector<GlobalIndexIOMeta>& files, const std::shared_ptr<MemoryPool>& pool) const {
PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Schema> arrow_schema,
arrow::ImportSchema(c_arrow_schema));
if (files.size() != 1) {
return Status::Invalid(fmt::format(
- "tantivy index only has one index file per shard, now num: {}", files.size()));
+ "full-text index expects exactly one index file per shard, now num: {}", files.size()));
}
if (arrow_schema->num_fields() != 1) {
- return Status::Invalid("TantivyGlobalIndex now only support one field");
+ return Status::Invalid(fmt::format("full-text index only supports one field, now num: {}",
+ arrow_schema->num_fields()));
}
auto index_field = arrow_schema->field(0);
if (index_field->type()->id() != arrow::Type::type::STRING) {
- return Status::Invalid("field type must be string");
+ return Status::Invalid(
+ fmt::format("full-text index only supports string fields, field {} is {}",
+ index_field->name(), index_field->type()->ToString()));
}
- return TantivyGlobalIndexReader::Create(index_field->name(), files[0], file_reader, options_,
- pool);
+ // The analyzer configuration is embedded in the index file, so neither the table options nor
+ // `GlobalIndexIOMeta::metadata` are needed to read it.
+ return FullTextGlobalIndexReader::Create(files[0], file_reader, pool);
}
-} // namespace paimon::tantivy
+} // namespace paimon::full_text
diff --git a/src/paimon/global_index/tantivy/tantivy_global_index.h b/src/paimon/global_index/full_text/full_text_global_index.h
similarity index 74%
rename from src/paimon/global_index/tantivy/tantivy_global_index.h
rename to src/paimon/global_index/full_text/full_text_global_index.h
index 4b7b944f..2bc12015 100644
--- a/src/paimon/global_index/tantivy/tantivy_global_index.h
+++ b/src/paimon/global_index/full_text/full_text_global_index.h
@@ -25,18 +25,17 @@
#include <string>
#include <vector>
-#include "arrow/type.h"
#include "paimon/global_index/global_indexer.h"
-#include "paimon/global_index/tantivy/tantivy_defs.h"
-namespace paimon::tantivy {
+namespace paimon::full_text {
-/// `GlobalIndexer` implementation backed by tantivy-fulltext. Counterpart to
-/// `LuceneGlobalIndex`; the two coexist and are NOT cross-readable. Selection
-/// between them happens at the factory layer via the `index_type` identifier.
-class TantivyGlobalIndex : public GlobalIndexer {
+/// `GlobalIndexer` backed by the native `paimon-full-text-index` engine, aligned with Java
+/// `NativeFullTextGlobalIndexer`. Index files are readable by Java Paimon and vice versa.
+class FullTextGlobalIndex : public GlobalIndexer {
public:
- explicit TantivyGlobalIndex(const std::map<std::string, std::string>& options);
+ /// @param options Table options. Only the `full-text.`-prefixed ones are kept, with the
+ /// prefix stripped once.
+ explicit FullTextGlobalIndex(const std::map<std::string, std::string>& options);
Result<std::optional<std::vector<std::string>>> GetExtraFieldNames() const override;
@@ -51,8 +50,7 @@ class TantivyGlobalIndex : public GlobalIndexer {
const std::shared_ptr<MemoryPool>& pool) const override;
private:
- /// Options after the `tantivy-fulltext.` prefix has been stripped.
std::map<std::string, std::string> options_;
};
-} // namespace paimon::tantivy
+} // namespace paimon::full_text
diff --git a/src/paimon/global_index/tantivy/tantivy_defs.cpp b/src/paimon/global_index/full_text/full_text_global_index_factory.cpp
similarity index 57%
rename from src/paimon/global_index/tantivy/tantivy_defs.cpp
rename to src/paimon/global_index/full_text/full_text_global_index_factory.cpp
index 142d727e..13fa0006 100644
--- a/src/paimon/global_index/tantivy/tantivy_defs.cpp
+++ b/src/paimon/global_index/full_text/full_text_global_index_factory.cpp
@@ -17,21 +17,24 @@
* under the License.
*/
-#include "paimon/global_index/tantivy/tantivy_defs.h"
+#include "paimon/global_index/full_text/full_text_global_index_factory.h"
-#include <cstdlib>
+#include <map>
+#include <memory>
+#include <string>
+#include <utility>
-namespace paimon::tantivy {
+#include "paimon/global_index/full_text/full_text_global_index.h"
-std::optional<std::string> GetJiebaDictionaryDirFromEnv() {
- const char* env_dir = std::getenv(kJiebaDictDirEnv);
- if (env_dir != nullptr && *env_dir != '\0') {
- return std::string(env_dir);
- }
-#ifdef JIEBA_TEST_DICT_DIR
- return std::string(JIEBA_TEST_DICT_DIR);
-#endif
- return std::nullopt;
+namespace paimon::full_text {
+
+const char FullTextGlobalIndexFactory::IDENTIFIER[] = "full-text-global";
+
+Result<std::unique_ptr<GlobalIndexer>> FullTextGlobalIndexFactory::Create(
+ const std::map<std::string, std::string>& options) const {
+ return std::make_unique<FullTextGlobalIndex>(options);
}
-} // namespace paimon::tantivy
+REGISTER_PAIMON_FACTORY(FullTextGlobalIndexFactory);
+
+} // namespace paimon::full_text
diff --git a/src/paimon/global_index/tantivy/tantivy_global_index_factory.h b/src/paimon/global_index/full_text/full_text_global_index_factory.h
similarity index 73%
rename from src/paimon/global_index/tantivy/tantivy_global_index_factory.h
rename to src/paimon/global_index/full_text/full_text_global_index_factory.h
index e9f9863d..d97c1a24 100644
--- a/src/paimon/global_index/tantivy/tantivy_global_index_factory.h
+++ b/src/paimon/global_index/full_text/full_text_global_index_factory.h
@@ -26,14 +26,12 @@
#include "paimon/global_index/global_indexer.h"
#include "paimon/global_index/global_indexer_factory.h"
-namespace paimon::tantivy {
-
-/// Factory for creating tantivy-fulltext global indexers. Registered into
-/// `FactoryCreator` via `REGISTER_PAIMON_FACTORY` so it is selectable
-/// alongside `lucene-fts-global` by passing `index_type = "tantivy-fulltext"`
-/// (the suffix `-global` is appended automatically by
-/// `GlobalIndexerFactory::Get`).
-class TantivyGlobalIndexFactory : public GlobalIndexerFactory {
+namespace paimon::full_text {
+
+/// Factory for creating full-text global indexers, selected by the index type `full-text` (the
+/// `-global` suffix is appended by `GlobalIndexerFactory::Get`), as in Java
+/// `NativeFullTextGlobalIndexerFactory`.
+class FullTextGlobalIndexFactory : public GlobalIndexerFactory {
public:
static const char IDENTIFIER[];
@@ -45,4 +43,4 @@ class TantivyGlobalIndexFactory : public GlobalIndexerFactory {
const std::map<std::string, std::string>& options) const override;
};
-} // namespace paimon::tantivy
+} // namespace paimon::full_text
diff --git a/src/paimon/global_index/full_text/full_text_global_index_reader.cpp b/src/paimon/global_index/full_text/full_text_global_index_reader.cpp
new file mode 100644
index 00000000..4c6e6152
--- /dev/null
+++ b/src/paimon/global_index/full_text/full_text_global_index_reader.cpp
@@ -0,0 +1,218 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/global_index/full_text/full_text_global_index_reader.h"
+
+#include <map>
+#include <optional>
+#include <utility>
+
+#include "fmt/format.h"
+#include "paimon/fs/file_system.h"
+#include "paimon/global_index/bitmap_global_index_result.h"
+#include "paimon/global_index/bitmap_scored_global_index_result.h"
+#include "paimon/memory/bytes.h"
+#include "paimon/utils/roaring_bitmap64.h"
+
+namespace paimon::full_text {
+
+struct FullTextInputContext {
+ std::unique_ptr<InputStream> stream;
+ /// Serializes reads, like Java: the engine may read concurrently, but some `InputStream`
+ /// implementations are not safe for concurrent positional reads.
+ std::mutex mutex;
+};
+
+namespace {
+
+/// Error raised by the input stream in the last failed read on this thread, reported instead of the
+/// generic native error. It is cleared before each native call that reads the index.
+thread_local Status last_read_error;
+
+int PreadFromInputStream(void* ctx, uint64_t pos, uint8_t* buf, size_t len) {
+ auto* input = static_cast<FullTextInputContext*>(ctx);
+ std::lock_guard<std::mutex> lock(input->mutex);
+ size_t total = 0;
+ while (total < len) {
+ Result<int64_t> result = input->stream->Read(reinterpret_cast<char*>(buf + total),
+ static_cast<int64_t>(len - total),
+ static_cast<int64_t>(pos + total));
+ if (!result.ok()) {
+ last_read_error = result.status();
+ return -1;
+ }
+ if (result.value() <= 0) {
+ last_read_error = Status::IOError(
+ fmt::format("unexpected end of full-text index file at offset {}", pos + total));
+ return -1;
+ }
+ total += static_cast<size_t>(result.value());
+ }
+ return 0;
+}
+
+Status TakeLastError(const std::string& action) {
+ return LastFtindexError(action, std::exchange(last_read_error, Status::OK()));
+}
+
+/// Converts the engine output, ordered by descending score, into a result ordered by row id.
+std::shared_ptr<GlobalIndexResult> ToGlobalIndexResult(const std::vector<int64_t>& row_ids,
+ const std::vector<float>& scores,
+ bool with_score,
+ const std::optional<float>& min_score) {
+ auto above_min_score = [&](size_t i) { return !min_score || scores[i] > min_score.value(); };
+ if (!with_score) {
+ RoaringBitmap64 bitmap;
+ for (size_t i = 0; i < row_ids.size(); ++i) {
+ if (above_min_score(i)) {
+ bitmap.Add(row_ids[i]);
+ }
+ }
+ return std::make_shared<BitmapGlobalIndexResult>(
+ [bitmap = std::move(bitmap)]() -> Result<RoaringBitmap64> { return bitmap; });
+ }
+
+ // As in Java, a row id that was indexed more than once keeps the score of its last hit.
+ std::map<int64_t, float> scores_by_row_id;
+ for (size_t i = 0; i < row_ids.size(); ++i) {
+ if (above_min_score(i)) {
+ scores_by_row_id[row_ids[i]] = scores[i];
+ }
+ }
+ RoaringBitmap64 bitmap;
+ std::vector<float> sorted_scores;
+ sorted_scores.reserve(scores_by_row_id.size());
+ for (const auto& [row_id, score] : scores_by_row_id) {
+ bitmap.Add(row_id);
+ sorted_scores.push_back(score);
+ }
+ return std::make_shared<BitmapScoredGlobalIndexResult>(std::move(bitmap),
+ std::move(sorted_scores));
+}
+
+} // namespace
+
+Result<std::shared_ptr<FullTextGlobalIndexReader>> FullTextGlobalIndexReader::Create(
+ const GlobalIndexIOMeta& io_meta, const std::shared_ptr<GlobalIndexFileReader>& file_reader,
+ const std::shared_ptr<MemoryPool>& pool) {
+ if (!file_reader) {
+ return Status::Invalid(
+ "file reader must not be null when create FullTextGlobalIndexReader");
+ }
+ return std::shared_ptr<FullTextGlobalIndexReader>(
+ new FullTextGlobalIndexReader(io_meta, file_reader, pool));
+}
+
+FullTextGlobalIndexReader::FullTextGlobalIndexReader(
+ const GlobalIndexIOMeta& io_meta, const std::shared_ptr<GlobalIndexFileReader>& file_reader,
+ const std::shared_ptr<MemoryPool>& pool)
+ : io_meta_(io_meta),
+ file_reader_(file_reader),
+ pool_(pool ? pool : GetDefaultPool()),
+ logger_(Logger::GetLogger("FullTextGlobalIndexReader")) {}
+
+FullTextGlobalIndexReader::~FullTextGlobalIndexReader() {
+ // Free the native reader before closing the stream it reads through.
+ reader_.reset();
+ if (input_) {
+ CloseInputStream(input_->stream.get());
+ }
+}
+
+void FullTextGlobalIndexReader::CloseInputStream(InputStream* stream) const {
+ Status status = stream->Close();
+ if (!status.ok()) {
+ PAIMON_LOG_WARN(logger_, "failed to close full-text index file %s: %s",
+ io_meta_.file_path.c_str(), status.ToString().c_str());
+ }
+}
+
+Result<PaimonFtindexReaderHandle*> FullTextGlobalIndexReader::GetOrOpenReader() {
+ std::lock_guard<std::mutex> lock(open_mutex_);
+ if (reader_) {
+ return reader_.get();
+ }
+ PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<InputStream> stream,
+ file_reader_->GetInputStream(io_meta_.file_path));
+ auto input = std::make_unique<FullTextInputContext>();
+ input->stream = std::move(stream);
+ PaimonFtindexInputFile input_file{static_cast<void*>(input.get()), &PreadFromInputStream};
+ last_read_error = Status::OK();
+ FtindexReaderPtr reader(paimon_ftindex_reader_open(input_file));
+ if (!reader) {
+ Status status = TakeLastError(fmt::format("open full-text index {}", io_meta_.file_path));
+ CloseInputStream(input->stream.get());
+ return status;
+ }
+ input_ = std::move(input);
+ reader_ = std::move(reader);
+ return reader_.get();
+}
+
+Result<std::shared_ptr<GlobalIndexResult>> FullTextGlobalIndexReader::VisitFullTextSearch(
+ const std::shared_ptr<FullTextSearch>& full_text_search) {
+ if (!full_text_search) {
+ return Status::Invalid("VisitFullTextSearch: null FullTextSearch pointer");
+ }
+ if (!full_text_search->limit || full_text_search->limit.value() <= 0) {
+ return Status::Invalid("full-text index search requires a positive limit");
+ }
+ const std::string& query = full_text_search->query;
+ if (query.find('\0') != std::string::npos) {
+ return Status::Invalid("full-text index query must not contain NUL characters");
+ }
+
+ PAIMON_UNIQUE_PTR<Bytes> filter_bytes;
+ if (full_text_search->pre_filter) {
+ const RoaringBitmap64& pre_filter = full_text_search->pre_filter.value();
+ if (pre_filter.IsEmpty()) {
+ return ToGlobalIndexResult({}, {}, full_text_search->with_score,
+ full_text_search->min_score);
+ }
+ // Serialize() optimizes the bitmap in place and the same search may be visited by
+ // concurrent readers, so serialize a copy.
+ RoaringBitmap64 filter = pre_filter;
+ filter_bytes = filter.Serialize(pool_.get());
+ }
+
+ PAIMON_ASSIGN_OR_RAISE(PaimonFtindexReaderHandle * reader, GetOrOpenReader());
+ auto limit = static_cast<size_t>(full_text_search->limit.value());
+ std::vector<int64_t> row_ids(limit);
+ std::vector<float> scores(limit);
+ size_t result_len = 0;
+ int status = 0;
+ last_read_error = Status::OK();
+ if (filter_bytes) {
+ status = paimon_ftindex_reader_search_with_roaring_filter(
+ reader, query.c_str(), limit, reinterpret_cast<const uint8_t*>(filter_bytes->data()),
+ filter_bytes->size(), row_ids.data(), scores.data(), limit, &result_len);
+ } else {
+ status = paimon_ftindex_reader_search(reader, query.c_str(), limit, row_ids.data(),
+ scores.data(), limit, &result_len);
+ }
+ if (status != 0) {
+ return TakeLastError(fmt::format("search full-text index {}", io_meta_.file_path));
+ }
+ row_ids.resize(result_len);
+ scores.resize(result_len);
+ return ToGlobalIndexResult(row_ids, scores, full_text_search->with_score,
+ full_text_search->min_score);
+}
+
+} // namespace paimon::full_text
diff --git a/src/paimon/global_index/tantivy/tantivy_global_index_reader.h b/src/paimon/global_index/full_text/full_text_global_index_reader.h
similarity index 51%
rename from src/paimon/global_index/tantivy/tantivy_global_index_reader.h
rename to src/paimon/global_index/full_text/full_text_global_index_reader.h
index fcfdc455..acb965ba 100644
--- a/src/paimon/global_index/tantivy/tantivy_global_index_reader.h
+++ b/src/paimon/global_index/full_text/full_text_global_index_reader.h
@@ -19,94 +19,95 @@
#pragma once
-#include <map>
#include <memory>
+#include <mutex>
#include <string>
+#include <vector>
-#include "paimon/global_index/bitmap_scored_global_index_result.h"
+#include "paimon/global_index/full_text/full_text_defs.h"
+#include "paimon/global_index/full_text/full_text_ffi_utils.h"
#include "paimon/global_index/global_index_io_meta.h"
#include "paimon/global_index/global_index_reader.h"
#include "paimon/global_index/io/global_index_file_reader.h"
-#include "paimon/global_index/tantivy/tantivy_defs.h"
-#include "paimon/global_index/tantivy/tantivy_ffi_handle.h"
+#include "paimon/logging.h"
#include "paimon/memory/memory_pool.h"
#include "paimon/predicate/full_text_search.h"
-namespace paimon::tantivy {
+namespace paimon::full_text {
-/// Tantivy-backed implementation of `GlobalIndexReader`.
-///
-/// Mirrors LuceneGlobalIndexReader's surface but delegates query construction
-/// + execution into Rust over FFI. Supports the 5 FullTextSearch SearchTypes
-/// (MATCH_ALL, MATCH_ANY, PHRASE, PREFIX, WILDCARD), optionally with limit and
-/// pre_filter.
+/// Context of the native positional-read callback.
+struct FullTextInputContext;
+
+/// Reads one full-text index archive through the native `paimon-full-text-index` engine,
+/// aligned with Java `NativeFullTextGlobalIndexReader`. The archive is opened lazily on the first
+/// search and carries its own analyzer configuration.
///
-/// All non-FullTextSearch visit methods return nullptr (matches
-/// LuceneGlobalIndexReader): the FTS index has no contribution for non-FTS
-/// predicates, framework treats nullptr as "no filter constraint".
-class TantivyGlobalIndexReader : public GlobalIndexReader {
+/// `FullTextSearch::query` is passed to the engine unchanged, so it must be a JSON DSL query such
+/// as `{"match":{"query":"paimon lake"}}`, and a positive `limit` is required. Other predicates are
+/// not supported and return nullptr.
+class FullTextGlobalIndexReader : public GlobalIndexReader {
public:
- static Result<std::shared_ptr<TantivyGlobalIndexReader>> Create(
- const std::string& field_name, const GlobalIndexIOMeta& io_meta,
- const std::shared_ptr<GlobalIndexFileReader>& file_reader,
- const std::map<std::string, std::string>& options, const std::shared_ptr<MemoryPool>& pool);
+ static Result<std::shared_ptr<FullTextGlobalIndexReader>> Create(
+ const GlobalIndexIOMeta& io_meta, const std::shared_ptr<GlobalIndexFileReader>& file_reader,
+ const std::shared_ptr<MemoryPool>& pool);
- // === FunctionVisitor surface — non-FTS predicates fall back to full range. ===
+ ~FullTextGlobalIndexReader() override;
Result<std::shared_ptr<GlobalIndexResult>> VisitIsNotNull() override {
- return CreateAllResult();
+ return std::shared_ptr<GlobalIndexResult>();
}
Result<std::shared_ptr<GlobalIndexResult>> VisitIsNull() override {
- return CreateAllResult();
+ return std::shared_ptr<GlobalIndexResult>();
}
Result<std::shared_ptr<GlobalIndexResult>> VisitEqual(const Literal&) override {
- return CreateAllResult();
+ return std::shared_ptr<GlobalIndexResult>();
}
Result<std::shared_ptr<GlobalIndexResult>> VisitNotEqual(const Literal&) override {
- return CreateAllResult();
+ return std::shared_ptr<GlobalIndexResult>();
}
Result<std::shared_ptr<GlobalIndexResult>> VisitLessThan(const Literal&) override {
- return CreateAllResult();
+ return std::shared_ptr<GlobalIndexResult>();
}
Result<std::shared_ptr<GlobalIndexResult>> VisitLessOrEqual(const Literal&) override {
- return CreateAllResult();
+ return std::shared_ptr<GlobalIndexResult>();
}
Result<std::shared_ptr<GlobalIndexResult>> VisitGreaterThan(const Literal&) override {
- return CreateAllResult();
+ return std::shared_ptr<GlobalIndexResult>();
}
Result<std::shared_ptr<GlobalIndexResult>> VisitGreaterOrEqual(const Literal&) override {
- return CreateAllResult();
+ return std::shared_ptr<GlobalIndexResult>();
}
Result<std::shared_ptr<GlobalIndexResult>> VisitIn(const std::vector<Literal>&) override {
- return CreateAllResult();
+ return std::shared_ptr<GlobalIndexResult>();
}
Result<std::shared_ptr<GlobalIndexResult>> VisitNotIn(const std::vector<Literal>&) override {
- return CreateAllResult();
+ return std::shared_ptr<GlobalIndexResult>();
}
Result<std::shared_ptr<GlobalIndexResult>> VisitStartsWith(const Literal&) override {
- return CreateAllResult();
+ return std::shared_ptr<GlobalIndexResult>();
}
Result<std::shared_ptr<GlobalIndexResult>> VisitEndsWith(const Literal&) override {
- return CreateAllResult();
+ return std::shared_ptr<GlobalIndexResult>();
}
Result<std::shared_ptr<GlobalIndexResult>> VisitContains(const Literal&) override {
- return CreateAllResult();
+ return std::shared_ptr<GlobalIndexResult>();
}
Result<std::shared_ptr<GlobalIndexResult>> VisitLike(const Literal&) override {
- return CreateAllResult();
+ return std::shared_ptr<GlobalIndexResult>();
}
Result<std::shared_ptr<ScoredGlobalIndexResult>> VisitVectorSearch(
const std::shared_ptr<VectorSearch>&) override {
return Status::Invalid(
- "TantivyGlobalIndexReader is not supposed to handle vector search query");
+ "FullTextGlobalIndexReader is not supposed to handle vector search query");
}
Result<std::shared_ptr<GlobalIndexResult>> VisitFullTextSearch(
const std::shared_ptr<FullTextSearch>& full_text_search) override;
+ /// The native reader supports concurrent searches, and opening it is guarded by a mutex.
bool IsThreadSafe() const override {
- return false;
+ return true;
}
std::string GetIndexType() const override {
@@ -114,17 +115,23 @@ class TantivyGlobalIndexReader : public GlobalIndexReader {
}
private:
- TantivyGlobalIndexReader(ReaderPtr reader, std::shared_ptr<MemoryPool> pool)
- : reader_(std::move(reader)), pool_(std::move(pool)) {}
+ FullTextGlobalIndexReader(const GlobalIndexIOMeta& io_meta,
+ const std::shared_ptr<GlobalIndexFileReader>& file_reader,
+ const std::shared_ptr<MemoryPool>& pool);
- std::shared_ptr<GlobalIndexResult> CreateAllResult() const {
- return nullptr;
- }
+ /// Returns the native reader, opening the archive on the first call.
+ Result<PaimonFtindexReaderHandle*> GetOrOpenReader();
+
+ void CloseInputStream(InputStream* stream) const;
- /// Owning handle to the Rust-side reader.
- ReaderPtr reader_;
- /// MemoryPool used for serializing pre-filter bitmaps to bytes for FFI.
+ GlobalIndexIOMeta io_meta_;
+ std::shared_ptr<GlobalIndexFileReader> file_reader_;
std::shared_ptr<MemoryPool> pool_;
+ std::mutex open_mutex_;
+ /// Must outlive `reader_`, which reads through it.
+ std::unique_ptr<FullTextInputContext> input_;
+ FtindexReaderPtr reader_;
+ std::unique_ptr<Logger> logger_;
};
-} // namespace paimon::tantivy
+} // namespace paimon::full_text
diff --git a/src/paimon/global_index/full_text/full_text_global_index_test.cpp b/src/paimon/global_index/full_text/full_text_global_index_test.cpp
new file mode 100644
index 00000000..e61364c0
--- /dev/null
+++ b/src/paimon/global_index/full_text/full_text_global_index_test.cpp
@@ -0,0 +1,737 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/global_index/full_text/full_text_global_index.h"
+
+#include <cstdint>
+#include <cstdlib>
+#include <map>
+#include <memory>
+#include <numeric>
+#include <optional>
+#include <string>
+#include <thread>
+#include <utility>
+#include <vector>
+
+#include "arrow/api.h"
+#include "arrow/c/bridge.h"
+#include "arrow/ipc/api.h"
+#include "gtest/gtest.h"
+#include "paimon/common/factories/io_hook.h"
+#include "paimon/common/utils/arrow/status_utils.h"
+#include "paimon/common/utils/path_util.h"
+#include "paimon/common/utils/scope_guard.h"
+#include "paimon/common/utils/string_utils.h"
+#include "paimon/core/global_index/global_index_file_manager.h"
+#include "paimon/fs/local/local_file_system.h"
+#include "paimon/global_index/bitmap_global_index_result.h"
+#include "paimon/global_index/bitmap_scored_global_index_result.h"
+#include "paimon/global_index/global_indexer_factory.h"
+#include "paimon/global_index/io/global_index_file_writer.h"
+#include "paimon/predicate/full_text_search.h"
+#include "paimon/predicate/literal.h"
+#include "paimon/testing/mock/mock_index_path_factory.h"
+#include "paimon/testing/utils/io_exception_helper.h"
+#include "paimon/testing/utils/testharness.h"
+#include "rapidjson/document.h"
+
+namespace paimon::full_text::test {
+
+namespace {
+
+struct GoldenSearch {
+ std::string query;
+ int32_t limit;
+ std::optional<std::vector<int64_t>> pre_filter;
+ std::vector<std::pair<int64_t, float>> hits;
+};
+
+struct GoldenFixture {
+ std::string name;
+ std::map<std::string, std::string> options;
+ std::string rows;
+ std::vector<GoldenSearch> searches;
+};
+
+const std::vector<GoldenFixture>& GoldenFixtures() {
+ static const std::vector<GoldenFixture> kFixtures = {
+ {"default",
+ {},
+ R"([["Apache Paimon is a streaming data lake platform"], [null],
+ ["Paimon supports real-time data ingestion and running queries"],
+ ["The runner runs quickly"], ["Lake formats store huge tables"]])",
+ {
+ {R"({"match":{"query":"paimon"}})",
+ 10,
+ std::nullopt,
+ {{0, 0.668293297f}, {2, 0.584465563f}}},
+ {R"({"match":{"query":"run"}})",
+ 10,
+ std::nullopt,
+ {{2, 0.584465563f}, {3, 0.851480305f}}},
+ {R"({"match":{"query":"paimon lake","operator":"And"}})",
+ 10,
+ std::nullopt,
+ {{0, 1.33658659f}}},
+ {R"({"match":{"query":"paimon lake"}})", 1, std::nullopt, {{0, 1.33658659f}}},
+ {R"({"match_phrase":{"query":"data lake"}})", 10, std::nullopt, {{0, 1.33658659f}}},
+ {R"({"boolean":{"must":[{"match":{"query":"paimon"}}],"must_not":[{"match":{"query":"streaming"}}]}})",
+ 10,
+ std::nullopt,
+ {{2, 0.584465563f}}},
+ {R"({"match":{"query":"lake"}})",
+ 10,
+ std::vector<int64_t>({0, 3}),
+ {{0, 0.668293297f}}},
+ }},
+ {"ngram",
+ {{"full-text.tokenizer", "ngram"},
+ {"full-text.ngram.min-gram", "2"},
+ {"full-text.ngram.max-gram", "3"}},
+ R"([["paimon"], ["lakehouse"], [null], ["streaming"]])",
+ {
+ {R"({"match":{"query":"aim"}})", 10, std::nullopt, {{0, 3.43641996f}}},
+ {R"({"match":{"query":"house"}})", 10, std::nullopt, {{1, 6.30786085f}}},
+ }},
+ {"jieba",
+ {{"full-text.tokenizer", "jieba"}},
+ R"([["张华在百货公司当售货员"], ["Apache Paimon supports full text search"], [null],
+ ["我们在数据湖中存储数据"]])",
+ {
+ {R"({"match":{"query":"售货员"}})", 10, std::nullopt, {{0, 2.89690685f}}},
+ {R"({"match":{"query":"数据湖"}})", 10, std::nullopt, {{3, 1.4764061f}}},
+ {R"({"match":{"query":"paimon"}})", 10, std::nullopt, {{1, 0.883518577f}}},
+ }},
+ };
+ return kFixtures;
+}
+
+class FlushFailingFileWriter : public GlobalIndexFileWriter {
+ public:
+ FlushFailingFileWriter(const std::shared_ptr<GlobalIndexFileWriter>& delegate,
+ int32_t fail_from)
+ : delegate_(delegate), fail_from_(fail_from) {}
+
+ Result<std::string> NewFileName(const std::string& prefix) const override {
+ return delegate_->NewFileName(prefix);
+ }
+
+ Result<std::unique_ptr<OutputStream>> NewOutputStream(
+ const std::string& file_name) const override {
+ PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<OutputStream> stream,
+ delegate_->NewOutputStream(file_name));
+ return std::make_unique<FlushFailingOutputStream>(std::move(stream), fail_from_,
+ close_count_);
+ }
+
+ Result<int64_t> GetFileSize(const std::string& file_name) const override {
+ return delegate_->GetFileSize(file_name);
+ }
+
+ std::string ToPath(const std::string& file_name) const override {
+ return delegate_->ToPath(file_name);
+ }
+
+ int32_t CloseCount() const {
+ return *close_count_;
+ }
+
+ private:
+ class FlushFailingOutputStream : public OutputStream {
+ public:
+ FlushFailingOutputStream(std::unique_ptr<OutputStream> delegate, int32_t fail_from,
+ const std::shared_ptr<int32_t>& close_count)
+ : delegate_(std::move(delegate)), fail_from_(fail_from), close_count_(close_count) {}
+
+ Result<int64_t> Write(const char* buffer, int64_t size) override {
+ return delegate_->Write(buffer, size);
+ }
+ Status Flush() override {
+ if (flush_count_++ >= fail_from_) {
+ return Status::IOError("injected flush failure");
+ }
+ return delegate_->Flush();
+ }
+ Result<int64_t> GetPos() const override {
+ return delegate_->GetPos();
+ }
+ Result<std::string> GetUri() const override {
+ return delegate_->GetUri();
+ }
+ Status Close() override {
+ ++*close_count_;
+ return delegate_->Close();
+ }
+
+ private:
+ std::unique_ptr<OutputStream> delegate_;
+ int32_t fail_from_;
+ int32_t flush_count_ = 0;
+ std::shared_ptr<int32_t> close_count_;
+ };
+
+ std::shared_ptr<GlobalIndexFileWriter> delegate_;
+ int32_t fail_from_;
+ std::shared_ptr<int32_t> close_count_ = std::make_shared<int32_t>(0);
+};
+
+} // namespace
+
+class FullTextGlobalIndexTest : public ::testing::Test {
+ public:
+ void SetUp() override {
+ index_dir_ = paimon::test::UniqueTestDirectory::Create();
+ ASSERT_TRUE(index_dir_);
+ }
+
+ static std::string FixturePath(const std::string& name) {
+ return paimon::test::GetDataDir() + "/full_text_fixtures/" + name + ".archive";
+ }
+
+ std::unique_ptr<::ArrowSchema> CreateArrowSchema(
+ const std::shared_ptr<arrow::DataType>& data_type) const {
+ auto c_schema = std::make_unique<::ArrowSchema>();
+ EXPECT_TRUE(arrow::ExportType(*data_type, c_schema.get()).ok());
+ return c_schema;
+ }
+
+ std::shared_ptr<GlobalIndexFileManager> CreateFileManager() const {
+ return std::make_shared<GlobalIndexFileManager>(
+ fs_, std::make_shared<paimon::test::MockIndexPathFactory>(index_dir_->Str()),
+ /*checkpoint_path_factory=*/nullptr);
+ }
+
+ Result<std::shared_ptr<GlobalIndexWriter>> CreateWriter(
+ const std::map<std::string, std::string>& options) const {
+ FullTextGlobalIndex global_index(options);
+ return global_index.CreateWriter("f0", CreateArrowSchema(data_type_).get(),
+ CreateFileManager(), pool_);
+ }
+
+ std::shared_ptr<arrow::Array> MakeArray(const std::string& json) const {
+ return arrow::ipc::internal::json::ArrayFromJSON(data_type_, json).ValueOrDie();
+ }
+
+ Status AddBatch(GlobalIndexWriter* writer, const std::shared_ptr<arrow::Array>& array,
+ int64_t first_row_id) const {
+ ArrowArray c_array;
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*array, &c_array));
+ std::vector<int64_t> row_ids(array->length());
+ std::iota(row_ids.begin(), row_ids.end(), first_row_id);
+ return writer->AddBatch(&c_array, std::move(row_ids));
+ }
+
+ Result<std::vector<GlobalIndexIOMeta>> WriteIndex(
+ const std::map<std::string, std::string>& options,
+ const std::shared_ptr<arrow::Array>& array) const {
+ PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<GlobalIndexWriter> writer, CreateWriter(options));
+ PAIMON_RETURN_NOT_OK(AddBatch(writer.get(), array, /*first_row_id=*/0));
+ return writer->Finish();
+ }
+
+ Result<std::shared_ptr<GlobalIndexReader>> CreateReader(
+ const std::vector<GlobalIndexIOMeta>& metas,
+ const std::shared_ptr<arrow::DataType>& field_type = arrow::utf8()) const {
+ FullTextGlobalIndex global_index(/*options=*/{});
+ auto schema = arrow::schema({arrow::field("f0", field_type)});
+ auto c_schema = std::make_unique<::ArrowSchema>();
+ EXPECT_TRUE(arrow::ExportSchema(*schema, c_schema.get()).ok());
+ return global_index.CreateReader(c_schema.get(), CreateFileManager(), metas, pool_);
+ }
+
+ static std::shared_ptr<FullTextSearch> MakeSearch(
+ const std::string& query, std::optional<int32_t> limit = 10,
+ const std::optional<RoaringBitmap64>& pre_filter = std::nullopt, bool with_score = true,
+ std::optional<float> min_score = std::nullopt) {
+ return std::make_shared<FullTextSearch>("f0", limit, query,
+ FullTextSearch::SearchType::UNKNOWN, pre_filter,
+ with_score, min_score);
+ }
+
+ static std::string MatchQuery(const std::string& terms) {
+ return R"({"match":{"query":")" + terms + R"("}})";
+ }
+
+ static std::vector<std::pair<int64_t, float>> ToHits(
+ const std::shared_ptr<GlobalIndexResult>& result) {
+ auto scored_result = std::dynamic_pointer_cast<BitmapScoredGlobalIndexResult>(result);
+ EXPECT_TRUE(scored_result);
+ std::vector<std::pair<int64_t, float>> hits;
+ if (!scored_result) {
+ return hits;
+ }
+ auto iter_result = scored_result->CreateScoredIterator();
+ EXPECT_TRUE(iter_result.ok()) << iter_result.status().ToString();
+ if (!iter_result.ok()) {
+ return hits;
+ }
+ auto iter = std::move(iter_result).value();
+ while (iter->HasNext()) {
+ hits.push_back(iter->NextWithScore());
+ }
+ return hits;
+ }
+
+ static void CheckRowIds(const std::shared_ptr<GlobalIndexResult>& result,
+ const std::vector<int64_t>& expected_ids) {
+ ASSERT_TRUE(result);
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<GlobalIndexResult::Iterator> iter,
+ result->CreateIterator());
+ std::vector<int64_t> row_ids;
+ while (iter->HasNext()) {
+ row_ids.push_back(iter->Next());
+ }
+ ASSERT_EQ(row_ids, expected_ids) << result->ToString();
+ }
+
+ void CheckGoldenSearches(const GoldenFixture& fixture, const GlobalIndexIOMeta& meta) const {
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<GlobalIndexReader> reader, CreateReader({meta}));
+ for (const auto& search : fixture.searches) {
+ std::optional<RoaringBitmap64> pre_filter;
+ if (search.pre_filter) {
+ pre_filter = RoaringBitmap64::From(search.pre_filter.value());
+ }
+ ASSERT_OK_AND_ASSIGN(
+ std::shared_ptr<GlobalIndexResult> result,
+ reader->VisitFullTextSearch(MakeSearch(search.query, search.limit, pre_filter)));
+ auto hits = ToHits(result);
+ ASSERT_EQ(hits.size(), search.hits.size())
+ << fixture.name << ": " << search.query << ": " << result->ToString();
+ for (size_t i = 0; i < hits.size(); ++i) {
+ ASSERT_EQ(hits[i].first, search.hits[i].first)
+ << fixture.name << ": " << search.query << ": " << result->ToString();
+ ASSERT_NEAR(hits[i].second, search.hits[i].second, 1e-5)
+ << fixture.name << ": " << search.query << ": row " << hits[i].first;
+ }
+ }
+ }
+
+ Result<std::unique_ptr<rapidjson::Document>> ReadHeader(const std::string& path) const {
+ std::string content;
+ PAIMON_RETURN_NOT_OK(fs_->ReadFile(path, &content));
+ if (content.size() < 16 || content.compare(0, 8, "PFTIDX01") != 0) {
+ return Status::Invalid("missing PFTIDX01 magic in ", path);
+ }
+ auto read_be_u32 = [&content](size_t pos) {
+ uint32_t value = 0;
+ for (size_t i = 0; i < 4; ++i) {
+ value = (value << 8) | static_cast<uint8_t>(content[pos + i]);
+ }
+ return value;
+ };
+ if (read_be_u32(8) != 1) {
+ return Status::Invalid("unexpected format version in ", path);
+ }
+ uint32_t header_len = read_be_u32(12);
+ if (content.size() < 16 + static_cast<size_t>(header_len)) {
+ return Status::Invalid("truncated header in ", path);
+ }
+ auto header = std::make_unique<rapidjson::Document>();
+ header->Parse(content.data() + 16, header_len);
+ if (header->HasParseError() || !header->IsObject() || !header->HasMember("metadata") ||
+ !header->HasMember("files")) {
+ return Status::Invalid("invalid header in ", path);
+ }
+ return header;
+ }
+
+ protected:
+ std::shared_ptr<MemoryPool> pool_ = GetDefaultPool();
+ std::shared_ptr<FileSystem> fs_ = std::make_shared<LocalFileSystem>();
+ std::shared_ptr<arrow::DataType> data_type_ =
+ arrow::struct_({arrow::field("f0", arrow::utf8())});
+ std::unique_ptr<paimon::test::UniqueTestDirectory> index_dir_;
+};
+
+TEST_F(FullTextGlobalIndexTest, TestWriteAndSearch) {
+ std::map<std::string, std::string> options = {{"full-text.tokenizer", "default"},
+ {"full-text.with-position", "true"},
+ {"tokenizer", "ignored"}};
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<GlobalIndexer> indexer,
+ GlobalIndexerFactory::Get("full-text", options));
+ ASSERT_TRUE(dynamic_cast<FullTextGlobalIndex*>(indexer.get()));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<GlobalIndexWriter> writer,
+ indexer->CreateWriter("f0", CreateArrowSchema(data_type_).get(),
+ CreateFileManager(), pool_));
+ auto array = MakeArray(R"([
+ ["Apache Paimon is a streaming data lake platform"],
+ [null],
+ ["Lake formats store huge tables"]
+ ])");
+ ASSERT_OK(AddBatch(writer.get(), array, /*first_row_id=*/0));
+ ASSERT_OK_AND_ASSIGN(std::vector<GlobalIndexIOMeta> metas, writer->Finish());
+ ASSERT_EQ(metas.size(), 1u);
+ const GlobalIndexIOMeta& meta = metas[0];
+
+ std::string file_name = PathUtil::GetName(meta.file_path);
+ ASSERT_TRUE(StringUtils::StartsWith(file_name, "full-text-global-index-")) << file_name;
+ ASSERT_TRUE(StringUtils::EndsWith(file_name, ".index")) << file_name;
+ ASSERT_OK_AND_ASSIGN(FileStatus file_status, fs_->GetFileStatus(meta.file_path));
+ ASSERT_EQ(file_status.GetLen(), meta.file_size);
+ ASSERT_TRUE(meta.metadata);
+ ASSERT_EQ(std::string(meta.metadata->data(), meta.metadata->size()),
+ R"({"tokenizer":"default","with-position":"true"})");
+
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<GlobalIndexReader> reader, CreateReader(metas));
+ ASSERT_EQ(reader->GetIndexType(), "full-text");
+ ASSERT_TRUE(reader->IsThreadSafe());
+ ASSERT_OK_AND_ASSIGN(auto result,
+ reader->VisitFullTextSearch(MakeSearch(MatchQuery("THE LAKE"))));
+ CheckRowIds(result, {0, 2});
+}
+
+TEST_F(FullTextGlobalIndexTest, TestWriterWithoutRows) {
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<GlobalIndexWriter> writer, CreateWriter(/*options=*/{}));
+ ASSERT_OK_AND_ASSIGN(std::vector<GlobalIndexIOMeta> metas, writer->Finish());
+ ASSERT_TRUE(metas.empty());
+
+ std::vector<BasicFileStatus> file_status_list;
+ ASSERT_OK(fs_->ListDir(index_dir_->Str(), &file_status_list));
+ ASSERT_TRUE(file_status_list.empty());
+
+ ASSERT_NOK_WITH_MSG(writer->Finish(), "already finished");
+ ASSERT_NOK_WITH_MSG(AddBatch(writer.get(), MakeArray(R"([["paimon"]])"), 0),
+ "already finished");
+}
+
+TEST_F(FullTextGlobalIndexTest, TestWriterWithOnlyNullRows) {
+ ASSERT_OK_AND_ASSIGN(std::vector<GlobalIndexIOMeta> metas,
+ WriteIndex(/*options=*/{}, MakeArray(R"([[null], [null]])")));
+ ASSERT_EQ(metas.size(), 1u);
+ ASSERT_EQ(std::string(metas[0].metadata->data(), metas[0].metadata->size()), "{}");
+
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<GlobalIndexReader> reader, CreateReader(metas));
+ ASSERT_OK_AND_ASSIGN(auto result,
+ reader->VisitFullTextSearch(MakeSearch(MatchQuery("paimon"))));
+ CheckRowIds(result, {});
+}
+
+TEST_F(FullTextGlobalIndexTest, TestRelativeRowIds) {
+ const int64_t large_row_id = (int64_t{1} << 32) + 1;
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<GlobalIndexWriter> writer, CreateWriter(/*options=*/{}));
+ ASSERT_OK(AddBatch(writer.get(), MakeArray(R"([["paimon lake"], ["vector search"]])"),
+ /*first_row_id=*/0));
+ ASSERT_OK(AddBatch(writer.get(), MakeArray(R"([[null], ["paimon lake"]])"),
+ /*first_row_id=*/2));
+ ASSERT_OK(AddBatch(writer.get(), MakeArray(R"([["paimon paimon paimon"]])"),
+ /*first_row_id=*/0));
+ ASSERT_OK(AddBatch(writer.get(), MakeArray(R"([["paimon lake"]])"), large_row_id));
+ ASSERT_OK_AND_ASSIGN(std::vector<GlobalIndexIOMeta> metas, writer->Finish());
+ ASSERT_EQ(metas.size(), 1u);
+
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<GlobalIndexReader> reader, CreateReader(metas));
+ ASSERT_OK_AND_ASSIGN(auto result,
+ reader->VisitFullTextSearch(MakeSearch(MatchQuery("paimon"))));
+ auto hits = ToHits(result);
+ ASSERT_EQ(hits.size(), 3u) << result->ToString();
+ ASSERT_EQ(hits[0].first, 0);
+ ASSERT_EQ(hits[1].first, 3);
+ ASSERT_EQ(hits[2].first, large_row_id);
+ ASSERT_FLOAT_EQ(hits[0].second, hits[1].second);
+ ASSERT_FLOAT_EQ(hits[1].second, hits[2].second);
+
+ ASSERT_OK_AND_ASSIGN(
+ result, reader->VisitFullTextSearch(
+ MakeSearch(MatchQuery("paimon"), 10,
+ RoaringBitmap64::From(std::vector<int64_t>{1, large_row_id}))));
+ CheckRowIds(result, {large_row_id});
+}
+
+TEST_F(FullTextGlobalIndexTest, TestInvalidWrite) {
+ FullTextGlobalIndex global_index(/*options=*/{});
+ auto int_type = arrow::struct_({arrow::field("f0", arrow::int32())});
+ ASSERT_NOK_WITH_MSG(global_index.CreateWriter("f0", CreateArrowSchema(int_type).get(),
+ CreateFileManager(), pool_),
+ "only supports string fields");
+ ASSERT_NOK_WITH_MSG(global_index.CreateWriter("f1", CreateArrowSchema(data_type_).get(),
+ CreateFileManager(), pool_),
+ "field f1 not exist");
+ ASSERT_NOK_WITH_MSG(CreateWriter({{"full-text.tokenizer", "unknown"}}),
+ "open full-text index writer for field f0");
+ ASSERT_NOK_WITH_MSG(CreateWriter({{"full-text.lower-case", "maybe"}}), "lower-case");
+ ASSERT_NOK_WITH_MSG(CreateWriter({{"full-text.tokenizer", std::string("raw\0jieba", 9)}}),
+ "full-text index option full-text.tokenizer must not contain NUL");
+ ASSERT_NOK_WITH_MSG(CreateWriter({{std::string("full-text.tokenizer\0x", 21), "raw"}}),
+ "option keys must not contain NUL");
+
+ arrow::StringBuilder builder;
+ ASSERT_TRUE(builder.Append("paimon").ok());
+ ASSERT_TRUE(builder.Append(std::string("pai\0mon", 7)).ok());
+ std::shared_ptr<arrow::Array> strings;
+ ASSERT_TRUE(builder.Finish(&strings).ok());
+ std::shared_ptr<arrow::Array> array =
+ arrow::StructArray::Make({strings}, {arrow::field("f0", arrow::utf8())}).ValueOrDie();
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<GlobalIndexWriter> writer, CreateWriter(/*options=*/{}));
+ ASSERT_NOK_WITH_MSG(AddBatch(writer.get(), array, /*first_row_id=*/0),
+ "full-text index value of row 1 must not contain NUL characters");
+}
+
+TEST_F(FullTextGlobalIndexTest, TestPreFilterLimitAndMinScore) {
+ auto array = MakeArray(R"([
+ ["paimon paimon paimon paimon"],
+ ["paimon paimon"],
+ ["paimon"],
+ ["lake"]
+ ])");
+ ASSERT_OK_AND_ASSIGN(std::vector<GlobalIndexIOMeta> metas, WriteIndex(/*options=*/{}, array));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<GlobalIndexReader> reader, CreateReader(metas));
+
+ ASSERT_OK_AND_ASSIGN(auto all_result,
+ reader->VisitFullTextSearch(MakeSearch(MatchQuery("paimon"))));
+ auto all_hits = ToHits(all_result);
+ ASSERT_EQ(all_hits.size(), 3u);
+ ASSERT_GT(all_hits[0].second, all_hits[1].second);
+ ASSERT_GT(all_hits[1].second, all_hits[2].second);
+ {
+ ASSERT_OK_AND_ASSIGN(auto result,
+ reader->VisitFullTextSearch(MakeSearch(MatchQuery("paimon"), 1)));
+ CheckRowIds(result, {0});
+ }
+ {
+ ASSERT_OK_AND_ASSIGN(
+ auto result,
+ reader->VisitFullTextSearch(MakeSearch(
+ MatchQuery("paimon"), 10, RoaringBitmap64::From(std::vector<int64_t>{1, 2, 3}))));
+ CheckRowIds(result, {1, 2});
+ auto hits = ToHits(result);
+ ASSERT_EQ(hits.size(), 2u);
+ ASSERT_FLOAT_EQ(hits[0].second, all_hits[1].second);
+ ASSERT_FLOAT_EQ(hits[1].second, all_hits[2].second);
+ }
+ {
+ ASSERT_OK_AND_ASSIGN(auto result, reader->VisitFullTextSearch(MakeSearch(
+ MatchQuery("paimon"), 1,
+ RoaringBitmap64::From(std::vector<int64_t>{1, 2}))));
+ CheckRowIds(result, {1});
+ }
+ {
+ ASSERT_OK_AND_ASSIGN(auto result, reader->VisitFullTextSearch(MakeSearch(
+ MatchQuery("paimon"), 10, RoaringBitmap64())));
+ CheckRowIds(result, {});
+ }
+ {
+ ASSERT_OK_AND_ASSIGN(auto result, reader->VisitFullTextSearch(
+ MakeSearch(MatchQuery("paimon"), 10, std::nullopt,
+ /*with_score=*/true, all_hits[1].second)));
+ CheckRowIds(result, {0});
+ }
+ {
+ ASSERT_OK_AND_ASSIGN(auto result,
+ reader->VisitFullTextSearch(MakeSearch(
+ MatchQuery("paimon"), 10, std::nullopt, /*with_score=*/false)));
+ ASSERT_TRUE(std::dynamic_pointer_cast<BitmapGlobalIndexResult>(result));
+ ASSERT_FALSE(std::dynamic_pointer_cast<BitmapScoredGlobalIndexResult>(result));
+ CheckRowIds(result, {0, 1, 2});
+ }
+}
+
+TEST_F(FullTextGlobalIndexTest, TestInvalidRead) {
+ GlobalIndexIOMeta meta("full-text-global-index-1.index", 1, nullptr);
+ ASSERT_NOK_WITH_MSG(CreateReader({}), "exactly one index file");
+ ASSERT_NOK_WITH_MSG(CreateReader({meta, meta}), "exactly one index file");
+ ASSERT_NOK_WITH_MSG(CreateReader({meta}, arrow::int32()), "only supports string fields");
+
+ ASSERT_OK_AND_ASSIGN(std::vector<GlobalIndexIOMeta> metas,
+ WriteIndex(/*options=*/{}, MakeArray(R"([["paimon"]])")));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<GlobalIndexReader> reader, CreateReader(metas));
+
+ ASSERT_NOK_WITH_MSG(reader->VisitFullTextSearch(nullptr), "null FullTextSearch");
+ ASSERT_NOK_WITH_MSG(reader->VisitFullTextSearch(MakeSearch(MatchQuery("paimon"), std::nullopt)),
+ "positive limit");
+ ASSERT_NOK_WITH_MSG(reader->VisitFullTextSearch(MakeSearch(MatchQuery("paimon"), 0)),
+ "positive limit");
+ ASSERT_NOK_WITH_MSG(
+ reader->VisitFullTextSearch(MakeSearch(MatchQuery(std::string("pai\0mon", 7)))),
+ "must not contain NUL characters");
+ ASSERT_NOK_WITH_MSG(reader->VisitFullTextSearch(MakeSearch("paimon")),
+ "search full-text index " + metas[0].file_path);
+ ASSERT_NOK_WITH_MSG(reader->VisitVectorSearch(nullptr), "vector search");
+
+ Literal literal(FieldType::STRING, "paimon", 6);
+ ASSERT_OK_AND_ASSIGN(auto result, reader->VisitEqual(literal));
+ ASSERT_FALSE(result);
+ ASSERT_OK_AND_ASSIGN(result, reader->VisitContains(literal));
+ ASSERT_FALSE(result);
+ ASSERT_OK_AND_ASSIGN(result, reader->VisitLike(literal));
+ ASSERT_FALSE(result);
+ ASSERT_OK_AND_ASSIGN(result, reader->VisitIsNull());
+ ASSERT_FALSE(result);
+}
+
+TEST_F(FullTextGlobalIndexTest, TestInvalidIndexFile) {
+ std::string file_path = PathUtil::JoinPath(index_dir_->Str(), "invalid.index");
+ ASSERT_OK(fs_->WriteFile(file_path, "this is not a full-text index", /*overwrite=*/false));
+ std::vector<GlobalIndexIOMeta> metas = {GlobalIndexIOMeta(file_path, 29, nullptr)};
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<GlobalIndexReader> reader, CreateReader(metas));
+ ASSERT_NOK_WITH_MSG(reader->VisitFullTextSearch(MakeSearch(MatchQuery("paimon"))), "bad magic");
+
+ std::string archive;
+ ASSERT_OK(fs_->ReadFile(FixturePath("default"), &archive));
+ std::string truncated_path = PathUtil::JoinPath(index_dir_->Str(), "truncated.index");
+ ASSERT_OK(fs_->WriteFile(truncated_path, archive.substr(0, 32), /*overwrite=*/false));
+ ASSERT_OK_AND_ASSIGN(reader, CreateReader({GlobalIndexIOMeta(truncated_path, 32, nullptr)}));
+ Status status = reader->VisitFullTextSearch(MakeSearch(MatchQuery("paimon"))).status();
+ ASSERT_TRUE(status.IsIOError()) << status.ToString();
+ ASSERT_NOK_WITH_MSG(status, "failed to open full-text index " + truncated_path);
+
+ std::string::size_type version_pos = archive.find("tantivy v0.26.1");
+ ASSERT_NE(version_pos, std::string::npos);
+ std::string other_version_archive = archive;
+ other_version_archive.replace(version_pos, 15, "tantivy v0.25.0");
+ std::string other_version_path = PathUtil::JoinPath(index_dir_->Str(), "other_version.index");
+ ASSERT_OK(fs_->WriteFile(other_version_path, other_version_archive, /*overwrite=*/false));
+ ASSERT_OK_AND_ASSIGN(
+ reader,
+ CreateReader({GlobalIndexIOMeta(
+ other_version_path, static_cast<int64_t>(other_version_archive.size()), nullptr)}));
+ ASSERT_NOK_WITH_MSG(reader->VisitFullTextSearch(MakeSearch(MatchQuery("paimon"))),
+ "unsupported Tantivy index version tantivy v0.25.0");
+}
+
+TEST_F(FullTextGlobalIndexTest, TestConcurrentSearch) {
+ auto array = MakeArray(R"([
+ ["Apache Paimon is a streaming data lake platform"],
+ ["Paimon supports real-time data ingestion"],
+ ["Native full-text search runs in Rust"]
+ ])");
+ ASSERT_OK_AND_ASSIGN(std::vector<GlobalIndexIOMeta> metas, WriteIndex(/*options=*/{}, array));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<GlobalIndexReader> reader, CreateReader(metas));
+ std::shared_ptr<FullTextSearch> paimon_search =
+ MakeSearch(MatchQuery("paimon"), 10, RoaringBitmap64::From(std::vector<int64_t>{1, 2}));
+ std::shared_ptr<FullTextSearch> rust_search = MakeSearch(MatchQuery("rust"));
+
+ constexpr int32_t kThreadNum = 8;
+ constexpr int32_t kRoundNum = 10;
+ std::vector<std::thread> threads;
+ std::vector<Status> statuses(kThreadNum);
+ std::vector<int32_t> finished_rounds(kThreadNum, 0);
+ for (int32_t i = 0; i < kThreadNum; ++i) {
+ threads.emplace_back([&, i]() {
+ const std::shared_ptr<FullTextSearch>& search =
+ i % 2 == 0 ? paimon_search : rust_search;
+ std::vector<int64_t> expected_ids =
+ i % 2 == 0 ? std::vector<int64_t>({1}) : std::vector<int64_t>({2});
+ for (int32_t round = 0; round < kRoundNum; ++round) {
+ Result<std::shared_ptr<GlobalIndexResult>> result =
+ reader->VisitFullTextSearch(search);
+ if (!result.ok()) {
+ statuses[i] = result.status();
+ return;
+ }
+ Result<std::unique_ptr<GlobalIndexResult::Iterator>> iter =
+ result.value()->CreateIterator();
+ if (!iter.ok()) {
+ statuses[i] = iter.status();
+ return;
+ }
+ std::vector<int64_t> row_ids;
+ while (iter.value()->HasNext()) {
+ row_ids.push_back(iter.value()->Next());
+ }
+ if (row_ids != expected_ids) {
+ statuses[i] = Status::Invalid("unexpected result in round ", round, ": ",
+ result.value()->ToString());
+ return;
+ }
+ ++finished_rounds[i];
+ }
+ });
+ }
+ for (auto& thread : threads) {
+ thread.join();
+ }
+ for (int32_t i = 0; i < kThreadNum; ++i) {
+ ASSERT_OK(statuses[i]) << "thread " << i;
+ ASSERT_EQ(finished_rounds[i], kRoundNum) << "thread " << i;
+ }
+}
+
+TEST_F(FullTextGlobalIndexTest, TestIOException) {
+ auto array = MakeArray(R"([["Apache Paimon"], [null], ["full-text search"]])");
+ bool run_complete = false;
+ auto io_hook = IOHook::GetInstance();
+ for (size_t i = 0; i < 500; ++i) {
+ ScopeGuard guard([&io_hook]() { io_hook->Clear(); });
+ io_hook->Reset(i, IOHook::Mode::RETURN_ERROR);
+ Result<std::vector<GlobalIndexIOMeta>> metas = WriteIndex(/*options=*/{}, array);
+ CHECK_HOOK_STATUS(metas.status(), i);
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<GlobalIndexReader> reader,
+ CreateReader(metas.value()));
+ Result<std::shared_ptr<GlobalIndexResult>> result =
+ reader->VisitFullTextSearch(MakeSearch(MatchQuery("paimon")));
+ CHECK_HOOK_STATUS(result.status(), i);
+ CheckRowIds(result.value(), {0});
+ run_complete = true;
+ break;
+ }
+ ASSERT_TRUE(run_complete);
+}
+
+TEST_F(FullTextGlobalIndexTest, TestFlushFailureClosesOutputStream) {
+ FullTextGlobalIndex global_index(/*options=*/{});
+ bool run_complete = false;
+ for (int32_t fail_from = 0; fail_from < 10; ++fail_from) {
+ auto file_writer = std::make_shared<FlushFailingFileWriter>(CreateFileManager(), fail_from);
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<GlobalIndexWriter> writer,
+ global_index.CreateWriter("f0", CreateArrowSchema(data_type_).get(),
+ file_writer, pool_));
+ ASSERT_OK(AddBatch(writer.get(), MakeArray(R"([["paimon"]])"), /*first_row_id=*/0));
+ Result<std::vector<GlobalIndexIOMeta>> metas = writer->Finish();
+ ASSERT_EQ(file_writer->CloseCount(), 1) << "fail_from " << fail_from;
+ if (metas.ok()) {
+ ASSERT_GT(fail_from, 0);
+ run_complete = true;
+ break;
+ }
+ ASSERT_NOK_WITH_MSG(metas.status(), "injected flush failure");
+ }
+ ASSERT_TRUE(run_complete);
+}
+
+TEST_F(FullTextGlobalIndexTest, TestCrossReadFixtures) {
+ const char* output_dir = std::getenv("PAIMON_FULL_TEXT_ARCHIVE_OUTPUT_DIR");
+ for (const auto& fixture : GoldenFixtures()) {
+ std::string fixture_path = FixturePath(fixture.name);
+ ASSERT_OK_AND_ASSIGN(FileStatus file_status, fs_->GetFileStatus(fixture_path));
+ CheckGoldenSearches(fixture, GlobalIndexIOMeta(fixture_path, file_status.GetLen(),
+ /*metadata=*/nullptr));
+
+ ASSERT_OK_AND_ASSIGN(std::vector<GlobalIndexIOMeta> metas,
+ WriteIndex(fixture.options, MakeArray(fixture.rows)));
+ ASSERT_EQ(metas.size(), 1u) << fixture.name;
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<rapidjson::Document> written_header,
+ ReadHeader(metas[0].file_path));
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<rapidjson::Document> fixture_header,
+ ReadHeader(fixture_path));
+ ASSERT_TRUE((*written_header)["metadata"] == (*fixture_header)["metadata"]) << fixture.name;
+ ASSERT_TRUE((*written_header)["files"].IsArray()) << fixture.name;
+ ASSERT_FALSE((*written_header)["files"].Empty()) << fixture.name;
+ CheckGoldenSearches(fixture, metas[0]);
+
+ if (output_dir != nullptr) {
+ std::string archive;
+ ASSERT_OK(fs_->ReadFile(metas[0].file_path, &archive));
+ ASSERT_OK(fs_->WriteFile(PathUtil::JoinPath(output_dir, fixture.name + ".archive"),
+ archive, /*overwrite=*/true));
+ }
+ }
+}
+
+} // namespace paimon::full_text::test
diff --git a/src/paimon/global_index/full_text/full_text_global_index_writer.cpp b/src/paimon/global_index/full_text/full_text_global_index_writer.cpp
new file mode 100644
index 00000000..4cf6dad2
--- /dev/null
+++ b/src/paimon/global_index/full_text/full_text_global_index_writer.cpp
@@ -0,0 +1,226 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/global_index/full_text/full_text_global_index_writer.h"
+
+#include <optional>
+#include <string_view>
+#include <utility>
+
+#include "arrow/array.h"
+#include "arrow/c/bridge.h"
+#include "arrow/c/helpers.h"
+#include "fmt/format.h"
+#include "paimon/common/global_index/global_index_utils.h"
+#include "paimon/common/utils/arrow/status_utils.h"
+#include "paimon/common/utils/rapidjson_util.h"
+#include "paimon/common/utils/scope_guard.h"
+#include "paimon/fs/file_system.h"
+#include "paimon/global_index/full_text/full_text_defs.h"
+#include "paimon/memory/bytes.h"
+
+namespace paimon::full_text {
+
+#define CHECK_NOT_NULL(pointer, error_msg) \
+ do { \
+ if (!(pointer)) { \
+ return Status::Invalid(error_msg); \
+ } \
+ } while (0)
+
+namespace {
+
+/// Context of the native output callbacks.
+struct OutputContext {
+ OutputStream* stream = nullptr;
+ /// First error raised by the stream, reported instead of the generic native error.
+ Status status = Status::OK();
+};
+
+int WriteToOutputStream(void* ctx, const uint8_t* buf, size_t len) {
+ auto* output = static_cast<OutputContext*>(ctx);
+ size_t written = 0;
+ while (written < len) {
+ Result<int64_t> result = output->stream->Write(reinterpret_cast<const char*>(buf + written),
+ static_cast<int64_t>(len - written));
+ if (!result.ok()) {
+ output->status = result.status();
+ return -1;
+ }
+ if (result.value() <= 0) {
+ output->status = Status::IOError(fmt::format(
+ "short write to full-text index file: wrote {} of {} bytes", written, len));
+ return -1;
+ }
+ written += static_cast<size_t>(result.value());
+ }
+ return 0;
+}
+
+int FlushOutputStream(void* ctx) {
+ auto* output = static_cast<OutputContext*>(ctx);
+ Status status = output->stream->Flush();
+ if (!status.ok()) {
+ output->status = std::move(status);
+ return -1;
+ }
+ return 0;
+}
+
+} // namespace
+
+Result<std::shared_ptr<FullTextGlobalIndexWriter>> FullTextGlobalIndexWriter::Create(
+ const std::string& field_name, const std::shared_ptr<arrow::DataType>& arrow_type,
+ const std::shared_ptr<GlobalIndexFileWriter>& file_writer,
+ const std::map<std::string, std::string>& options, const std::shared_ptr<MemoryPool>& pool) {
+ std::vector<const char*> keys;
+ std::vector<const char*> values;
+ keys.reserve(options.size());
+ values.reserve(options.size());
+ for (const auto& [key, value] : options) {
+ if (key.find('\0') != std::string::npos) {
+ return Status::Invalid("full-text index option keys must not contain NUL characters");
+ }
+ if (value.find('\0') != std::string::npos) {
+ return Status::Invalid(
+ fmt::format("full-text index option {}{} must not contain NUL characters",
+ kOptionKeyPrefix, key));
+ }
+ keys.push_back(key.c_str());
+ values.push_back(value.c_str());
+ }
+ FtindexWriterPtr writer(paimon_ftindex_writer_open(keys.data(), values.data(), keys.size()));
+ if (!writer) {
+ return LastFtindexError(
+ fmt::format("open full-text index writer for field {}", field_name));
+ }
+ return std::shared_ptr<FullTextGlobalIndexWriter>(new FullTextGlobalIndexWriter(
+ field_name, arrow_type, std::move(writer), file_writer, options, pool));
+}
+
+FullTextGlobalIndexWriter::FullTextGlobalIndexWriter(
+ const std::string& field_name, const std::shared_ptr<arrow::DataType>& arrow_type,
+ FtindexWriterPtr writer, const std::shared_ptr<GlobalIndexFileWriter>& file_writer,
+ const std::map<std::string, std::string>& options, const std::shared_ptr<MemoryPool>& pool)
+ : pool_(pool ? pool : GetDefaultPool()),
+ field_name_(field_name),
+ arrow_type_(arrow_type),
+ writer_(std::move(writer)),
+ file_writer_(file_writer),
+ options_(options),
+ logger_(Logger::GetLogger("FullTextGlobalIndexWriter")) {}
+
+Status FullTextGlobalIndexWriter::AddBatch(::ArrowArray* arrow_array,
+ std::vector<int64_t>&& relative_row_ids) {
+ if (finished_) {
+ if (arrow_array != nullptr) {
+ ArrowArrayRelease(arrow_array);
+ }
+ return Status::Invalid("FullTextGlobalIndexWriter is already finished");
+ }
+ // Documents are keyed by the given relative row ids, as in Java, so only the lengths are
+ // checked here.
+ PAIMON_RETURN_NOT_OK(
+ GlobalIndexUtils::CheckRelativeRowIds(arrow_array, relative_row_ids,
+ /*expected_next_row_id=*/std::nullopt));
+ PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> array,
+ arrow::ImportArray(arrow_array, arrow_type_));
+ auto struct_array = std::dynamic_pointer_cast<arrow::StructArray>(array);
+ CHECK_NOT_NULL(struct_array,
+ "invalid input array in FullTextGlobalIndexWriter, must be struct array");
+ std::shared_ptr<arrow::Array> field_array = struct_array->GetFieldByName(field_name_);
+ CHECK_NOT_NULL(
+ field_array,
+ fmt::format("invalid input array in FullTextGlobalIndexWriter, field {} not in input array",
+ field_name_));
+ auto string_array = std::dynamic_pointer_cast<arrow::StringArray>(field_array);
+ CHECK_NOT_NULL(string_array,
+ fmt::format("invalid input array in FullTextGlobalIndexWriter, field array {} "
+ "is not a string array",
+ field_name_));
+
+ std::string text;
+ for (int64_t i = 0; i < string_array->length(); ++i) {
+ if (!string_array->IsNull(i)) {
+ std::string_view value = string_array->GetView(i);
+ // The native API takes NUL-terminated strings, so an embedded NUL would silently
+ // truncate the document.
+ if (value.find('\0') != std::string_view::npos) {
+ return Status::Invalid(
+ fmt::format("full-text index value of row {} must not contain NUL characters",
+ relative_row_ids[i]));
+ }
+ text.assign(value.data(), value.size());
+ if (paimon_ftindex_writer_add_document(writer_.get(), relative_row_ids[i],
+ text.c_str()) != 0) {
+ return LastFtindexError(
+ fmt::format("add row {} to full-text index", relative_row_ids[i]));
+ }
+ }
+ row_count_++;
+ }
+ return Status::OK();
+}
+
+Result<std::vector<GlobalIndexIOMeta>> FullTextGlobalIndexWriter::Finish() {
+ if (finished_) {
+ return Status::Invalid("FullTextGlobalIndexWriter is already finished");
+ }
+ finished_ = true;
+ // The native writer is finalized by any write attempt, so release it whatever happens.
+ FtindexWriterPtr writer = std::move(writer_);
+ if (row_count_ == 0) {
+ return std::vector<GlobalIndexIOMeta>();
+ }
+ PAIMON_ASSIGN_OR_RAISE(GlobalIndexIOMeta meta, WriteIndex(writer.get()));
+ return std::vector<GlobalIndexIOMeta>({meta});
+}
+
+Result<GlobalIndexIOMeta> FullTextGlobalIndexWriter::WriteIndex(
+ PaimonFtindexWriterHandle* writer) const {
+ PAIMON_ASSIGN_OR_RAISE(std::string file_name, file_writer_->NewFileName(kFileNamePrefix));
+ PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<OutputStream> out,
+ file_writer_->NewOutputStream(file_name));
+ // Closes the stream on every error path, keeping the original error.
+ ScopeGuard close_guard([&]() {
+ Status status = out->Close();
+ if (!status.ok()) {
+ PAIMON_LOG_WARN(logger_, "failed to close full-text index file %s: %s",
+ file_name.c_str(), status.ToString().c_str());
+ }
+ });
+ OutputContext ctx;
+ ctx.stream = out.get();
+ PaimonFtindexOutputFile output{static_cast<void*>(&ctx), &WriteToOutputStream,
+ &FlushOutputStream};
+ if (paimon_ftindex_writer_write_index(writer, output) != 0) {
+ return LastFtindexError(fmt::format("write full-text index {}", file_name), ctx.status);
+ }
+ PAIMON_RETURN_NOT_OK(out->Flush());
+ close_guard.Release();
+ PAIMON_RETURN_NOT_OK(out->Close());
+
+ PAIMON_ASSIGN_OR_RAISE(int64_t file_size, file_writer_->GetFileSize(file_name));
+ std::string options_json;
+ PAIMON_RETURN_NOT_OK(RapidJsonUtil::ToJsonString(options_, &options_json));
+ auto metadata = std::make_shared<Bytes>(options_json, pool_.get());
+ return GlobalIndexIOMeta(file_writer_->ToPath(file_name), file_size, metadata);
+}
+
+} // namespace paimon::full_text
diff --git a/src/paimon/global_index/tantivy/tantivy_global_index_writer.h b/src/paimon/global_index/full_text/full_text_global_index_writer.h
similarity index 52%
rename from src/paimon/global_index/tantivy/tantivy_global_index_writer.h
rename to src/paimon/global_index/full_text/full_text_global_index_writer.h
index 0d3247ab..d10da2b7 100644
--- a/src/paimon/global_index/tantivy/tantivy_global_index_writer.h
+++ b/src/paimon/global_index/full_text/full_text_global_index_writer.h
@@ -25,52 +25,52 @@
#include <vector>
#include "arrow/type.h"
+#include "paimon/global_index/full_text/full_text_ffi_utils.h"
#include "paimon/global_index/global_index_writer.h"
#include "paimon/global_index/io/global_index_file_writer.h"
-#include "paimon/global_index/tantivy/tantivy_defs.h"
-#include "paimon/global_index/tantivy/tantivy_ffi_handle.h"
+#include "paimon/logging.h"
+#include "paimon/memory/memory_pool.h"
-namespace paimon::tantivy {
+namespace paimon::full_text {
-/// Tantivy-backed implementation of GlobalIndexWriter.
+/// Writes one full-text index archive per shard through the native `paimon-full-text-index`
+/// engine, aligned with Java `NativeFullTextGlobalIndexWriter`.
///
-/// Mirrors LuceneGlobalIndexWriter's lifecycle:
-/// Create() → AddBatch()* → Finish()
-/// Each shard produces exactly one .index file via the GlobalIndexFileWriter,
-/// containing the full packed tantivy on-disk index in a single contiguous blob.
-///
-/// Indexes written by this class are NOT cross-readable with lucene-fts. The
-/// C++ side of this writer is intentionally thin: index construction, segment
-/// merging, and packing all happen in Rust behind the FFI boundary.
-class TantivyGlobalIndexWriter : public GlobalIndexWriter {
+/// Each non-null value is added as a document keyed by its relative row id. Null values advance
+/// the row count but are not indexed. A writer that received no rows writes no file.
+class FullTextGlobalIndexWriter : public GlobalIndexWriter {
public:
- static Result<std::shared_ptr<TantivyGlobalIndexWriter>> Create(
+ /// @param options Full-text options with the `full-text.` prefix already stripped. They are
+ /// passed to the native engine and stored as flat JSON in the index metadata.
+ static Result<std::shared_ptr<FullTextGlobalIndexWriter>> Create(
const std::string& field_name, const std::shared_ptr<arrow::DataType>& arrow_type,
const std::shared_ptr<GlobalIndexFileWriter>& file_writer,
const std::map<std::string, std::string>& options, const std::shared_ptr<MemoryPool>& pool);
- ~TantivyGlobalIndexWriter() override = default;
-
Status AddBatch(::ArrowArray* arrow_array, std::vector<int64_t>&& relative_row_ids) override;
Result<std::vector<GlobalIndexIOMeta>> Finish() override;
private:
- TantivyGlobalIndexWriter(const std::string& field_name,
- const std::shared_ptr<arrow::DataType>& arrow_type, WriterPtr writer,
- const std::shared_ptr<GlobalIndexFileWriter>& file_writer,
- const std::map<std::string, std::string>& options,
- const std::shared_ptr<MemoryPool>& pool);
+ FullTextGlobalIndexWriter(const std::string& field_name,
+ const std::shared_ptr<arrow::DataType>& arrow_type,
+ FtindexWriterPtr writer,
+ const std::shared_ptr<GlobalIndexFileWriter>& file_writer,
+ const std::map<std::string, std::string>& options,
+ const std::shared_ptr<MemoryPool>& pool);
+
+ Result<GlobalIndexIOMeta> WriteIndex(PaimonFtindexWriterHandle* writer) const;
std::shared_ptr<MemoryPool> pool_;
std::string field_name_;
std::shared_ptr<arrow::DataType> arrow_type_;
- /// Owning handle to the Rust-side writer.
- WriterPtr writer_;
+ FtindexWriterPtr writer_;
std::shared_ptr<GlobalIndexFileWriter> file_writer_;
std::map<std::string, std::string> options_;
- /// Last document index processed (matches caller-passed relative_row_ids).
- int64_t row_id_ = 0;
+ /// Number of rows received, including null values.
+ int64_t row_count_ = 0;
+ bool finished_ = false;
+ std::unique_ptr<Logger> logger_;
};
-} // namespace paimon::tantivy
+} // namespace paimon::full_text
diff --git a/src/paimon/global_index/lucene/lucene_global_index_reader.cpp b/src/paimon/global_index/lucene/lucene_global_index_reader.cpp
index c037f6a0..88bbfab4 100644
--- a/src/paimon/global_index/lucene/lucene_global_index_reader.cpp
+++ b/src/paimon/global_index/lucene/lucene_global_index_reader.cpp
@@ -267,7 +267,7 @@ Result<std::shared_ptr<GlobalIndexResult>> LuceneGlobalIndexReader::VisitFullTex
// results, which would be a correctness bug for the caller.
return Status::NotImplemented(
"lucene full-text search does not support min_score; "
- "min_score pushdown is only available on the tantivy backend");
+ "min_score pushdown is only available on the full-text backend");
}
try {
Lucene::QueryPtr query;
diff --git a/src/paimon/global_index/tantivy/CMakeLists.txt b/src/paimon/global_index/tantivy/CMakeLists.txt
deleted file mode 100644
index 8e1da23f..00000000
--- a/src/paimon/global_index/tantivy/CMakeLists.txt
+++ /dev/null
@@ -1,268 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# Tantivy-fulltext global index support library and tests (Rust FFI).
-
-if(NOT PAIMON_ENABLE_TANTIVY)
- return()
-endif()
-
-set(PAIMON_TANTIVY_SUPPORT_SRCS
- tantivy_defs.cpp
- tantivy_ffi_log.cpp
- tantivy_archive_layout.cpp
- tantivy_stream_ctx.cpp
- tantivy_global_index_writer.cpp
- tantivy_global_index_reader.cpp
- tantivy_global_index.cpp
- tantivy_global_index_factory.cpp)
-
-add_paimon_lib(paimon_tantivy_support
- SOURCES
- ${PAIMON_TANTIVY_SUPPORT_SRCS}
- DEPENDENCIES
- paimon_shared
- paimon_tantivy_ffi
- STATIC_LINK_LIBS
- paimon_tantivy_ffi
- arrow
- glog
- fmt
- SHARED_LINK_LIBS
- paimon_shared
- SHARED_LINK_FLAGS
- ${PAIMON_VERSION_SCRIPT_FLAGS})
-# Corrosion's paimon_tantivy_ffi target carries INTERFACE_INCLUDE_DIRECTORIES
-# (cbindgen-generated header path). The objlib in add_paimon_lib doesn't link
-# against deps,so its compile step misses include dirs.Wire them explicitly.
-target_link_libraries(paimon_tantivy_support_objlib PUBLIC paimon_tantivy_ffi)
-
-# In test builds, bake the jieba dict dir into GetJiebaDictionaryDirFromEnv so
-# tests don't have to setenv() process-wide state. Mirrors the lucene module.
-if(PAIMON_BUILD_TESTS)
- target_compile_definitions(paimon_tantivy_support_objlib
- PRIVATE JIEBA_TEST_DICT_DIR="${JIEBA_DICT_DIR}")
-endif()
-
-if(PAIMON_BUILD_TESTS)
- add_paimon_test(tantivy_smoke_test
- SOURCES
- tantivy_smoke_test.cpp
- STATIC_LINK_LIBS
- paimon_tantivy_ffi
- ${GTEST_LINK_TOOLCHAIN})
-
- add_paimon_test(tantivy_ffi_test
- SOURCES
- tantivy_ffi_test.cpp
- STATIC_LINK_LIBS
- paimon_shared
- "-Wl,--whole-archive"
- paimon_tantivy_support_static
- "-Wl,--no-whole-archive"
- paimon_tantivy_ffi
- glog
- fmt
- ${GTEST_LINK_TOOLCHAIN})
-
- # Golden-sample tokenizer diff (cppjieba vs jieba-rs). Links against the
- # lucene index module to reuse JiebaTokenizer::CutWithMode + Normalize, so it
- # can only be built when lucene-fts is enabled (the C++ JiebaTokenizer lives
- # in the lucene module). Guarded so the default LUCENE=OFF / TANTIVY=ON build
- # doesn't try to link the non-existent paimon_lucene_index_static.
- # Note: we mirror the lucene-fts test's link line (see lucene/CMakeLists.txt)
- # rather than using the `jieba` imported target, whose INTERFACE_INCLUDE
- # concatenates two paths in one string (upstream quirk).
- if(PAIMON_ENABLE_LUCENE)
- add_paimon_test(tantivy_tokenizer_test
- SOURCES
- tantivy_tokenizer_test.cpp
- EXTRA_INCLUDES
- ${LUCENE_INCLUDE_DIR}
- STATIC_LINK_LIBS
- paimon_shared
- test_utils_static
- "-Wl,--whole-archive"
- paimon_local_file_system_static
- paimon_lucene_index_static
- "-Wl,--no-whole-archive"
- paimon_tantivy_ffi
- ${GTEST_LINK_TOOLCHAIN})
- target_compile_definitions(paimon-tantivy-tokenizer-test
- PRIVATE JIEBA_TEST_DICT_DIR="${JIEBA_DICT_DIR}"
- PAIMON_TANTIVY_GOLDEN_DIR="${CMAKE_SOURCE_DIR}/test/test_data/tokenizer_golden"
- )
- target_include_directories(paimon-tantivy-tokenizer-test SYSTEM
- PRIVATE ${JIEBA_INCLUDE_DIR} ${JIEBA_DICT_DIR})
- endif()
-
- # Writer test. Builds an Arrow batch, runs the writer through
- # GlobalIndexFileManager + LocalFileSystem, then validates the packed
- # on-disk format.
- add_paimon_test(tantivy_writer_test
- SOURCES
- tantivy_writer_test.cpp
- STATIC_LINK_LIBS
- paimon_shared
- test_utils_static
- "-Wl,--whole-archive"
- paimon_local_file_system_static
- paimon_tantivy_support_static
- "-Wl,--no-whole-archive"
- paimon_tantivy_ffi
- arrow
- glog
- fmt
- ${GTEST_LINK_TOOLCHAIN})
- target_compile_definitions(paimon-tantivy-writer-test
- PRIVATE JIEBA_TEST_DICT_DIR="${JIEBA_DICT_DIR}")
-
- # Reader + five query types end-to-end.
- add_paimon_test(tantivy_filter_limit_test
- SOURCES
- tantivy_filter_limit_test.cpp
- STATIC_LINK_LIBS
- paimon_shared
- test_utils_static
- "-Wl,--whole-archive"
- paimon_local_file_system_static
- paimon_tantivy_support_static
- "-Wl,--no-whole-archive"
- paimon_tantivy_ffi
- arrow
- glog
- fmt
- ${GTEST_LINK_TOOLCHAIN})
- target_compile_definitions(paimon-tantivy-filter-limit-test
- PRIVATE JIEBA_TEST_DICT_DIR="${JIEBA_DICT_DIR}")
-
- # Java → C++ cross-read test. The fixture is produced by paimon-java's
- # `TantivyIndexFixtureGen` and checked in under
- # test/test_data/java_tantivy_fixtures/. See that directory's README for
- # regeneration instructions.
- add_paimon_test(tantivy_java_compat_test
- SOURCES
- tantivy_java_compat_test.cpp
- STATIC_LINK_LIBS
- paimon_shared
- test_utils_static
- "-Wl,--whole-archive"
- paimon_local_file_system_static
- paimon_tantivy_support_static
- "-Wl,--no-whole-archive"
- paimon_tantivy_ffi
- arrow
- glog
- fmt
- ${GTEST_LINK_TOOLCHAIN})
- target_compile_definitions(paimon-tantivy-java-compat-test
- PRIVATE JIEBA_TEST_DICT_DIR="${JIEBA_DICT_DIR}"
- PAIMON_TANTIVY_JAVA_FIXTURE_DIR="${CMAKE_SOURCE_DIR}/test/test_data/java_tantivy_fixtures"
- PAIMON_TANTIVY_CPP_FIXTURE_DIR="${CMAKE_SOURCE_DIR}/test/test_data/cpp_tantivy_fixtures"
- )
-
- # Streaming reader and writer integration coverage:
- # ParseArchiveHeader fuzz, concurrent query on shared reader, concurrent
- # reader create+drop lifecycle, streaming benchmark log.
- add_paimon_test(tantivy_streaming_test
- SOURCES
- tantivy_streaming_test.cpp
- STATIC_LINK_LIBS
- paimon_shared
- test_utils_static
- "-Wl,--whole-archive"
- paimon_local_file_system_static
- paimon_tantivy_support_static
- "-Wl,--no-whole-archive"
- paimon_tantivy_ffi
- arrow
- glog
- fmt
- ${GTEST_LINK_TOOLCHAIN})
- target_compile_definitions(paimon-tantivy-streaming-test
- PRIVATE JIEBA_TEST_DICT_DIR="${JIEBA_DICT_DIR}")
-
- # TantivyGlobalIndex + factory + end-to-end integration test.
- # `--whole-archive` is required so the static REGISTER_PAIMON_FACTORY
- # symbols are not stripped out of the test binary.
- add_paimon_test(tantivy_index_test
- SOURCES
- tantivy_index_test.cpp
- STATIC_LINK_LIBS
- paimon_shared
- test_utils_static
- "-Wl,--whole-archive"
- paimon_local_file_system_static
- paimon_tantivy_support_static
- "-Wl,--no-whole-archive"
- paimon_tantivy_ffi
- arrow
- glog
- fmt
- ${GTEST_LINK_TOOLCHAIN})
- target_compile_definitions(paimon-tantivy-index-test
- PRIVATE JIEBA_TEST_DICT_DIR="${JIEBA_DICT_DIR}")
-
- # Cross-implementation coexistence. Links against both the
- # lucene and tantivy support static libs to verify they resolve their
- # `REGISTER_PAIMON_FACTORY` registrations side by side and don't
- # collide on shared symbols. Only built when lucene-fts is enabled.
- if(PAIMON_ENABLE_LUCENE)
- add_paimon_test(tantivy_lucene_coexist_test
- SOURCES
- tantivy_lucene_coexist_test.cpp
- EXTRA_INCLUDES
- ${LUCENE_INCLUDE_DIR}
- STATIC_LINK_LIBS
- paimon_shared
- test_utils_static
- "-Wl,--whole-archive"
- paimon_local_file_system_static
- paimon_lucene_index_static
- paimon_tantivy_support_static
- "-Wl,--no-whole-archive"
- paimon_tantivy_ffi
- arrow
- glog
- fmt
- ${GTEST_LINK_TOOLCHAIN})
- target_compile_definitions(paimon-tantivy-lucene-coexist-test
- PRIVATE JIEBA_TEST_DICT_DIR="${JIEBA_DICT_DIR}")
-
- # Equivalence + benchmark. Same link line as the coexistence
- # test (needs both impls); benchmark output goes to stderr.
- add_paimon_test(tantivy_equivalence_test
- SOURCES
- tantivy_equivalence_test.cpp
- EXTRA_INCLUDES
- ${LUCENE_INCLUDE_DIR}
- STATIC_LINK_LIBS
- paimon_shared
- test_utils_static
- "-Wl,--whole-archive"
- paimon_local_file_system_static
- paimon_lucene_index_static
- paimon_tantivy_support_static
- "-Wl,--no-whole-archive"
- paimon_tantivy_ffi
- arrow
- glog
- fmt
- ${GTEST_LINK_TOOLCHAIN})
- target_compile_definitions(paimon-tantivy-equivalence-test
- PRIVATE JIEBA_TEST_DICT_DIR="${JIEBA_DICT_DIR}")
- endif()
-endif()
diff --git a/src/paimon/global_index/tantivy/tantivy_archive_layout.cpp b/src/paimon/global_index/tantivy/tantivy_archive_layout.cpp
deleted file mode 100644
index 21a009b1..00000000
--- a/src/paimon/global_index/tantivy/tantivy_archive_layout.cpp
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#include "paimon/global_index/tantivy/tantivy_archive_layout.h"
-
-#include <cstring>
-#include <memory>
-
-#include "fmt/format.h"
-#include "paimon/common/utils/math.h"
-#include "paimon/fs/file_system.h"
-#include "paimon/io/data_input_stream.h"
-
-namespace paimon::tantivy {
-
-namespace {
-
-/// Wrap the (non-owning) raw InputStream* in a shared_ptr-like handle so
-/// DataInputStream — which takes `shared_ptr<InputStream>` — can be used
-/// without transferring ownership. We use a no-op deleter to avoid double-free.
-struct NoopDeleter {
- void operator()(InputStream*) const {}
-};
-
-} // namespace
-
-Result<ArchiveLayout> ArchiveLayout::Parse(InputStream* in) {
- if (in == nullptr) {
- return Status::Invalid("ArchiveLayout::Parse: null input stream");
- }
-
- // DataInputStream defaults to BE — matches paimon-java archive format.
- std::shared_ptr<InputStream> wrapped(in, NoopDeleter{});
- DataInputStream dis(wrapped);
-
- PAIMON_RETURN_NOT_OK(dis.Seek(0));
-
- PAIMON_ASSIGN_OR_RAISE(int32_t file_count, dis.ReadValue<int32_t>());
- if (!InRange<std::size_t>(file_count)) {
- return Status::Invalid(fmt::format("ArchiveLayout::Parse: bad file_count {}", file_count));
- }
-
- ArchiveLayout layout;
- layout.count = static_cast<std::size_t>(file_count);
- layout.names.reserve(layout.count);
- layout.offsets.reserve(layout.count);
- layout.lengths.reserve(layout.count);
-
- for (int32_t i = 0; i < file_count; ++i) {
- PAIMON_ASSIGN_OR_RAISE(int32_t name_len, dis.ReadValue<int32_t>());
- if (name_len <= 0 || name_len > 1 << 20) {
- return Status::Invalid(
- fmt::format("ArchiveLayout::Parse: bad name_len {} at entry {}", name_len, i));
- }
- std::string name(static_cast<std::size_t>(name_len), '\0');
- PAIMON_RETURN_NOT_OK(dis.Read(name.data(), static_cast<uint32_t>(name_len)));
-
- PAIMON_ASSIGN_OR_RAISE(int64_t data_len, dis.ReadValue<int64_t>());
- if (!InRange<uint64_t>(data_len)) {
- return Status::Invalid(
- fmt::format("ArchiveLayout::Parse: bad data_len {} for '{}'", data_len, name));
- }
-
- PAIMON_ASSIGN_OR_RAISE(int64_t data_offset, dis.GetPos());
-
- layout.names.push_back(std::move(name));
- layout.offsets.push_back(static_cast<uint64_t>(data_offset));
- layout.lengths.push_back(static_cast<uint64_t>(data_len));
-
- // Skip past the payload without reading it.
- PAIMON_RETURN_NOT_OK(dis.Seek(data_offset + data_len));
- }
-
- return layout;
-}
-
-} // namespace paimon::tantivy
diff --git a/src/paimon/global_index/tantivy/tantivy_archive_layout.h b/src/paimon/global_index/tantivy/tantivy_archive_layout.h
deleted file mode 100644
index 73cb3f7f..00000000
--- a/src/paimon/global_index/tantivy/tantivy_archive_layout.h
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <cstdint>
-#include <memory>
-#include <string>
-#include <vector>
-
-#include "paimon/result.h"
-
-namespace paimon {
-class InputStream;
-} // namespace paimon
-
-namespace paimon::tantivy {
-
-/// Parsed layout of a packed tantivy archive. Arrays are parallel; `count` is
-/// their common length.
-///
-/// Archive byte format (matches paimon-java `TantivyFullTextGlobalIndexReader.
-/// parseArchiveHeader`; big-endian, no version header):
-/// `[BE i32 file_count | (BE i32 name_len, name_utf8, BE i64 data_len, data)*]`
-///
-/// `offsets[i]` is the archive-absolute byte offset of file `i`'s payload
-/// (points past the per-entry header). `lengths[i]` is the payload size.
-struct ArchiveLayout {
- std::vector<std::string> names;
- std::vector<uint64_t> offsets;
- std::vector<uint64_t> lengths;
- std::size_t count = 0;
-
- /// Read the archive header from `in` (seeking past payloads) and return the
- /// layout. Does NOT read file payloads — only header bytes (a few KB).
- ///
- /// `in` must support `Seek` (all production `paimon::InputStream` subclasses
- /// do; we call `Seek(cur + data_len)` to skip over each file's payload).
- ///
- /// On return, `in`'s internal position is at the end of the archive; callers
- /// typically don't care (the stream is subsequently read via pread callbacks).
- static Result<ArchiveLayout> Parse(InputStream* in);
-};
-
-} // namespace paimon::tantivy
diff --git a/src/paimon/global_index/tantivy/tantivy_defs.h b/src/paimon/global_index/tantivy/tantivy_defs.h
deleted file mode 100644
index f9e1ec13..00000000
--- a/src/paimon/global_index/tantivy/tantivy_defs.h
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include <cstddef>
-#include <cstdint>
-#include <optional>
-#include <string>
-
-namespace paimon::tantivy {
-
-/// Identifier used by GlobalIndexFileWriter::NewFileName to prefix on-disk
-/// filenames. Tantivy and lucene file prefixes intentionally differ so a
-/// reader can dispatch the right implementation by filename pattern.
-static inline const char kIdentifier[] = "tantivy-fulltext";
-
-/// Schema field names — fixed to match paimon-java. Callers
-/// MUST NOT rename these even though `TantivyGlobalIndexWriter::Create` accepts
-/// a `field_name` argument (that argument is used only to extract the correct
-/// arrow column; the tantivy schema field name is always `"text"`).
-static inline const char kTantivyTextFieldName[] = "text";
-static inline const char kTantivyRowIdFieldName[] = "row_id";
-
-/// Option-key prefix consumed by TantivyGlobalIndex. Matches the lucene-fts
-/// convention so users can configure both implementations with a uniform
-/// "<impl>.<knob>" key style.
-static inline const char kOptionKeyPrefix[] = "tantivy-fulltext.";
-
-/// Buffer size for streaming raw packed bytes from FFI to OutputStream
-/// (Writer) and from InputStream into Rust (Reader).
-static inline const int32_t kDefaultReadBufferSize = 1024 * 1024;
-/// Read buffer size knob for the reader.
-static inline const char kTantivyReadBufferSize[] = "read.buffer-size";
-
-/// If true, omit term frequencies/positions when indexing (smaller index, but
-/// no PhraseQuery support). Default false, mirroring lucene-fts.
-static inline const char kTantivyWriteOmitTermFreqAndPositions[] =
- "write.omit-term-freq-and-position";
-
-/// Env var carrying jieba dictionary directory; consumed by both writer and
-/// reader. Same name as lucene-fts: a single env var configures both backends.
-static inline const char kJiebaDictDirEnv[] = "PAIMON_JIEBA_DICT_DIR";
-
-/// Default tokenize mode if not specified in options.
-static inline const char kDefaultJiebaTokenizeMode[] = "mix";
-/// Tokenize mode option key. Values: "mp", "mix", "full", "query".
-/// "hmm" is rejected with Unsupported (jieba-rs does not expose standalone HMM).
-static inline const char kJiebaTokenizeMode[] = "jieba.tokenize-mode";
-
-/// Writer-side tokenizer selector. Values:
-/// "default" (default) — tantivy built-in SimpleTokenizer;
-/// "paimon_jieba" — jieba-rs CJK tokenizer; opt-in for Chinese workloads
-/// "whitespace" / "raw" / "en_stem" — other tantivy built-ins
-/// The reader side is schema-driven and auto-dispatches to whatever tokenizer
-/// name is baked into the archive, so the default here also determines what
-/// paimon-java sees when it cross-reads the archive.
-static inline const char kTantivyWriteTokenizer[] = "tantivy.write.tokenizer";
-/// Default tokenizer for writer: tantivy built-in "default" (SimpleTokenizer),
-/// chosen so paimon-cpp ↔ paimon-java cross-read works out of the box.
-/// Chinese workloads must opt into "paimon_jieba" via kTantivyWriteTokenizer.
-static inline const char kDefaultTantivyWriteTokenizer[] = "default";
-
-/// Reads the jieba dictionary directory from kJiebaDictDirEnv. Returns the
-/// directory when the env var is set and non-empty, otherwise std::nullopt.
-/// Shared by the writer and reader so the env-lookup lives in one place; each
-/// caller applies its own policy for the missing case (the writer treats it as
-/// an error because a jieba index needs a dictionary, while the reader tolerates
-/// it because paimon-java archives use the built-in tokenizer and need none).
-///
-/// In test builds, falls back to the JIEBA_TEST_DICT_DIR compile-time macro (set
-/// on the support objlib) so tests don't have to mutate process-wide env state.
-/// Defined in tantivy_defs.cpp (single TU) and mirrors LuceneUtils::
-/// GetJiebaDictionaryDir.
-std::optional<std::string> GetJiebaDictionaryDirFromEnv();
-
-} // namespace paimon::tantivy
diff --git a/src/paimon/global_index/tantivy/tantivy_equivalence_test.cpp b/src/paimon/global_index/tantivy/tantivy_equivalence_test.cpp
deleted file mode 100644
index 07f06200..00000000
--- a/src/paimon/global_index/tantivy/tantivy_equivalence_test.cpp
+++ /dev/null
@@ -1,436 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-/*
- * Equivalence + benchmark.
- *
- * EQUIVALENCE: a parametric corpus × query battery that compares lucene-fts
- * and tantivy-fulltext result *sets* (doc_id only — not score order, not score
- * values). Coverage targets:
- * - English bag-of-words: MATCH_ALL / MATCH_ANY / PHRASE / PREFIX / WILDCARD
- * - Chinese (jieba "query" mode): MATCH_ALL / MATCH_ANY / PHRASE
- * - Pre_filter intersection (no scoring)
- * PREFIX and WILDCARD equivalence covers ASCII token patterns, including case
- * normalization. Engine-specific regex edge cases are outside this test.
- *
- * BENCHMARK: build a 200-doc index per backend and time write + 100 queries.
- * Prints to stderr; never fails on perf — guarding against perf regressions
- * is out of scope here. Numbers are a reportable baseline.
- */
-
-#include <chrono>
-#include <iostream>
-#include <memory>
-#include <random>
-#include <set>
-#include <string>
-#include <vector>
-
-#include "arrow/array.h"
-#include "arrow/c/bridge.h"
-#include "arrow/ipc/api.h"
-#include "arrow/type.h"
-#include "fmt/format.h"
-#include "gtest/gtest.h"
-#include "paimon/common/utils/path_util.h"
-#include "paimon/core/global_index/global_index_file_manager.h"
-#include "paimon/core/index/index_path_factory.h"
-#include "paimon/fs/local/local_file_system.h"
-#include "paimon/global_index/bitmap_global_index_result.h"
-#include "paimon/global_index/bitmap_scored_global_index_result.h"
-#include "paimon/global_index/global_index_io_meta.h"
-#include "paimon/global_index/global_index_reader.h"
-#include "paimon/global_index/global_index_writer.h"
-#include "paimon/global_index/global_indexer.h"
-#include "paimon/global_index/global_indexer_factory.h"
-#include "paimon/global_index/lucene/lucene_defs.h"
-#include "paimon/global_index/tantivy/tantivy_defs.h"
-#include "paimon/predicate/full_text_search.h"
-#include "paimon/testing/utils/testharness.h"
-
-#ifndef JIEBA_TEST_DICT_DIR
-#error "JIEBA_TEST_DICT_DIR must be set at compile time"
-#endif
-
-namespace paimon::tantivy::test {
-
-namespace {
-
-class FakeIndexPathFactory : public IndexPathFactory {
- public:
- explicit FakeIndexPathFactory(const std::string& root) : root_(root) {}
- std::string NewPath() const override {
- assert(false);
- return "";
- }
- std::string ToPath(const std::shared_ptr<IndexFileMeta>&) const override {
- assert(false);
- return "";
- }
- std::string ToPath(const std::string& file_name) const override {
- return PathUtil::JoinPath(root_, file_name);
- }
- bool IsExternalPath() const override {
- return false;
- }
-
- private:
- std::string root_;
-};
-
-struct ReaderPair {
- std::shared_ptr<GlobalIndexReader> lucene;
- std::shared_ptr<GlobalIndexReader> tantivy;
- std::unique_ptr<paimon::test::UniqueTestDirectory> lucene_root;
- std::unique_ptr<paimon::test::UniqueTestDirectory> tantivy_root;
-};
-
-class TantivyEquivalenceTest : public ::testing::Test {
- public:
- std::unique_ptr<::ArrowSchema> CreateArrowSchema(
- const std::shared_ptr<arrow::DataType>& data_type) const {
- auto c_schema = std::make_unique<::ArrowSchema>();
- EXPECT_TRUE(arrow::ExportType(*data_type, c_schema.get()).ok());
- return c_schema;
- }
-
- GlobalIndexIOMeta WriteOne(const std::string& factory_id,
- const std::shared_ptr<arrow::DataType>& data_type,
- const std::map<std::string, std::string>& options,
- const std::shared_ptr<arrow::Array>& array,
- const std::string& root) {
- EXPECT_OK_AND_ASSIGN(auto indexer, GlobalIndexerFactory::Get(factory_id, options));
- auto path_factory = std::make_shared<FakeIndexPathFactory>(root);
- auto file_writer = std::make_shared<GlobalIndexFileManager>(
- fs_, path_factory, /*checkpoint_path_factory=*/nullptr);
- EXPECT_OK_AND_ASSIGN(
- auto writer,
- indexer->CreateWriter("f0", CreateArrowSchema(data_type).get(), file_writer, pool_));
- ::ArrowArray c_array;
- EXPECT_TRUE(arrow::ExportArray(*array, &c_array).ok());
- std::vector<int64_t> relative_row_ids(array->length());
- for (int64_t i = 0; i < array->length(); ++i) {
- relative_row_ids[i] = i;
- }
- EXPECT_OK(writer->AddBatch(&c_array, std::move(relative_row_ids)));
- EXPECT_OK_AND_ASSIGN(auto metas, writer->Finish());
- return metas[0];
- }
-
- std::shared_ptr<GlobalIndexReader> OpenOne(const std::string& factory_id,
- const std::shared_ptr<arrow::DataType>& data_type,
- const std::map<std::string, std::string>& options,
- const GlobalIndexIOMeta& meta,
- const std::string& root) {
- EXPECT_OK_AND_ASSIGN(auto indexer, GlobalIndexerFactory::Get(factory_id, options));
- auto path_factory = std::make_shared<FakeIndexPathFactory>(root);
- auto file_reader = std::make_shared<GlobalIndexFileManager>(
- fs_, path_factory, /*checkpoint_path_factory=*/nullptr);
- EXPECT_OK_AND_ASSIGN(auto reader, indexer->CreateReader(CreateArrowSchema(data_type).get(),
- file_reader, {meta}, pool_));
- return reader;
- }
-
- /// Build BOTH lucene + tantivy indexes for the same corpus + options.
- /// Returns an opened-reader pair plus owning UniqueTestDirectory handles.
- ReaderPair WriteAndOpenBoth(const std::shared_ptr<arrow::DataType>& data_type,
- const std::shared_ptr<arrow::Array>& array,
- std::map<std::string, std::string> lucene_opts,
- const std::map<std::string, std::string>& tantivy_opts) {
- auto lroot = paimon::test::UniqueTestDirectory::Create();
- auto troot = paimon::test::UniqueTestDirectory::Create();
- EXPECT_TRUE(lroot && troot);
- // lucene requires a tmp directory option; reuse lroot if caller didn't set one.
- lucene_opts.emplace("lucene-fts.write.tmp.directory", lroot->Str());
- auto lmeta = WriteOne("lucene-fts", data_type, lucene_opts, array, lroot->Str());
- auto tmeta = WriteOne("tantivy-fulltext", data_type, tantivy_opts, array, troot->Str());
- ReaderPair p;
- p.lucene = OpenOne("lucene-fts", data_type, lucene_opts, lmeta, lroot->Str());
- p.tantivy = OpenOne("tantivy-fulltext", data_type, tantivy_opts, tmeta, troot->Str());
- p.lucene_root = std::move(lroot);
- p.tantivy_root = std::move(troot);
- return p;
- }
-
- static std::set<int64_t> Ids(const std::shared_ptr<GlobalIndexResult>& result) {
- Result<const RoaringBitmap64*> br = Status::Invalid("unrecognized result type");
- if (auto scored = std::dynamic_pointer_cast<BitmapScoredGlobalIndexResult>(result)) {
- br = scored->GetBitmap();
- } else if (auto plain = std::dynamic_pointer_cast<BitmapGlobalIndexResult>(result)) {
- br = plain->GetBitmap();
- }
- EXPECT_OK_AND_ASSIGN(const RoaringBitmap64* bitmap, std::move(br));
- std::set<int64_t> out;
- if (bitmap) {
- for (auto it = bitmap->Begin(); it != bitmap->End(); ++it) {
- out.insert(static_cast<int64_t>(*it));
- }
- }
- return out;
- }
-
- /// Run a single FullTextSearch through both readers, return (lucene, tantivy)
- /// doc id sets.
- std::pair<std::set<int64_t>, std::set<int64_t>> RunPair(
- const ReaderPair& p, const std::string& q, FullTextSearch::SearchType t,
- std::optional<int32_t> limit = std::nullopt,
- std::optional<RoaringBitmap64> filter = std::nullopt) {
- auto lr = p.lucene->VisitFullTextSearch(
- std::make_shared<FullTextSearch>("f0", limit, q, t, filter));
- auto tr = p.tantivy->VisitFullTextSearch(
- std::make_shared<FullTextSearch>("f0", limit, q, t, filter));
- EXPECT_OK_AND_ASSIGN(auto lresult, std::move(lr));
- EXPECT_OK_AND_ASSIGN(auto tresult, std::move(tr));
- return {Ids(lresult), Ids(tresult)};
- }
-
- protected:
- std::shared_ptr<MemoryPool> pool_ = GetDefaultPool();
- std::shared_ptr<FileSystem> fs_ = std::make_shared<LocalFileSystem>();
-};
-
-} // namespace
-
-TEST_F(TantivyEquivalenceTest, EnglishBagOfWordsBattery) {
- auto data_type = arrow::struct_({arrow::field("f0", arrow::utf8())});
- auto array = arrow::ipc::internal::json::ArrayFromJSON(data_type, R"([
- ["alpha beta gamma delta"],
- ["alpha alpha alpha beta"],
- ["beta gamma delta epsilon"],
- ["zeta eta theta iota"],
- ["alpha gamma epsilon iota"],
- ["lone outlier word here"],
- ["alpha beta gamma alpha beta"],
- ["delta epsilon zeta eta theta"],
- ["nothing matches this row"],
- ["alpha"]
- ])")
- .ValueOrDie();
- auto pair = WriteAndOpenBoth(data_type, array, {}, {});
-
- struct Case {
- std::string query;
- FullTextSearch::SearchType type;
- std::set<int64_t> expected_ids;
- };
- std::vector<Case> cases = {
- {"alpha", FullTextSearch::SearchType::MATCH_ALL, {}},
- {"alpha", FullTextSearch::SearchType::MATCH_ANY, {}},
- {"alpha beta", FullTextSearch::SearchType::MATCH_ALL, {}},
- {"alpha beta", FullTextSearch::SearchType::MATCH_ANY, {}},
- {"alpha gamma delta", FullTextSearch::SearchType::MATCH_ALL, {}},
- {"alpha gamma delta", FullTextSearch::SearchType::MATCH_ANY, {}},
- {"epsilon iota", FullTextSearch::SearchType::MATCH_ALL, {}},
- {"alpha beta gamma", FullTextSearch::SearchType::PHRASE, {}},
- {"beta gamma delta", FullTextSearch::SearchType::PHRASE, {}},
- {"delta epsilon", FullTextSearch::SearchType::PHRASE, {}},
- {"ALP", FullTextSearch::SearchType::PREFIX, {0, 1, 4, 6, 9}},
- {"*ALPHA*", FullTextSearch::SearchType::WILDCARD, {0, 1, 4, 6, 9}},
- {"*ALP?A*", FullTextSearch::SearchType::WILDCARD, {0, 1, 4, 6, 9}},
- };
- for (const auto& c : cases) {
- auto [l, t] = RunPair(pair, c.query, c.type);
- ASSERT_EQ(l, t) << "diverge: query=" << c.query << " type=" << static_cast<int32_t>(c.type);
- if (!c.expected_ids.empty()) {
- ASSERT_EQ(l, c.expected_ids) << "unexpected matches: query=" << c.query
- << " type=" << static_cast<int32_t>(c.type);
- }
- }
-}
-
-TEST_F(TantivyEquivalenceTest, ChineseQueryModeBattery) {
- auto data_type = arrow::struct_({arrow::field("f0", arrow::utf8())});
- auto array = arrow::ipc::internal::json::ArrayFromJSON(data_type, R"([
-["智能助手 AI 模块 开发"],
-["智能助手 在 Python 开发 中"],
-["AI 助手 开发 框架"],
-["智能 模块 技术 实现"],
-["发展方向 是 智能 助手"]
- ])")
- .ValueOrDie();
- std::map<std::string, std::string> lopts = {{"lucene-fts.jieba.tokenize-mode", "query"}};
- std::map<std::string, std::string> topts = {
- {"tantivy-fulltext.tantivy.write.tokenizer", "paimon_jieba"},
- {"tantivy-fulltext.jieba.tokenize-mode", "query"},
- };
- auto pair = WriteAndOpenBoth(data_type, array, lopts, topts);
-
- struct Case {
- std::string query;
- FullTextSearch::SearchType type;
- };
- // Note: jieba is shared (same dictionary), so tokenization should agree
- // for plain Chinese text. Differences (if any) come from the lowercase /
- // stopword normalization step — tested with neutral CJK terms below.
- std::vector<Case> cases = {
- {"智能", FullTextSearch::SearchType::MATCH_ALL},
- {"智能 助手", FullTextSearch::SearchType::MATCH_ALL},
- {"模块", FullTextSearch::SearchType::MATCH_ANY},
- {"发展方向", FullTextSearch::SearchType::PHRASE},
- };
- for (const auto& c : cases) {
- auto [l, t] = RunPair(pair, c.query, c.type);
- ASSERT_EQ(l, t) << "diverge: query=" << c.query << " type=" << static_cast<int32_t>(c.type);
- }
-}
-
-TEST_F(TantivyEquivalenceTest, MixedAsciiCjkPrefixAndWildcard) {
- auto data_type = arrow::struct_({arrow::field("f0", arrow::utf8())});
- auto array = arrow::ipc::internal::json::ArrayFromJSON(data_type, R"([
- ["B超检查"],
- ["T恤"]
- ])")
- .ValueOrDie();
- std::map<std::string, std::string> lopts = {{"lucene-fts.jieba.tokenize-mode", "query"}};
- std::map<std::string, std::string> topts = {
- {"tantivy-fulltext.tantivy.write.tokenizer", "paimon_jieba"},
- {"tantivy-fulltext.jieba.tokenize-mode", "query"},
- };
- auto pair = WriteAndOpenBoth(data_type, array, lopts, topts);
-
- {
- auto [l, t] = RunPair(pair, "B", FullTextSearch::SearchType::PREFIX);
- ASSERT_EQ(l, (std::set<int64_t>{0}));
- ASSERT_EQ(t, l);
- }
- {
- auto [l, t] = RunPair(pair, "*T*", FullTextSearch::SearchType::WILDCARD);
- ASSERT_EQ(l, (std::set<int64_t>{1}));
- ASSERT_EQ(t, l);
- }
-}
-
-TEST_F(TantivyEquivalenceTest, PreFilterIntersectionEquivalent) {
- auto data_type = arrow::struct_({arrow::field("f0", arrow::utf8())});
- auto array = arrow::ipc::internal::json::ArrayFromJSON(data_type, R"([
- ["alpha beta"],
- ["alpha gamma"],
- ["alpha delta"],
- ["beta gamma"],
- ["beta delta"]
- ])")
- .ValueOrDie();
- auto pair = WriteAndOpenBoth(data_type, array, {}, {});
-
- auto pf = RoaringBitmap64::From({0l, 2l, 4l});
- {
- auto [l, t] =
- RunPair(pair, "alpha", FullTextSearch::SearchType::MATCH_ALL, std::nullopt, pf);
- ASSERT_EQ(l, t);
- ASSERT_EQ(l, (std::set<int64_t>{0, 2}));
- }
- {
- auto [l, t] =
- RunPair(pair, "beta gamma", FullTextSearch::SearchType::MATCH_ANY, std::nullopt, pf);
- ASSERT_EQ(l, t);
- }
- {
- auto empty = RoaringBitmap64();
- auto [l, t] =
- RunPair(pair, "alpha", FullTextSearch::SearchType::MATCH_ALL, std::nullopt, empty);
- ASSERT_EQ(l, t);
- ASSERT_TRUE(l.empty());
- }
-}
-
-TEST_F(TantivyEquivalenceTest, BenchmarkBuildAndQuery) {
- // Build a synthetic 200-doc corpus and time write + 100 random queries.
- // This is a reportable baseline, NOT a perf gate — assertions only check
- // semantic correctness (each query returns >= 0 docs without erroring).
- constexpr int32_t kDocCount = 200;
- constexpr int32_t kQueryCount = 100;
- std::vector<std::string> vocab = {"alpha", "beta", "gamma", "delta", "epsilon",
- "zeta", "eta", "theta", "iota", "kappa",
- "lambda", "mu", "nu", "xi", "omicron"};
- std::mt19937 rng(0xC0DE);
- std::uniform_int_distribution<size_t> word_pick(0, vocab.size() - 1);
- std::uniform_int_distribution<int32_t> word_count(3, 12);
-
- // Build the corpus as a JSON Arrow array.
- std::string json = "[";
- for (int32_t i = 0; i < kDocCount; ++i) {
- json += "[\"";
- int32_t n = word_count(rng);
- for (int32_t w = 0; w < n; ++w) {
- if (w > 0) {
- json += ' ';
- }
- json += vocab[word_pick(rng)];
- }
- json += "\"]";
- if (i + 1 < kDocCount) {
- json += ",";
- }
- }
- json += "]";
-
- auto data_type = arrow::struct_({arrow::field("f0", arrow::utf8())});
- auto array = arrow::ipc::internal::json::ArrayFromJSON(data_type, json).ValueOrDie();
-
- auto time_ms = [](auto&& fn) {
- auto t0 = std::chrono::steady_clock::now();
- fn();
- auto t1 = std::chrono::steady_clock::now();
- return std::chrono::duration_cast<std::chrono::milliseconds>(t1 - t0).count();
- };
-
- // -------- Lucene: write + open + queries --------
- auto lroot = paimon::test::UniqueTestDirectory::Create();
- std::map<std::string, std::string> lopt = {{"lucene-fts.write.tmp.directory", lroot->Str()}};
- GlobalIndexIOMeta lmeta{"", 0, nullptr};
- auto lwrite_ms =
- time_ms([&] { lmeta = WriteOne("lucene-fts", data_type, lopt, array, lroot->Str()); });
- auto lreader = OpenOne("lucene-fts", data_type, lopt, lmeta, lroot->Str());
-
- auto lquery_ms = time_ms([&] {
- for (int32_t i = 0; i < kQueryCount; ++i) {
- const std::string& w = vocab[word_pick(rng)];
- ASSERT_OK_AND_ASSIGN(
- auto r,
- lreader->VisitFullTextSearch(std::make_shared<FullTextSearch>(
- "f0", std::nullopt, w, FullTextSearch::SearchType::MATCH_ALL, std::nullopt)));
- }
- });
-
- // -------- Tantivy: write + open + queries --------
- auto troot = paimon::test::UniqueTestDirectory::Create();
- GlobalIndexIOMeta tmeta{"", 0, nullptr};
- auto twrite_ms =
- time_ms([&] { tmeta = WriteOne("tantivy-fulltext", data_type, {}, array, troot->Str()); });
- auto treader = OpenOne("tantivy-fulltext", data_type, {}, tmeta, troot->Str());
-
- auto tquery_ms = time_ms([&] {
- for (int32_t i = 0; i < kQueryCount; ++i) {
- const std::string& w = vocab[word_pick(rng)];
- ASSERT_OK_AND_ASSIGN(
- auto r,
- treader->VisitFullTextSearch(std::make_shared<FullTextSearch>(
- "f0", std::nullopt, w, FullTextSearch::SearchType::MATCH_ALL, std::nullopt)));
- }
- });
-
- std::cerr << fmt::format(
- "[STAGE10-BENCH docs={} queries={}] lucene_write={}ms lucene_query={}ms"
- " tantivy_write={}ms tantivy_query={}ms file_size_lucene={} file_size_tantivy={}\n",
- kDocCount, kQueryCount, lwrite_ms, lquery_ms, twrite_ms, tquery_ms, lmeta.file_size,
- tmeta.file_size);
- SUCCEED() << "benchmark prints to stderr";
-}
-
-} // namespace paimon::tantivy::test
diff --git a/src/paimon/global_index/tantivy/tantivy_ffi_handle.h b/src/paimon/global_index/tantivy/tantivy_ffi_handle.h
deleted file mode 100644
index 3c8ecac2..00000000
--- a/src/paimon/global_index/tantivy/tantivy_ffi_handle.h
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-#pragma once
-
-#include <cstddef>
-#include <cstdint>
-#include <memory>
-
-extern "C" {
-#include "paimon_tantivy_ffi.h" // NOLINT(build/include_subdir)
-}
-
-namespace paimon::tantivy {
-
-/// Deleter template; specialize per handle type with the matching free function.
-/// Usage:
-/// template <> struct FfiDeleter<paimon_tantivy_writer_t> {
-/// void operator()(paimon_tantivy_writer_t* p) const noexcept {
-/// paimon_tantivy_writer_free(p);
-/// }
-/// };
-/// using WriterPtr = FfiUniquePtr<paimon_tantivy_writer_t>;
-template <typename Handle>
-struct FfiDeleter {
- // Default unsupported so missing specializations fail at compile time
- void operator()(Handle*) const noexcept {
- static_assert(sizeof(Handle) == 0, "FfiDeleter must be specialized for this handle type");
- }
-};
-
-/// Generic RAII owning pointer for an FFI handle.
-template <typename Handle>
-using FfiUniquePtr = std::unique_ptr<Handle, FfiDeleter<Handle>>;
-
-/// Tokenizer handle.
-template <>
-struct FfiDeleter<PaimonJiebaTokenizer> {
- void operator()(PaimonJiebaTokenizer* p) const noexcept {
- paimon_tantivy_tokenizer_free(p);
- }
-};
-using JiebaTokenizerPtr = FfiUniquePtr<PaimonJiebaTokenizer>;
-
-/// Writer handle.
-template <>
-struct FfiDeleter<PaimonTantivyWriter> {
- void operator()(PaimonTantivyWriter* p) const noexcept {
- paimon_tantivy_writer_free(p);
- }
-};
-using WriterPtr = FfiUniquePtr<PaimonTantivyWriter>;
-
-/// Reader handle.
-template <>
-struct FfiDeleter<PaimonTantivyReader> {
- void operator()(PaimonTantivyReader* p) const noexcept {
- paimon_tantivy_reader_free(p);
- }
-};
-using ReaderPtr = FfiUniquePtr<PaimonTantivyReader>;
-
-/// Specialization: buffer_t is special - not an opaque handle but a value
-/// struct owned on the stack. The contained `data` pointer is the Rust-owned
-/// allocation; we call `paimon_tantivy_buffer_free` on the struct pointer.
-/// Use BufferGuard to ensure free-on-scope-exit even on early return.
-class BufferGuard {
- public:
- BufferGuard() noexcept {
- buf_.data = nullptr;
- buf_.len = 0;
- buf_.capacity = 0;
- }
- BufferGuard(const BufferGuard&) = delete;
- BufferGuard& operator=(const BufferGuard&) = delete;
- BufferGuard(BufferGuard&&) = delete;
- BufferGuard& operator=(BufferGuard&&) = delete;
-
- ~BufferGuard() noexcept {
- paimon_tantivy_buffer_free(&buf_);
- }
-
- PaimonTantivyBuffer* out() noexcept {
- return &buf_;
- }
-
- const uint8_t* data() const noexcept {
- return buf_.data;
- }
- std::size_t size() const noexcept {
- return buf_.len;
- }
-
- private:
- PaimonTantivyBuffer buf_{};
-};
-
-} // namespace paimon::tantivy
diff --git a/src/paimon/global_index/tantivy/tantivy_ffi_log.cpp b/src/paimon/global_index/tantivy/tantivy_ffi_log.cpp
deleted file mode 100644
index 151cc1e9..00000000
--- a/src/paimon/global_index/tantivy/tantivy_ffi_log.cpp
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#include "paimon/global_index/tantivy/tantivy_ffi_log.h"
-
-#include <cstring>
-#include <string>
-
-#include "glog/logging.h"
-
-extern "C" {
-#include "paimon_tantivy_ffi.h" // NOLINT(build/include_subdir)
-}
-
-namespace paimon::tantivy {
-
-// Not in an anonymous namespace: `extern "C"` (external linkage) and an
-// anonymous namespace (internal linkage) are contradictory. Only referenced
-// via function pointer below, so symbol visibility is not a concern.
-/// Level mapping matches Rust side (0=trace..4=error).
-extern "C" void PaimonTantivyLogAdapter(int32_t level, const char* msg, std::size_t len) {
- // msg is NOT null-terminated; slice with len.
- std::string s(msg, len);
- switch (level) {
- case 4:
- LOG(ERROR) << "[tantivy] " << s;
- break;
- case 3:
- LOG(WARNING) << "[tantivy] " << s;
- break;
- case 2:
- LOG(INFO) << "[tantivy] " << s;
- break;
- case 1:
- VLOG(1) << "[tantivy] " << s;
- break;
- case 0:
- VLOG(2) << "[tantivy] " << s;
- break;
- default:
- LOG(INFO) << "[tantivy:lvl=" << level << "] " << s;
- break;
- }
-}
-
-void InstallTantivyLogBridge() {
- paimon_tantivy_set_log_callback(&PaimonTantivyLogAdapter);
-}
-
-void UninstallTantivyLogBridge() {
- paimon_tantivy_clear_log_callback();
-}
-
-} // namespace paimon::tantivy
diff --git a/src/paimon/global_index/tantivy/tantivy_ffi_status.h b/src/paimon/global_index/tantivy/tantivy_ffi_status.h
deleted file mode 100644
index 74c0fd3e..00000000
--- a/src/paimon/global_index/tantivy/tantivy_ffi_status.h
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-#pragma once
-
-#include "fmt/format.h"
-#include "paimon/status.h"
-
-extern "C" {
-#include "paimon_tantivy_ffi.h" // NOLINT(build/include_subdir)
-}
-
-namespace paimon::tantivy {
-
-/// Translate an FFI status code to a paimon::Status. OK returns Status::OK().
-/// On error, the returned Status carries the thread-local last_error() text
-/// prefixed with the status code name for easier grep.
-///
-/// Note: cbindgen emits `PaimonTantivyStatus` in the **global** namespace as
-/// a C-style enum, so we accept it via its global type here. C++ ADL still
-/// lets call sites write the unqualified enumerator names.
-inline Status FfiStatusToStatus(::PaimonTantivyStatus code) {
- if (code == PAIMON_TANTIVY_STATUS_OK) {
- return Status::OK();
- }
- const char* err = paimon_tantivy_last_error();
- const char* name = [code]() -> const char* {
- switch (code) {
- case PAIMON_TANTIVY_STATUS_INVALID_ARGUMENT:
- return "InvalidArgument";
- case PAIMON_TANTIVY_STATUS_NOT_FOUND:
- return "NotFound";
- case PAIMON_TANTIVY_STATUS_IO_ERROR:
- return "IoError";
- case PAIMON_TANTIVY_STATUS_UNSUPPORTED:
- return "Unsupported";
- case PAIMON_TANTIVY_STATUS_TOKENIZER_ERROR:
- return "TokenizerError";
- case PAIMON_TANTIVY_STATUS_QUERY_PARSE_ERROR:
- return "QueryParseError";
- case PAIMON_TANTIVY_STATUS_INDEX_FORMAT_ERROR:
- return "IndexFormatError";
- case PAIMON_TANTIVY_STATUS_INTERNAL_ERROR:
- return "InternalError";
- default:
- return "UnknownFfiStatus";
- }
- }();
- std::string msg = fmt::format("tantivy-ffi[{}({})]: {}", name, static_cast<int32_t>(code),
- err ? err : "(null)");
- switch (code) {
- case PAIMON_TANTIVY_STATUS_NOT_FOUND:
- return Status::NotExist(msg);
- case PAIMON_TANTIVY_STATUS_IO_ERROR:
- return Status::IOError(msg);
- case PAIMON_TANTIVY_STATUS_UNSUPPORTED:
- return Status::NotImplemented(msg);
- case PAIMON_TANTIVY_STATUS_INVALID_ARGUMENT:
- case PAIMON_TANTIVY_STATUS_TOKENIZER_ERROR:
- case PAIMON_TANTIVY_STATUS_QUERY_PARSE_ERROR:
- case PAIMON_TANTIVY_STATUS_INDEX_FORMAT_ERROR:
- return Status::Invalid(msg);
- default:
- return Status::UnknownError(msg);
- }
-}
-
-/// Like PAIMON_RETURN_NOT_OK but for FFI calls returning PaimonTantivyStatus.
-#define PAIMON_TANTIVY_RETURN_NOT_OK(expr) \
- do { \
- ::PaimonTantivyStatus _paimon_tantivy_status_ = (expr); \
- if (_paimon_tantivy_status_ != PAIMON_TANTIVY_STATUS_OK) { \
- return ::paimon::tantivy::FfiStatusToStatus(_paimon_tantivy_status_); \
- } \
- } while (0)
-
-} // namespace paimon::tantivy
diff --git a/src/paimon/global_index/tantivy/tantivy_ffi_test.cpp b/src/paimon/global_index/tantivy/tantivy_ffi_test.cpp
deleted file mode 100644
index a10f3231..00000000
--- a/src/paimon/global_index/tantivy/tantivy_ffi_test.cpp
+++ /dev/null
@@ -1,138 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#include <atomic>
-#include <cstring>
-#include <string>
-#include <thread>
-#include <vector>
-
-#include "gtest/gtest.h"
-#include "paimon/global_index/tantivy/tantivy_ffi_handle.h"
-#include "paimon/global_index/tantivy/tantivy_ffi_log.h"
-#include "paimon/global_index/tantivy/tantivy_ffi_status.h"
-
-extern "C" {
-#include "paimon_tantivy_ffi.h" // NOLINT(build/include_subdir)
-}
-
-namespace paimon::tantivy::test {
-
-// ------------------------- last_error contract -------------------------
-
-TEST(TantivyFfiError, LastErrorIsNeverNull) {
- // Before anything, last_error should be a valid non-null pointer to ""
- const char* ptr = paimon_tantivy_last_error();
- ASSERT_NE(ptr, nullptr);
- // Content is thread-local; for freshly-spawned thread it must be empty
- std::atomic<bool> child_ok{false};
- std::thread t([&]() {
- const char* p = paimon_tantivy_last_error();
- child_ok.store(p != nullptr && p[0] == '\0');
- });
- t.join();
- ASSERT_TRUE(child_ok.load());
-}
-
-// ------------------------- status translation -------------------------
-
-TEST(TantivyFfiStatus, OkTranslates) {
- Status s = FfiStatusToStatus(PaimonTantivyStatus::PAIMON_TANTIVY_STATUS_OK);
- ASSERT_TRUE(s.ok()) << s.ToString();
-}
-
-TEST(TantivyFfiStatus, ErrorCodeNamesShowUp) {
- // Translate a few codes and ensure the name appears in the string form.
- struct Case {
- PaimonTantivyStatus code;
- const char* expected_substr;
- };
- const Case cases[] = {
- {PaimonTantivyStatus::PAIMON_TANTIVY_STATUS_INVALID_ARGUMENT, "InvalidArgument"},
- {PaimonTantivyStatus::PAIMON_TANTIVY_STATUS_NOT_FOUND, "NotFound"},
- {PaimonTantivyStatus::PAIMON_TANTIVY_STATUS_IO_ERROR, "IoError"},
- {PaimonTantivyStatus::PAIMON_TANTIVY_STATUS_UNSUPPORTED, "Unsupported"},
- {PaimonTantivyStatus::PAIMON_TANTIVY_STATUS_TOKENIZER_ERROR, "TokenizerError"},
- };
- for (const auto& c : cases) {
- Status s = FfiStatusToStatus(c.code);
- ASSERT_FALSE(s.ok());
- ASSERT_NE(s.ToString().find(c.expected_substr), std::string::npos)
- << "got: " << s.ToString();
- }
-}
-
-// ------------------------- buffer lifetime -------------------------
-
-TEST(TantivyFfiBuffer, EmptyBufferGuard) {
- BufferGuard g;
- ASSERT_EQ(g.size(), 0u);
- ASSERT_EQ(g.data(), nullptr);
- // Destructor must accept empty buffer
-}
-
-// ------------------------- handle stress -------------------------
-
-// Sanity stress: create/destroy a dummy "handle" via into_handle/free_handle.
-// Since the Rust side doesn't yet export writer/reader, we stress via a
-// temporary wrapping of the buffer API: alloc buffers repeatedly, ensure no
-// crash (LSAN / ASAN would catch leaks).
-TEST(TantivyFfiBuffer, StressAllocFree) {
- for (int32_t i = 0; i < 1000; ++i) {
- BufferGuard g;
- // We don't have a way to populate the buffer from C++ here;
- // this just exercises empty construction + destruction path.
... 3569 lines suppressed ...