You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by lz...@apache.org on 2026/07/04 13:21:27 UTC
(paimon-full-text) 01/01: Initialize Paimon full text index
This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch codex/paimon-full-text-ci
in repository https://gitbox.apache.org/repos/asf/paimon-full-text.git
commit 998666fc1ad8ea00a20a3bb35f486bccc5df149d
Author: JingsongLi <ji...@gmail.com>
AuthorDate: Sat Jul 4 21:21:08 2026 +0800
Initialize Paimon full text index
---
.github/workflows/ci.yml | 109 ++
.gitignore | 9 +
Cargo.lock | 1456 ++++++++++++++++++++
Cargo.toml | 18 +
README.md | 90 +-
core/Cargo.toml | 17 +
core/src/config.rs | 43 +
core/src/error.rs | 24 +
core/src/index.rs | 384 ++++++
core/src/io.rs | 80 ++
core/src/lib.rs | 13 +
core/src/query.rs | 93 ++
core/src/storage.rs | 66 +
core/src/tokenizer.rs | 167 +++
core/tests/core_roundtrip.rs | 68 +
docs/designs/2026-07-04-paimon-full-text-index.md | 445 ++++++
docs/java-api.md | 35 +
docs/paimon-integration.md | 32 +
...-07-04-paimon-full-text-index-implementation.md | 65 +
docs/python-api.md | 35 +
docs/storage-format.md | 24 +
ffi/Cargo.toml | 18 +
ffi/src/lib.rs | 398 ++++++
include/paimon_ftindex.h | 60 +
java/pom.xml | 17 +
.../paimon/index/fulltext/FullTextIndexInput.java | 8 +
.../paimon/index/fulltext/FullTextIndexOutput.java | 10 +
.../paimon/index/fulltext/FullTextIndexReader.java | 36 +
.../paimon/index/fulltext/FullTextIndexWriter.java | 53 +
.../paimon/index/fulltext/FullTextNative.java | 29 +
.../paimon/index/fulltext/FullTextQuery.java | 75 +
.../index/fulltext/FullTextSearchResult.java | 24 +
jni/Cargo.toml | 19 +
jni/src/lib.rs | 334 +++++
python/paimon_ftindex/__init__.py | 11 +
python/paimon_ftindex/_ffi.py | 122 ++
python/paimon_ftindex/query.py | 43 +
python/paimon_ftindex/reader.py | 69 +
python/paimon_ftindex/writer.py | 81 ++
python/pyproject.toml | 16 +
python/setup.py | 3 +
python/tests/test_roundtrip.py | 25 +
42 files changed, 4723 insertions(+), 1 deletion(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
new file mode 100644
index 0000000..6111495
--- /dev/null
+++ b/.github/workflows/ci.yml
@@ -0,0 +1,109 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: CI
+
+on:
+ push:
+ branches:
+ - main
+ - release-*
+ pull_request:
+ branches:
+ - main
+ - release-*
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ rust:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Rust cache
+ uses: actions/cache@v4
+ with:
+ path: |
+ ~/.cargo/registry
+ ~/.cargo/git
+ target
+ key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+ restore-keys: |
+ ${{ runner.os }}-cargo-
+
+ - name: Format
+ run: cargo fmt --all -- --check
+
+ - name: Clippy
+ run: cargo clippy --all-targets --workspace -- -D warnings
+
+ - name: Test Rust workspace
+ run: cargo test --workspace
+
+ java:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Set up JDK 8
+ uses: actions/setup-java@v4
+ with:
+ distribution: temurin
+ java-version: '8'
+ cache: maven
+
+ - name: Test Java API
+ run: mvn -q -f java/pom.xml test
+
+ python:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Rust cache
+ uses: actions/cache@v4
+ with:
+ path: |
+ ~/.cargo/registry
+ ~/.cargo/git
+ target
+ key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+ restore-keys: |
+ ${{ runner.os }}-cargo-
+
+ - name: Set up Python
+ uses: actions/setup-python@v5
+ with:
+ python-version: '3.12'
+
+ - name: Build FFI library
+ run: cargo build --release -p paimon-ftindex-ffi
+
+ - name: Test Python API
+ working-directory: python
+ run: |
+ python -m pip install --upgrade pip
+ python -m pip install -e ".[test]"
+ python -m pytest -q tests
+ env:
+ PAIMON_FTINDEX_LIB_PATH: ${{ github.workspace }}/target/release
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..bb825d1
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,9 @@
+/target/
+/java/target/
+**/*.egg-info/
+**/build/
+**/dist/
+**/.pytest_cache/
+**/__pycache__/
+*.py[cod]
+.DS_Store
diff --git a/Cargo.lock b/Cargo.lock
new file mode 100644
index 0000000..35b4f2b
--- /dev/null
+++ b/Cargo.lock
@@ -0,0 +1,1456 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 3
+
+[[package]]
+name = "aho-corasick"
+version = "1.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301"
+dependencies = [
+ "memchr",
+]
+
+[[package]]
+name = "allocator-api2"
+version = "0.2.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923"
+
+[[package]]
+name = "anyhow"
+version = "1.0.103"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3"
+
+[[package]]
+name = "arc-swap"
+version = "1.9.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b"
+dependencies = [
+ "rustversion",
+]
+
+[[package]]
+name = "async-trait"
+version = "0.1.89"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "autocfg"
+version = "1.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
+
+[[package]]
+name = "base64"
+version = "0.22.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
+
+[[package]]
+name = "bitflags"
+version = "2.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8"
+
+[[package]]
+name = "bitpacking"
+version = "0.9.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "96a7139abd3d9cebf8cd6f920a389cf3dc9576172e32f4563f188cae3c3eb019"
+dependencies = [
+ "crunchy",
+]
+
+[[package]]
+name = "bumpalo"
+version = "3.20.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649"
+
+[[package]]
+name = "byteorder"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
+
+[[package]]
+name = "bytes"
+version = "1.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593"
+
+[[package]]
+name = "cc"
+version = "1.2.65"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96"
+dependencies = [
+ "find-msvc-tools",
+ "jobserver",
+ "libc",
+ "shlex",
+]
+
+[[package]]
+name = "census"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4f4c707c6a209cbe82d10abd08e1ea8995e9ea937d2550646e02798948992be0"
+
+[[package]]
+name = "cesu8"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c"
+
+[[package]]
+name = "cfg-if"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
+
+[[package]]
+name = "combine"
+version = "4.6.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd"
+dependencies = [
+ "bytes",
+ "memchr",
+]
+
+[[package]]
+name = "crc32fast"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511"
+dependencies = [
+ "cfg-if",
+]
+
+[[package]]
+name = "crossbeam-channel"
+version = "0.5.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2"
+dependencies = [
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-deque"
+version = "0.8.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51"
+dependencies = [
+ "crossbeam-epoch",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-epoch"
+version = "0.9.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
+dependencies = [
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-utils"
+version = "0.8.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"
+
+[[package]]
+name = "crunchy"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5"
+
+[[package]]
+name = "deranged"
+version = "0.5.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c"
+dependencies = [
+ "serde_core",
+]
+
+[[package]]
+name = "downcast-rs"
+version = "1.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2"
+
+[[package]]
+name = "either"
+version = "1.16.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e"
+
+[[package]]
+name = "equivalent"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
+
+[[package]]
+name = "errno"
+version = "0.3.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
+dependencies = [
+ "libc",
+ "windows-sys 0.61.2",
+]
+
+[[package]]
+name = "fastdivide"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9afc2bd4d5a73106dd53d10d73d3401c2f32730ba2c0b93ddb888a8983680471"
+
+[[package]]
+name = "fastrand"
+version = "2.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6"
+
+[[package]]
+name = "find-msvc-tools"
+version = "0.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
+
+[[package]]
+name = "fnv"
+version = "1.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
+
+[[package]]
+name = "foldhash"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
+
+[[package]]
+name = "fs4"
+version = "0.8.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f7e180ac76c23b45e767bd7ae9579bc0bb458618c4bc71835926e098e61d15f8"
+dependencies = [
+ "rustix 0.38.44",
+ "windows-sys 0.52.0",
+]
+
+[[package]]
+name = "futures-core"
+version = "0.3.32"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d"
+
+[[package]]
+name = "futures-task"
+version = "0.3.32"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393"
+
+[[package]]
+name = "futures-util"
+version = "0.3.32"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6"
+dependencies = [
+ "futures-core",
+ "futures-task",
+ "pin-project-lite",
+ "slab",
+]
+
+[[package]]
+name = "getrandom"
+version = "0.2.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "wasi",
+]
+
+[[package]]
+name = "getrandom"
+version = "0.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "r-efi 5.3.0",
+ "wasip2",
+]
+
+[[package]]
+name = "getrandom"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "r-efi 6.0.0",
+]
+
+[[package]]
+name = "hashbrown"
+version = "0.15.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1"
+dependencies = [
+ "allocator-api2",
+ "equivalent",
+ "foldhash",
+]
+
+[[package]]
+name = "hermit-abi"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c"
+
+[[package]]
+name = "htmlescape"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e9025058dae765dee5070ec375f591e2ba14638c63feff74f13805a72e523163"
+
+[[package]]
+name = "instant"
+version = "0.1.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222"
+dependencies = [
+ "cfg-if",
+ "js-sys",
+ "wasm-bindgen",
+ "web-sys",
+]
+
+[[package]]
+name = "itertools"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569"
+dependencies = [
+ "either",
+]
+
+[[package]]
+name = "itoa"
+version = "1.0.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
+
+[[package]]
+name = "jni"
+version = "0.21.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97"
+dependencies = [
+ "cesu8",
+ "cfg-if",
+ "combine",
+ "jni-sys 0.3.1",
+ "log",
+ "thiserror",
+ "walkdir",
+ "windows-sys 0.45.0",
+]
+
+[[package]]
+name = "jni-sys"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "41a652e1f9b6e0275df1f15b32661cf0d4b78d4d87ddec5e0c3c20f097433258"
+dependencies = [
+ "jni-sys 0.4.1",
+]
+
+[[package]]
+name = "jni-sys"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2"
+dependencies = [
+ "jni-sys-macros",
+]
+
+[[package]]
+name = "jni-sys-macros"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264"
+dependencies = [
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "jobserver"
+version = "0.1.34"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33"
+dependencies = [
+ "getrandom 0.3.4",
+ "libc",
+]
+
+[[package]]
+name = "js-sys"
+version = "0.3.103"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102"
+dependencies = [
+ "cfg-if",
+ "futures-util",
+ "wasm-bindgen",
+]
+
+[[package]]
+name = "levenshtein_automata"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0c2cdeb66e45e9f36bfad5bbdb4d2384e70936afbee843c6f6543f0c551ebb25"
+
+[[package]]
+name = "libc"
+version = "0.2.186"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
+
+[[package]]
+name = "libm"
+version = "0.2.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981"
+
+[[package]]
+name = "linux-raw-sys"
+version = "0.4.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab"
+
+[[package]]
+name = "linux-raw-sys"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53"
+
+[[package]]
+name = "log"
+version = "0.4.33"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad"
+
+[[package]]
+name = "lru"
+version = "0.12.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38"
+dependencies = [
+ "hashbrown",
+]
+
+[[package]]
+name = "lz4_flex"
+version = "0.11.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "373f5eceeeab7925e0c1098212f2fbc4d416adec9d35051a6ab251e824c1854a"
+
+[[package]]
+name = "measure_time"
+version = "0.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dbefd235b0aadd181626f281e1d684e116972988c14c264e42069d5e8a5775cc"
+dependencies = [
+ "instant",
+ "log",
+]
+
+[[package]]
+name = "memchr"
+version = "2.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4"
+
+[[package]]
+name = "memmap2"
+version = "0.9.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0"
+dependencies = [
+ "libc",
+]
+
+[[package]]
+name = "minimal-lexical"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
+
+[[package]]
+name = "murmurhash32"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2195bf6aa996a481483b29d62a7663eed3fe39600c460e323f8ff41e90bdd89b"
+
+[[package]]
+name = "nom"
+version = "7.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"
+dependencies = [
+ "memchr",
+ "minimal-lexical",
+]
+
+[[package]]
+name = "num-conv"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441"
+
+[[package]]
+name = "num-traits"
+version = "0.2.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
+dependencies = [
+ "autocfg",
+ "libm",
+]
+
+[[package]]
+name = "num_cpus"
+version = "1.17.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b"
+dependencies = [
+ "hermit-abi",
+ "libc",
+]
+
+[[package]]
+name = "once_cell"
+version = "1.21.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
+
+[[package]]
+name = "oneshot"
+version = "0.1.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "269bca4c2591a28585d6bf10d9ed0332b7d76900a1b02bec41bdc3a2cdcda107"
+
+[[package]]
+name = "ownedbytes"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c3a059efb063b8f425b948e042e6b9bd85edfe60e913630ed727b23e2dfcc558"
+dependencies = [
+ "stable_deref_trait",
+]
+
+[[package]]
+name = "paimon-ftindex-core"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "serde",
+ "serde_json",
+ "tantivy",
+ "tempfile",
+ "thiserror",
+]
+
+[[package]]
+name = "paimon-ftindex-ffi"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "paimon-ftindex-core",
+ "serde_json",
+]
+
+[[package]]
+name = "paimon-ftindex-jni"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "jni",
+ "paimon-ftindex-core",
+ "serde_json",
+]
+
+[[package]]
+name = "pin-project-lite"
+version = "0.2.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd"
+
+[[package]]
+name = "pkg-config"
+version = "0.3.33"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e"
+
+[[package]]
+name = "powerfmt"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
+
+[[package]]
+name = "ppv-lite86"
+version = "0.2.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9"
+dependencies = [
+ "zerocopy",
+]
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.106"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.46"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "r-efi"
+version = "5.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
+
+[[package]]
+name = "r-efi"
+version = "6.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf"
+
+[[package]]
+name = "rand"
+version = "0.8.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a"
+dependencies = [
+ "libc",
+ "rand_chacha",
+ "rand_core",
+]
+
+[[package]]
+name = "rand_chacha"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
+dependencies = [
+ "ppv-lite86",
+ "rand_core",
+]
+
+[[package]]
+name = "rand_core"
+version = "0.6.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
+dependencies = [
+ "getrandom 0.2.17",
+]
+
+[[package]]
+name = "rand_distr"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31"
+dependencies = [
+ "num-traits",
+ "rand",
+]
+
+[[package]]
+name = "rayon"
+version = "1.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d"
+dependencies = [
+ "either",
+ "rayon-core",
+]
+
+[[package]]
+name = "rayon-core"
+version = "1.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91"
+dependencies = [
+ "crossbeam-deque",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "regex"
+version = "1.12.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-automata",
+ "regex-syntax",
+]
+
+[[package]]
+name = "regex-automata"
+version = "0.4.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-syntax",
+]
+
+[[package]]
+name = "regex-syntax"
+version = "0.8.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
+
+[[package]]
+name = "rust-stemmers"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e46a2036019fdb888131db7a4c847a1063a7493f971ed94ea82c67eada63ca54"
+dependencies = [
+ "serde",
+ "serde_derive",
+]
+
+[[package]]
+name = "rustc-hash"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
+
+[[package]]
+name = "rustix"
+version = "0.38.44"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154"
+dependencies = [
+ "bitflags",
+ "errno",
+ "libc",
+ "linux-raw-sys 0.4.15",
+ "windows-sys 0.59.0",
+]
+
+[[package]]
+name = "rustix"
+version = "1.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190"
+dependencies = [
+ "bitflags",
+ "errno",
+ "libc",
+ "linux-raw-sys 0.12.1",
+ "windows-sys 0.61.2",
+]
+
+[[package]]
+name = "rustversion"
+version = "1.0.22"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d"
+
+[[package]]
+name = "same-file"
+version = "1.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
+dependencies = [
+ "winapi-util",
+]
+
+[[package]]
+name = "serde"
+version = "1.0.228"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e"
+dependencies = [
+ "serde_core",
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_core"
+version = "1.0.228"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad"
+dependencies = [
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_derive"
+version = "1.0.228"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "serde_json"
+version = "1.0.150"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9"
+dependencies = [
+ "itoa",
+ "memchr",
+ "serde",
+ "serde_core",
+ "zmij",
+]
+
+[[package]]
+name = "shlex"
+version = "2.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba"
+
+[[package]]
+name = "sketches-ddsketch"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "85636c14b73d81f541e525f585c0a2109e6744e1565b5c1668e31c70c10ed65c"
+dependencies = [
+ "serde",
+]
+
+[[package]]
+name = "slab"
+version = "0.4.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5"
+
+[[package]]
+name = "smallvec"
+version = "1.15.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90"
+
+[[package]]
+name = "stable_deref_trait"
+version = "1.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596"
+
+[[package]]
+name = "syn"
+version = "2.0.118"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "tantivy"
+version = "0.22.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "96599ea6fccd844fc833fed21d2eecac2e6a7c1afd9e044057391d78b1feb141"
+dependencies = [
+ "aho-corasick",
+ "arc-swap",
+ "base64",
+ "bitpacking",
+ "byteorder",
+ "census",
+ "crc32fast",
+ "crossbeam-channel",
+ "downcast-rs",
+ "fastdivide",
+ "fnv",
+ "fs4",
+ "htmlescape",
+ "itertools",
+ "levenshtein_automata",
+ "log",
+ "lru",
+ "lz4_flex",
+ "measure_time",
+ "memmap2",
+ "num_cpus",
+ "once_cell",
+ "oneshot",
+ "rayon",
+ "regex",
+ "rust-stemmers",
+ "rustc-hash",
+ "serde",
+ "serde_json",
+ "sketches-ddsketch",
+ "smallvec",
+ "tantivy-bitpacker",
+ "tantivy-columnar",
+ "tantivy-common",
+ "tantivy-fst",
+ "tantivy-query-grammar",
+ "tantivy-stacker",
+ "tantivy-tokenizer-api",
+ "tempfile",
+ "thiserror",
+ "time",
+ "uuid",
+ "winapi",
+]
+
+[[package]]
+name = "tantivy-bitpacker"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "284899c2325d6832203ac6ff5891b297fc5239c3dc754c5bc1977855b23c10df"
+dependencies = [
+ "bitpacking",
+]
+
+[[package]]
+name = "tantivy-columnar"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "12722224ffbe346c7fec3275c699e508fd0d4710e629e933d5736ec524a1f44e"
+dependencies = [
+ "downcast-rs",
+ "fastdivide",
+ "itertools",
+ "serde",
+ "tantivy-bitpacker",
+ "tantivy-common",
+ "tantivy-sstable",
+ "tantivy-stacker",
+]
+
+[[package]]
+name = "tantivy-common"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8019e3cabcfd20a1380b491e13ff42f57bb38bf97c3d5fa5c07e50816e0621f4"
+dependencies = [
+ "async-trait",
+ "byteorder",
+ "ownedbytes",
+ "serde",
+ "time",
+]
+
+[[package]]
+name = "tantivy-fst"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d60769b80ad7953d8a7b2c70cdfe722bbcdcac6bccc8ac934c40c034d866fc18"
+dependencies = [
+ "byteorder",
+ "regex-syntax",
+ "utf8-ranges",
+]
+
+[[package]]
+name = "tantivy-query-grammar"
+version = "0.22.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "847434d4af57b32e309f4ab1b4f1707a6c566656264caa427ff4285c4d9d0b82"
+dependencies = [
+ "nom",
+]
+
+[[package]]
+name = "tantivy-sstable"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c69578242e8e9fc989119f522ba5b49a38ac20f576fc778035b96cc94f41f98e"
+dependencies = [
+ "tantivy-bitpacker",
+ "tantivy-common",
+ "tantivy-fst",
+ "zstd",
+]
+
+[[package]]
+name = "tantivy-stacker"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c56d6ff5591fc332739b3ce7035b57995a3ce29a93ffd6012660e0949c956ea8"
+dependencies = [
+ "murmurhash32",
+ "rand_distr",
+ "tantivy-common",
+]
+
+[[package]]
+name = "tantivy-tokenizer-api"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2a0dcade25819a89cfe6f17d932c9cedff11989936bf6dd4f336d50392053b04"
+dependencies = [
+ "serde",
+]
+
+[[package]]
+name = "tempfile"
+version = "3.27.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
+dependencies = [
+ "fastrand",
+ "getrandom 0.4.3",
+ "once_cell",
+ "rustix 1.1.4",
+ "windows-sys 0.61.2",
+]
+
+[[package]]
+name = "thiserror"
+version = "1.0.69"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52"
+dependencies = [
+ "thiserror-impl",
+]
+
+[[package]]
+name = "thiserror-impl"
+version = "1.0.69"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "time"
+version = "0.3.53"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "18dfaaeddcb932337b5e7866ee7d0ce9b76d2fd092997146f187ec09b4558a50"
+dependencies = [
+ "deranged",
+ "num-conv",
+ "powerfmt",
+ "serde_core",
+ "time-core",
+ "time-macros",
+]
+
+[[package]]
+name = "time-core"
+version = "0.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109"
+
+[[package]]
+name = "time-macros"
+version = "0.2.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c431b87111666e491a90baa837f914fb45cd5dc3c268591b0220ff5057f2085f"
+dependencies = [
+ "num-conv",
+ "time-core",
+]
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.24"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
+
+[[package]]
+name = "utf8-ranges"
+version = "1.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7fcfc827f90e53a02eaef5e535ee14266c1d569214c6aa70133a624d8a3164ba"
+
+[[package]]
+name = "uuid"
+version = "1.23.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bf80a72845275afea99e7f2b434723d3bc7e38470fcd1c7ed39a599c73319a53"
+dependencies = [
+ "getrandom 0.4.3",
+ "js-sys",
+ "serde_core",
+ "wasm-bindgen",
+]
+
+[[package]]
+name = "walkdir"
+version = "2.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b"
+dependencies = [
+ "same-file",
+ "winapi-util",
+]
+
+[[package]]
+name = "wasi"
+version = "0.11.1+wasi-snapshot-preview1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b"
+
+[[package]]
+name = "wasip2"
+version = "1.0.4+wasi-0.2.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487"
+dependencies = [
+ "wit-bindgen",
+]
+
+[[package]]
+name = "wasm-bindgen"
+version = "0.2.126"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4"
+dependencies = [
+ "cfg-if",
+ "once_cell",
+ "rustversion",
+ "wasm-bindgen-macro",
+ "wasm-bindgen-shared",
+]
+
+[[package]]
+name = "wasm-bindgen-macro"
+version = "0.2.126"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1"
+dependencies = [
+ "quote",
+ "wasm-bindgen-macro-support",
+]
+
+[[package]]
+name = "wasm-bindgen-macro-support"
+version = "0.2.126"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e"
+dependencies = [
+ "bumpalo",
+ "proc-macro2",
+ "quote",
+ "syn",
+ "wasm-bindgen-shared",
+]
+
+[[package]]
+name = "wasm-bindgen-shared"
+version = "0.2.126"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "web-sys"
+version = "0.3.103"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141"
+dependencies = [
+ "js-sys",
+ "wasm-bindgen",
+]
+
+[[package]]
+name = "winapi"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
+dependencies = [
+ "winapi-i686-pc-windows-gnu",
+ "winapi-x86_64-pc-windows-gnu",
+]
+
+[[package]]
+name = "winapi-i686-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
+
+[[package]]
+name = "winapi-util"
+version = "0.1.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
+dependencies = [
+ "windows-sys 0.61.2",
+]
+
+[[package]]
+name = "winapi-x86_64-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
+
+[[package]]
+name = "windows-link"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
+
+[[package]]
+name = "windows-sys"
+version = "0.45.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0"
+dependencies = [
+ "windows-targets 0.42.2",
+]
+
+[[package]]
+name = "windows-sys"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
+dependencies = [
+ "windows-targets 0.52.6",
+]
+
+[[package]]
+name = "windows-sys"
+version = "0.59.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
+dependencies = [
+ "windows-targets 0.52.6",
+]
+
+[[package]]
+name = "windows-sys"
+version = "0.61.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc"
+dependencies = [
+ "windows-link",
+]
+
+[[package]]
+name = "windows-targets"
+version = "0.42.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071"
+dependencies = [
+ "windows_aarch64_gnullvm 0.42.2",
+ "windows_aarch64_msvc 0.42.2",
+ "windows_i686_gnu 0.42.2",
+ "windows_i686_msvc 0.42.2",
+ "windows_x86_64_gnu 0.42.2",
+ "windows_x86_64_gnullvm 0.42.2",
+ "windows_x86_64_msvc 0.42.2",
+]
+
+[[package]]
+name = "windows-targets"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
+dependencies = [
+ "windows_aarch64_gnullvm 0.52.6",
+ "windows_aarch64_msvc 0.52.6",
+ "windows_i686_gnu 0.52.6",
+ "windows_i686_gnullvm",
+ "windows_i686_msvc 0.52.6",
+ "windows_x86_64_gnu 0.52.6",
+ "windows_x86_64_gnullvm 0.52.6",
+ "windows_x86_64_msvc 0.52.6",
+]
+
+[[package]]
+name = "windows_aarch64_gnullvm"
+version = "0.42.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8"
+
+[[package]]
+name = "windows_aarch64_gnullvm"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
+
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.42.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43"
+
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
+
+[[package]]
+name = "windows_i686_gnu"
+version = "0.42.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f"
+
+[[package]]
+name = "windows_i686_gnu"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
+
+[[package]]
+name = "windows_i686_gnullvm"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
+
+[[package]]
+name = "windows_i686_msvc"
+version = "0.42.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060"
+
+[[package]]
+name = "windows_i686_msvc"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
+
+[[package]]
+name = "windows_x86_64_gnu"
+version = "0.42.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36"
+
+[[package]]
+name = "windows_x86_64_gnu"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
+
+[[package]]
+name = "windows_x86_64_gnullvm"
+version = "0.42.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3"
+
+[[package]]
+name = "windows_x86_64_gnullvm"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
+
+[[package]]
+name = "windows_x86_64_msvc"
+version = "0.42.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0"
+
+[[package]]
+name = "windows_x86_64_msvc"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
+
+[[package]]
+name = "wit-bindgen"
+version = "0.57.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e"
+
+[[package]]
+name = "zerocopy"
+version = "0.8.52"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f"
+dependencies = [
+ "zerocopy-derive",
+]
+
+[[package]]
+name = "zerocopy-derive"
+version = "0.8.52"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "zmij"
+version = "1.0.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa"
+
+[[package]]
+name = "zstd"
+version = "0.13.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a"
+dependencies = [
+ "zstd-safe",
+]
+
+[[package]]
+name = "zstd-safe"
+version = "7.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d"
+dependencies = [
+ "zstd-sys",
+]
+
+[[package]]
+name = "zstd-sys"
+version = "2.0.16+zstd.1.5.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748"
+dependencies = [
+ "cc",
+ "pkg-config",
+]
diff --git a/Cargo.toml b/Cargo.toml
new file mode 100644
index 0000000..1e0c7ec
--- /dev/null
+++ b/Cargo.toml
@@ -0,0 +1,18 @@
+[workspace]
+members = ["core", "ffi", "jni"]
+resolver = "2"
+
+[workspace.package]
+edition = "2021"
+license = "Apache-2.0"
+repository = "https://github.com/apache/paimon-full-text"
+rust-version = "1.75"
+
+[workspace.dependencies]
+anyhow = "1"
+jni = "0.21"
+serde = { version = "1", features = ["derive"] }
+serde_json = "1"
+tantivy = "0.22"
+tempfile = "3"
+thiserror = "1"
diff --git a/README.md b/README.md
index 9e46b54..6c0f2a8 100644
--- a/README.md
+++ b/README.md
@@ -1 +1,89 @@
-# paimon-full-text
+# Apache Paimon Full Text Index
+
+Standalone Tantivy-based full-text index library for Apache Paimon-style data
+lake storage. The project follows the same shape as `paimon-vector-index`:
+
+- `core`: Rust implementation and v1 storage format.
+- `ffi`: C ABI over the Rust core.
+- `jni`: Java JNI bridge over the Rust core.
+- `java`: public Java API.
+- `python`: Python ctypes API over the C ABI.
+
+The index file is self-describing. Readers only need positional `read_at` I/O
+and do not depend on Paimon manifest metadata.
+
+## Current Status
+
+Implemented:
+
+- Rust writer, reader, v1 envelope, and search.
+- C FFI writer/reader/search JSON.
+- Java API and JNI bridge.
+- Python ctypes package.
+- Cross-boundary round-trip tests for Rust core, FFI, and Python.
+
+Supported tokenizers in this first implementation:
+
+- `default`
+- `simple`
+- `whitespace`
+- `raw`
+- `ngram`
+
+Reserved for follow-up:
+
+- `jieba`
+- stemming
+- built-in and custom stop-word filters
+- true seek-on-demand Tantivy directory instead of loading segment files into
+ memory at reader open
+
+## Build
+
+```bash
+cargo test -p paimon-ftindex-core
+cargo test -p paimon-ftindex-ffi
+cargo build -p paimon-ftindex-ffi
+cargo check -p paimon-ftindex-jni
+mvn -q -f java/pom.xml test
+PYTHONPATH=python python3 -m pytest -q python/tests
+```
+
+## Rust Example
+
+```rust
+use paimon_ftindex_core::io::{PosWriter, SliceReader};
+use paimon_ftindex_core::{
+ FullTextIndexConfig, FullTextIndexReader, FullTextIndexWriter, FullTextQuery,
+};
+
+let mut writer = FullTextIndexWriter::new(FullTextIndexConfig::new())?;
+writer.add_document(1, "Apache Paimon full text search")?;
+
+let mut bytes = Vec::new();
+writer.write(&mut PosWriter::new(&mut bytes))?;
+
+let mut reader = FullTextIndexReader::open(SliceReader::new(bytes))?;
+let result = reader.search(FullTextQuery::match_query("paimon", "text"), 10)?;
+```
+
+## Python Example
+
+```python
+from io import BytesIO
+from paimon_ftindex import FullTextIndexReader, FullTextIndexWriter, MatchQuery
+
+out = BytesIO()
+with FullTextIndexWriter() as writer:
+ writer.add_document(1, "Apache Paimon full text search")
+ writer.write(out)
+
+class Input:
+ def __init__(self, data):
+ self.data = data
+ def pread(self, pos, length):
+ return self.data[pos:pos + length]
+
+with FullTextIndexReader(Input(out.getvalue())) as reader:
+ ids, scores = reader.search(MatchQuery("paimon"), limit=10)
+```
diff --git a/core/Cargo.toml b/core/Cargo.toml
new file mode 100644
index 0000000..a1124c0
--- /dev/null
+++ b/core/Cargo.toml
@@ -0,0 +1,17 @@
+[package]
+name = "paimon-ftindex-core"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+rust-version.workspace = true
+
+[dependencies]
+serde.workspace = true
+serde_json.workspace = true
+tantivy.workspace = true
+tempfile.workspace = true
+thiserror.workspace = true
+
+[dev-dependencies]
+anyhow.workspace = true
diff --git a/core/src/config.rs b/core/src/config.rs
new file mode 100644
index 0000000..d6cf964
--- /dev/null
+++ b/core/src/config.rs
@@ -0,0 +1,43 @@
+use crate::tokenizer::TokenizerConfig;
+use serde::{Deserialize, Serialize};
+
+#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
+pub struct FullTextIndexConfig {
+ pub row_id_field: String,
+ pub text_field: String,
+ pub tokenizer: TokenizerConfig,
+}
+
+impl Default for FullTextIndexConfig {
+ fn default() -> Self {
+ Self {
+ row_id_field: "row_id".to_string(),
+ text_field: "text".to_string(),
+ tokenizer: TokenizerConfig::default(),
+ }
+ }
+}
+
+impl FullTextIndexConfig {
+ pub fn new() -> Self {
+ Self::default()
+ }
+
+ pub fn tokenizer(mut self, tokenizer: TokenizerConfig) -> Self {
+ self.tokenizer = tokenizer;
+ self
+ }
+
+ pub fn with_positions(mut self, with_positions: bool) -> Self {
+ self.tokenizer.with_position = with_positions;
+ self
+ }
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
+pub struct FullTextIndexMetadata {
+ pub format_version: u32,
+ pub config: FullTextIndexConfig,
+ pub document_count: u64,
+ pub tantivy_version: String,
+}
diff --git a/core/src/error.rs b/core/src/error.rs
new file mode 100644
index 0000000..d4d9598
--- /dev/null
+++ b/core/src/error.rs
@@ -0,0 +1,24 @@
+use thiserror::Error;
+
+pub type Result<T> = std::result::Result<T, FtIndexError>;
+
+#[derive(Debug, Error)]
+pub enum FtIndexError {
+ #[error("invalid option {key}: {message}")]
+ InvalidOption { key: String, message: String },
+
+ #[error("invalid query: {0}")]
+ InvalidQuery(String),
+
+ #[error("invalid storage format: {0}")]
+ InvalidStorage(String),
+
+ #[error("io error: {0}")]
+ Io(#[from] std::io::Error),
+
+ #[error("json error: {0}")]
+ Json(#[from] serde_json::Error),
+
+ #[error("tantivy error: {0}")]
+ Tantivy(#[from] tantivy::TantivyError),
+}
diff --git a/core/src/index.rs b/core/src/index.rs
new file mode 100644
index 0000000..101f223
--- /dev/null
+++ b/core/src/index.rs
@@ -0,0 +1,384 @@
+use crate::config::{FullTextIndexConfig, FullTextIndexMetadata};
+use crate::error::{FtIndexError, Result};
+use crate::io::{SeekRead, SeekWrite};
+use crate::query::{BooleanOccur, FullTextQuery, MatchOperator};
+use crate::storage::{read_exact_at, read_header, write_envelope, ArchiveFileEntry, IndexHeader};
+use crate::tokenizer::{TokenizerConfig, TokenizerKind};
+use std::fs;
+use std::path::Path;
+use tantivy::collector::TopDocs;
+use tantivy::directory::{Directory, RamDirectory};
+use tantivy::query::{BooleanQuery, BoostQuery, Occur, Query, QueryParser};
+use tantivy::schema::{IndexRecordOption, NumericOptions, Schema, TextFieldIndexing, TextOptions};
+use tantivy::tokenizer::{
+ AsciiFoldingFilter, LowerCaser, NgramTokenizer, RawTokenizer, RemoveLongFilter,
+ SimpleTokenizer, TextAnalyzer, WhitespaceTokenizer,
+};
+use tantivy::{Index, TantivyDocument};
+use tempfile::TempDir;
+
+#[derive(Clone, Debug, PartialEq)]
+pub struct FullTextSearchResult {
+ pub row_ids: Vec<i64>,
+ pub scores: Vec<f32>,
+}
+
+pub struct FullTextIndexWriter {
+ config: FullTextIndexConfig,
+ documents: Vec<(i64, String)>,
+}
+
+impl FullTextIndexWriter {
+ pub fn new(config: FullTextIndexConfig) -> Result<Self> {
+ config.tokenizer.validate()?;
+ Ok(Self {
+ config,
+ documents: Vec::new(),
+ })
+ }
+
+ pub fn add_document(&mut self, row_id: i64, text: impl Into<String>) -> Result<()> {
+ if row_id < 0 {
+ return Err(FtIndexError::InvalidStorage(format!(
+ "row id must be non-negative, got {row_id}"
+ )));
+ }
+ self.documents.push((row_id, text.into()));
+ Ok(())
+ }
+
+ pub fn write<W: SeekWrite>(&mut self, output: &mut W) -> Result<()> {
+ let temp_dir = TempDir::new()?;
+ let schema = build_schema(&self.config);
+ let mut index = Index::create_in_dir(temp_dir.path(), schema.clone())?;
+ register_tokenizer(&mut index, &self.config.tokenizer)?;
+ let row_id_field = schema
+ .get_field(&self.config.row_id_field)
+ .map_err(|_| FtIndexError::InvalidStorage("missing row_id field".to_string()))?;
+ let text_field = schema
+ .get_field(&self.config.text_field)
+ .map_err(|_| FtIndexError::InvalidStorage("missing text field".to_string()))?;
+
+ {
+ let mut index_writer = index.writer(50_000_000)?;
+ for (row_id, text) in &self.documents {
+ let mut doc = TantivyDocument::new();
+ doc.add_u64(row_id_field, *row_id as u64);
+ doc.add_text(text_field, text);
+ index_writer.add_document(doc)?;
+ }
+ index_writer.commit()?;
+ }
+
+ let files = collect_index_files(temp_dir.path())?;
+ let mut offset = 0u64;
+ let mut entries = Vec::with_capacity(files.len());
+ for (name, data) in &files {
+ entries.push(ArchiveFileEntry {
+ name: name.clone(),
+ offset,
+ length: data.len() as u64,
+ });
+ offset += data.len() as u64;
+ }
+
+ let header = IndexHeader {
+ metadata: FullTextIndexMetadata {
+ format_version: crate::storage::FORMAT_VERSION,
+ config: self.config.clone(),
+ document_count: self.documents.len() as u64,
+ tantivy_version: tantivy::version().to_string(),
+ },
+ files: entries,
+ };
+ write_envelope(output, &header, &files)
+ }
+}
+
+pub struct FullTextIndexReader<R> {
+ _input: R,
+ index: Index,
+ metadata: FullTextIndexMetadata,
+}
+
+impl<R: SeekRead> FullTextIndexReader<R> {
+ pub fn open(mut input: R) -> Result<Self> {
+ let (header, body_start) = read_header(&mut input)?;
+ let directory = RamDirectory::create();
+ for file in &header.files {
+ let mut data = vec![0u8; file.length as usize];
+ read_exact_at(&mut input, body_start + file.offset, &mut data)?;
+ directory.atomic_write(Path::new(&file.name), &data)?;
+ }
+ let mut index = Index::open(directory)?;
+ register_tokenizer(&mut index, &header.metadata.config.tokenizer)?;
+ Ok(Self {
+ _input: input,
+ index,
+ metadata: header.metadata,
+ })
+ }
+
+ pub fn optimize_for_search(&mut self) -> Result<()> {
+ Ok(())
+ }
+
+ pub fn metadata(&self) -> &FullTextIndexMetadata {
+ &self.metadata
+ }
+
+ pub fn search(&mut self, query: FullTextQuery, limit: usize) -> Result<FullTextSearchResult> {
+ if limit == 0 {
+ return Err(FtIndexError::InvalidQuery(
+ "search limit must be positive".to_string(),
+ ));
+ }
+ let reader = self.index.reader()?;
+ let searcher = reader.searcher();
+ let tantivy_query = build_query(&self.index, &self.metadata.config, &query)?;
+ let top_docs = searcher.search(&tantivy_query, &TopDocs::with_limit(limit))?;
+ let mut row_ids = Vec::with_capacity(top_docs.len());
+ let mut scores = Vec::with_capacity(top_docs.len());
+ for (score, doc_address) in top_docs {
+ let segment_reader = searcher.segment_reader(doc_address.segment_ord);
+ let row_id_column = segment_reader
+ .fast_fields()
+ .u64(&self.metadata.config.row_id_field)?
+ .first_or_default_col(0);
+ row_ids.push(row_id_column.get_val(doc_address.doc_id) as i64);
+ scores.push(score);
+ }
+ Ok(FullTextSearchResult { row_ids, scores })
+ }
+}
+
+fn build_schema(config: &FullTextIndexConfig) -> Schema {
+ let mut builder = Schema::builder();
+ builder.add_u64_field(
+ &config.row_id_field,
+ NumericOptions::default()
+ .set_fast()
+ .set_stored()
+ .set_indexed(),
+ );
+ let index_option = if config.tokenizer.with_position {
+ IndexRecordOption::WithFreqsAndPositions
+ } else {
+ IndexRecordOption::WithFreqs
+ };
+ let tokenizer_name = tokenizer_name(&config.tokenizer);
+ let indexing = TextFieldIndexing::default()
+ .set_tokenizer(tokenizer_name)
+ .set_index_option(index_option);
+ builder.add_text_field(
+ &config.text_field,
+ TextOptions::default().set_indexing_options(indexing),
+ );
+ builder.build()
+}
+
+fn tokenizer_name(config: &TokenizerConfig) -> &'static str {
+ match config.tokenizer {
+ TokenizerKind::Default if !needs_custom_default(config) => "default",
+ TokenizerKind::Default | TokenizerKind::Simple => "paimon_custom",
+ TokenizerKind::Whitespace => "paimon_custom",
+ TokenizerKind::Raw => "paimon_custom",
+ TokenizerKind::Ngram => "paimon_ngram",
+ TokenizerKind::Jieba => "paimon_jieba",
+ }
+}
+
+fn needs_custom_default(config: &TokenizerConfig) -> bool {
+ !config.lower_case
+ || config.max_token_length != 40
+ || config.ascii_folding
+ || config.stem
+ || config.remove_stop_words
+ || !config.stop_words.is_empty()
+}
+
+fn register_tokenizer(index: &mut Index, config: &TokenizerConfig) -> Result<()> {
+ match config.tokenizer {
+ TokenizerKind::Default if !needs_custom_default(config) => Ok(()),
+ TokenizerKind::Jieba => Err(FtIndexError::InvalidOption {
+ key: "tokenizer".to_string(),
+ message: "jieba tokenizer is not enabled in this first implementation".to_string(),
+ }),
+ _ => {
+ let analyzer = build_text_analyzer(config)?;
+ index
+ .tokenizers()
+ .register(tokenizer_name(config), analyzer);
+ Ok(())
+ }
+ }
+}
+
+fn build_text_analyzer(config: &TokenizerConfig) -> Result<TextAnalyzer> {
+ if config.stem || config.remove_stop_words || !config.stop_words.is_empty() {
+ return Err(FtIndexError::InvalidOption {
+ key: "tokenizer filters".to_string(),
+ message: "stemming and stop-word filters are not enabled in this first implementation"
+ .to_string(),
+ });
+ }
+ let mut builder = match config.tokenizer {
+ TokenizerKind::Default | TokenizerKind::Simple => {
+ TextAnalyzer::builder(SimpleTokenizer::default()).dynamic()
+ }
+ TokenizerKind::Whitespace => {
+ TextAnalyzer::builder(WhitespaceTokenizer::default()).dynamic()
+ }
+ TokenizerKind::Raw => TextAnalyzer::builder(RawTokenizer::default()).dynamic(),
+ TokenizerKind::Ngram => {
+ let tokenizer = NgramTokenizer::new(
+ config.ngram_min_gram,
+ config.ngram_max_gram,
+ config.ngram_prefix_only,
+ )
+ .map_err(|e| FtIndexError::InvalidOption {
+ key: "ngram".to_string(),
+ message: e.to_string(),
+ })?;
+ TextAnalyzer::builder(tokenizer).dynamic()
+ }
+ TokenizerKind::Jieba => {
+ return Err(FtIndexError::InvalidOption {
+ key: "tokenizer".to_string(),
+ message: "jieba tokenizer is not enabled in this first implementation".to_string(),
+ })
+ }
+ };
+ builder = builder.filter_dynamic(RemoveLongFilter::limit(config.max_token_length));
+ if config.lower_case {
+ builder = builder.filter_dynamic(LowerCaser);
+ }
+ if config.ascii_folding {
+ builder = builder.filter_dynamic(AsciiFoldingFilter);
+ }
+ Ok(builder.build())
+}
+
+fn collect_index_files(path: &Path) -> Result<Vec<(String, Vec<u8>)>> {
+ let mut paths = Vec::new();
+ for entry in fs::read_dir(path)? {
+ let entry = entry?;
+ if entry.file_type()?.is_file() {
+ paths.push(entry.path());
+ }
+ }
+ paths.sort();
+ let mut files = Vec::with_capacity(paths.len());
+ for path in paths {
+ let name = path
+ .file_name()
+ .and_then(|name| name.to_str())
+ .ok_or_else(|| FtIndexError::InvalidStorage("non-utf8 file name".to_string()))?
+ .to_string();
+ if name.ends_with(".lock") {
+ continue;
+ }
+ files.push((name, fs::read(path)?));
+ }
+ Ok(files)
+}
+
+fn build_query(
+ index: &Index,
+ config: &FullTextIndexConfig,
+ query: &FullTextQuery,
+) -> Result<Box<dyn Query>> {
+ match query {
+ FullTextQuery::Match {
+ column,
+ terms,
+ operator,
+ boost,
+ } => {
+ validate_column(config, column)?;
+ let text_field = index
+ .schema()
+ .get_field(&config.text_field)
+ .map_err(|_| FtIndexError::InvalidQuery("missing text field".to_string()))?;
+ let mut parser = QueryParser::for_index(index, vec![text_field]);
+ if *operator == MatchOperator::And {
+ parser.set_conjunction_by_default();
+ }
+ let parsed = parser
+ .parse_query(terms)
+ .map_err(|e| FtIndexError::InvalidQuery(e.to_string()))?;
+ if (*boost - 1.0).abs() > f32::EPSILON {
+ Ok(Box::new(BoostQuery::new(parsed, *boost)))
+ } else {
+ Ok(parsed)
+ }
+ }
+ FullTextQuery::MatchPhrase {
+ column,
+ terms,
+ slop,
+ } => {
+ validate_column(config, column)?;
+ if !config.tokenizer.with_position {
+ return Err(FtIndexError::InvalidQuery(
+ "phrase query requires positions".to_string(),
+ ));
+ }
+ let text_field = index
+ .schema()
+ .get_field(&config.text_field)
+ .map_err(|_| FtIndexError::InvalidQuery("missing text field".to_string()))?;
+ let parser = QueryParser::for_index(index, vec![text_field]);
+ let escaped = terms.replace('\\', "\\\\").replace('"', "\\\"");
+ let query_text = if *slop == 0 {
+ format!("\"{escaped}\"")
+ } else {
+ format!("\"{escaped}\"~{slop}")
+ };
+ parser
+ .parse_query(&query_text)
+ .map_err(|e| FtIndexError::InvalidQuery(e.to_string()))
+ }
+ FullTextQuery::Boolean { queries } => {
+ if queries.is_empty() {
+ return Err(FtIndexError::InvalidQuery(
+ "boolean query must contain at least one clause".to_string(),
+ ));
+ }
+ let mut children = Vec::with_capacity(queries.len());
+ for (occur, child) in queries {
+ let occur = match occur {
+ BooleanOccur::Should => Occur::Should,
+ BooleanOccur::Must => Occur::Must,
+ BooleanOccur::MustNot => Occur::MustNot,
+ };
+ children.push((occur, build_query(index, config, child)?));
+ }
+ Ok(Box::new(BooleanQuery::new(children)))
+ }
+ FullTextQuery::Boost {
+ positive,
+ negative,
+ negative_boost,
+ } => {
+ let boosted_negative = Box::new(BoostQuery::new(
+ build_query(index, config, negative)?,
+ *negative_boost,
+ ));
+ Ok(Box::new(BooleanQuery::new(vec![
+ (Occur::Should, build_query(index, config, positive)?),
+ (Occur::Should, boosted_negative),
+ ])))
+ }
+ }
+}
+
+fn validate_column(config: &FullTextIndexConfig, column: &str) -> Result<()> {
+ if column == config.text_field {
+ Ok(())
+ } else {
+ Err(FtIndexError::InvalidQuery(format!(
+ "unknown full-text column '{column}'"
+ )))
+ }
+}
diff --git a/core/src/io.rs b/core/src/io.rs
new file mode 100644
index 0000000..2d16b31
--- /dev/null
+++ b/core/src/io.rs
@@ -0,0 +1,80 @@
+use std::io;
+
+pub struct ReadRequest<'a> {
+ pub pos: u64,
+ pub buf: &'a mut [u8],
+}
+
+pub trait SeekRead: Send {
+ fn pread(&mut self, ranges: &mut [ReadRequest<'_>]) -> io::Result<()>;
+}
+
+pub trait SeekWrite: Send {
+ fn write_all(&mut self, buf: &[u8]) -> io::Result<()>;
+
+ fn flush(&mut self) -> io::Result<()> {
+ Ok(())
+ }
+}
+
+pub struct PosWriter<W> {
+ inner: W,
+}
+
+impl<W> PosWriter<W> {
+ pub fn new(inner: W) -> Self {
+ Self { inner }
+ }
+
+ pub fn into_inner(self) -> W {
+ self.inner
+ }
+}
+
+impl<W: io::Write + Send> SeekWrite for PosWriter<W> {
+ fn write_all(&mut self, buf: &[u8]) -> io::Result<()> {
+ io::Write::write_all(&mut self.inner, buf)
+ }
+
+ fn flush(&mut self) -> io::Result<()> {
+ io::Write::flush(&mut self.inner)
+ }
+}
+
+pub struct SliceReader {
+ data: Vec<u8>,
+}
+
+impl SliceReader {
+ pub fn new(data: Vec<u8>) -> Self {
+ Self { data }
+ }
+
+ pub fn len(&self) -> usize {
+ self.data.len()
+ }
+
+ pub fn is_empty(&self) -> bool {
+ self.data.is_empty()
+ }
+}
+
+impl SeekRead for SliceReader {
+ fn pread(&mut self, ranges: &mut [ReadRequest<'_>]) -> io::Result<()> {
+ for range in ranges {
+ let start = usize::try_from(range.pos)
+ .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "offset overflow"))?;
+ let end = start
+ .checked_add(range.buf.len())
+ .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "range overflow"))?;
+ if end > self.data.len() {
+ return Err(io::Error::new(
+ io::ErrorKind::UnexpectedEof,
+ "read past end of slice",
+ ));
+ }
+ range.buf.copy_from_slice(&self.data[start..end]);
+ }
+ Ok(())
+ }
+}
diff --git a/core/src/lib.rs b/core/src/lib.rs
new file mode 100644
index 0000000..4750845
--- /dev/null
+++ b/core/src/lib.rs
@@ -0,0 +1,13 @@
+pub mod config;
+pub mod error;
+pub mod index;
+pub mod io;
+pub mod query;
+pub mod storage;
+pub mod tokenizer;
+
+pub use config::{FullTextIndexConfig, FullTextIndexMetadata};
+pub use error::{FtIndexError, Result};
+pub use index::{FullTextIndexReader, FullTextIndexWriter, FullTextSearchResult};
+pub use query::{BooleanOccur, FullTextQuery, MatchOperator};
+pub use tokenizer::{TokenizerConfig, TokenizerKind};
diff --git a/core/src/query.rs b/core/src/query.rs
new file mode 100644
index 0000000..1b1d0a6
--- /dev/null
+++ b/core/src/query.rs
@@ -0,0 +1,93 @@
+use crate::error::{FtIndexError, Result};
+use serde::{Deserialize, Serialize};
+
+#[derive(Clone, Copy, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
+pub enum MatchOperator {
+ #[default]
+ Or,
+ And,
+}
+
+#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq)]
+pub enum BooleanOccur {
+ Should,
+ Must,
+ MustNot,
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "snake_case")]
+pub enum FullTextQuery {
+ Match {
+ column: String,
+ terms: String,
+ #[serde(default)]
+ operator: MatchOperator,
+ #[serde(default = "default_boost")]
+ boost: f32,
+ },
+ MatchPhrase {
+ column: String,
+ terms: String,
+ #[serde(default)]
+ slop: u32,
+ },
+ Boolean {
+ queries: Vec<(BooleanOccur, FullTextQuery)>,
+ },
+ Boost {
+ positive: Box<FullTextQuery>,
+ negative: Box<FullTextQuery>,
+ #[serde(default = "default_negative_boost")]
+ negative_boost: f32,
+ },
+}
+
+fn default_boost() -> f32 {
+ 1.0
+}
+
+fn default_negative_boost() -> f32 {
+ 0.5
+}
+
+impl FullTextQuery {
+ pub fn match_query(terms: impl Into<String>, column: impl Into<String>) -> Self {
+ Self::Match {
+ column: column.into(),
+ terms: terms.into(),
+ operator: MatchOperator::Or,
+ boost: 1.0,
+ }
+ }
+
+ pub fn phrase(terms: impl Into<String>, column: impl Into<String>) -> Self {
+ Self::MatchPhrase {
+ column: column.into(),
+ terms: terms.into(),
+ slop: 0,
+ }
+ }
+
+ pub fn operator_and(mut self) -> Self {
+ if let Self::Match { operator, .. } = &mut self {
+ *operator = MatchOperator::And;
+ }
+ self
+ }
+
+ pub fn operator_or(mut self) -> Self {
+ if let Self::Match { operator, .. } = &mut self {
+ *operator = MatchOperator::Or;
+ }
+ self
+ }
+
+ pub fn to_json(&self) -> Result<String> {
+ serde_json::to_string(self).map_err(FtIndexError::from)
+ }
+
+ pub fn from_json(json: &str) -> Result<Self> {
+ serde_json::from_str(json).map_err(FtIndexError::from)
+ }
+}
diff --git a/core/src/storage.rs b/core/src/storage.rs
new file mode 100644
index 0000000..8e4121f
--- /dev/null
+++ b/core/src/storage.rs
@@ -0,0 +1,66 @@
+use crate::config::FullTextIndexMetadata;
+use crate::error::{FtIndexError, Result};
+use crate::io::{ReadRequest, SeekRead, SeekWrite};
+use serde::{Deserialize, Serialize};
+
+pub const FORMAT_MAGIC: &[u8; 8] = b"PFTIDX01";
+pub const FORMAT_VERSION: u32 = 1;
+
+#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
+pub struct ArchiveFileEntry {
+ pub name: String,
+ pub offset: u64,
+ pub length: u64,
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
+pub struct IndexHeader {
+ pub metadata: FullTextIndexMetadata,
+ pub files: Vec<ArchiveFileEntry>,
+}
+
+pub fn write_envelope<W: SeekWrite>(
+ output: &mut W,
+ header: &IndexHeader,
+ files: &[(String, Vec<u8>)],
+) -> Result<()> {
+ let header_json = serde_json::to_vec(header)?;
+ output.write_all(FORMAT_MAGIC)?;
+ output.write_all(&FORMAT_VERSION.to_be_bytes())?;
+ output.write_all(
+ &u32::try_from(header_json.len())
+ .map_err(|_| FtIndexError::InvalidStorage("header too large".to_string()))?
+ .to_be_bytes(),
+ )?;
+ output.write_all(&header_json)?;
+ for (_, data) in files {
+ output.write_all(data)?;
+ }
+ output.flush()?;
+ Ok(())
+}
+
+pub fn read_header<R: SeekRead>(input: &mut R) -> Result<(IndexHeader, u64)> {
+ let mut fixed = [0u8; 16];
+ read_exact_at(input, 0, &mut fixed)?;
+ if &fixed[0..8] != FORMAT_MAGIC {
+ return Err(FtIndexError::InvalidStorage("bad magic".to_string()));
+ }
+ let version = u32::from_be_bytes(fixed[8..12].try_into().expect("slice length"));
+ if version != FORMAT_VERSION {
+ return Err(FtIndexError::InvalidStorage(format!(
+ "unsupported format version {version}"
+ )));
+ }
+ let header_len = u32::from_be_bytes(fixed[12..16].try_into().expect("slice length")) as usize;
+ let mut header_json = vec![0u8; header_len];
+ read_exact_at(input, 16, &mut header_json)?;
+ let header = serde_json::from_slice(&header_json)?;
+ Ok((header, 16 + header_len as u64))
+}
+
+pub fn read_exact_at<R: SeekRead>(input: &mut R, pos: u64, buf: &mut [u8]) -> Result<()> {
+ let mut request = [ReadRequest { pos, buf }];
+ input.pread(&mut request)?;
+ Ok(())
+}
diff --git a/core/src/tokenizer.rs b/core/src/tokenizer.rs
new file mode 100644
index 0000000..65635b2
--- /dev/null
+++ b/core/src/tokenizer.rs
@@ -0,0 +1,167 @@
+use crate::error::{FtIndexError, Result};
+use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
+
+#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
+#[serde(rename_all = "snake_case")]
+pub enum TokenizerKind {
+ #[default]
+ Default,
+ Simple,
+ Whitespace,
+ Raw,
+ Ngram,
+ Jieba,
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
+pub struct TokenizerConfig {
+ pub tokenizer: TokenizerKind,
+ pub ngram_min_gram: usize,
+ pub ngram_max_gram: usize,
+ pub ngram_prefix_only: bool,
+ pub lower_case: bool,
+ pub max_token_length: usize,
+ pub ascii_folding: bool,
+ pub stem: bool,
+ pub language: String,
+ pub remove_stop_words: bool,
+ pub stop_words: Vec<String>,
+ pub with_position: bool,
+}
+
+impl Default for TokenizerConfig {
+ fn default() -> Self {
+ Self {
+ tokenizer: TokenizerKind::Default,
+ ngram_min_gram: 2,
+ ngram_max_gram: 2,
+ ngram_prefix_only: false,
+ lower_case: true,
+ max_token_length: 40,
+ ascii_folding: false,
+ stem: false,
+ language: "english".to_string(),
+ remove_stop_words: false,
+ stop_words: Vec::new(),
+ with_position: true,
+ }
+ }
+}
+
+impl TokenizerConfig {
+ pub fn from_options(options: &HashMap<String, String>) -> Result<Self> {
+ let mut config = Self::default();
+ for (raw_key, value) in options {
+ let key = raw_key
+ .strip_prefix("fulltext.")
+ .or_else(|| raw_key.strip_prefix("tantivy."))
+ .unwrap_or(raw_key.as_str());
+ match key {
+ "tokenizer" => {
+ config.tokenizer = parse_tokenizer(value)?;
+ }
+ "ngram.min-gram" => {
+ config.ngram_min_gram = parse_usize(key, value)?;
+ }
+ "ngram.max-gram" => {
+ config.ngram_max_gram = parse_usize(key, value)?;
+ }
+ "ngram.prefix-only" => {
+ config.ngram_prefix_only = parse_bool(key, value)?;
+ }
+ "lower-case" => {
+ config.lower_case = parse_bool(key, value)?;
+ }
+ "max-token-length" => {
+ config.max_token_length = parse_usize(key, value)?;
+ }
+ "ascii-folding" => {
+ config.ascii_folding = parse_bool(key, value)?;
+ }
+ "stem" => {
+ config.stem = parse_bool(key, value)?;
+ }
+ "language" => {
+ config.language = value.trim().to_lowercase();
+ }
+ "remove-stop-words" => {
+ config.remove_stop_words = parse_bool(key, value)?;
+ }
+ "stop-words" => {
+ config.stop_words = value
+ .split(';')
+ .map(str::trim)
+ .filter(|word| !word.is_empty())
+ .map(ToOwned::to_owned)
+ .collect();
+ }
+ "with-position" => {
+ config.with_position = parse_bool(key, value)?;
+ }
+ _ => {}
+ }
+ }
+ config.validate()?;
+ Ok(config)
+ }
+
+ pub fn validate(&self) -> Result<()> {
+ if self.ngram_min_gram == 0 {
+ return invalid("ngram.min-gram", "must be positive");
+ }
+ if self.ngram_max_gram == 0 {
+ return invalid("ngram.max-gram", "must be positive");
+ }
+ if self.ngram_min_gram > self.ngram_max_gram {
+ return invalid("ngram.min-gram", "must not exceed ngram.max-gram");
+ }
+ if self.max_token_length == 0 {
+ return invalid("max-token-length", "must be positive");
+ }
+ Ok(())
+ }
+}
+
+fn parse_tokenizer(value: &str) -> Result<TokenizerKind> {
+ match value.trim().to_lowercase().as_str() {
+ "default" => Ok(TokenizerKind::Default),
+ "simple" => Ok(TokenizerKind::Simple),
+ "whitespace" => Ok(TokenizerKind::Whitespace),
+ "raw" => Ok(TokenizerKind::Raw),
+ "ngram" => Ok(TokenizerKind::Ngram),
+ "jieba" => Ok(TokenizerKind::Jieba),
+ other => Err(FtIndexError::InvalidOption {
+ key: "tokenizer".to_string(),
+ message: format!("unsupported tokenizer '{other}'"),
+ }),
+ }
+}
+
+fn parse_usize(key: &str, value: &str) -> Result<usize> {
+ value
+ .trim()
+ .parse()
+ .map_err(|_| FtIndexError::InvalidOption {
+ key: key.to_string(),
+ message: format!("expected unsigned integer, got '{value}'"),
+ })
+}
+
+fn parse_bool(key: &str, value: &str) -> Result<bool> {
+ match value.trim().to_lowercase().as_str() {
+ "true" => Ok(true),
+ "false" => Ok(false),
+ _ => Err(FtIndexError::InvalidOption {
+ key: key.to_string(),
+ message: format!("expected boolean, got '{value}'"),
+ }),
+ }
+}
+
+fn invalid<T>(key: &str, message: &str) -> Result<T> {
+ Err(FtIndexError::InvalidOption {
+ key: key.to_string(),
+ message: message.to_string(),
+ })
+}
diff --git a/core/tests/core_roundtrip.rs b/core/tests/core_roundtrip.rs
new file mode 100644
index 0000000..2188611
--- /dev/null
+++ b/core/tests/core_roundtrip.rs
@@ -0,0 +1,68 @@
+use paimon_ftindex_core::io::{PosWriter, SliceReader};
+use paimon_ftindex_core::{
+ FullTextIndexConfig, FullTextIndexReader, FullTextIndexWriter, FullTextQuery, MatchOperator,
+};
+
+fn build_index() -> anyhow::Result<Vec<u8>> {
+ let mut writer = FullTextIndexWriter::new(FullTextIndexConfig::new())?;
+ writer.add_document(10, "Apache Paimon supports full text search")?;
+ writer.add_document(11, "Tantivy is a Rust search engine")?;
+ writer.add_document(12, "Paimon tables can use indexes")?;
+
+ let mut bytes = Vec::new();
+ {
+ let mut output = PosWriter::new(&mut bytes);
+ writer.write(&mut output)?;
+ }
+ Ok(bytes)
+}
+
+#[test]
+fn match_query_round_trip() -> anyhow::Result<()> {
+ let bytes = build_index()?;
+ let mut reader = FullTextIndexReader::open(SliceReader::new(bytes))?;
+ let result = reader.search(FullTextQuery::match_query("paimon", "text"), 10)?;
+
+ assert_eq!(reader.metadata().document_count, 3);
+ assert_eq!(result.row_ids.len(), 2);
+ assert!(result.row_ids.contains(&10));
+ assert!(result.row_ids.contains(&12));
+ assert_eq!(result.scores.len(), 2);
+ Ok(())
+}
+
+#[test]
+fn match_query_and_operator_filters_terms() -> anyhow::Result<()> {
+ let bytes = build_index()?;
+ let mut reader = FullTextIndexReader::open(SliceReader::new(bytes))?;
+ let query = FullTextQuery::Match {
+ column: "text".to_string(),
+ terms: "paimon indexes".to_string(),
+ operator: MatchOperator::And,
+ boost: 1.0,
+ };
+ let result = reader.search(query, 10)?;
+
+ assert_eq!(result.row_ids, vec![12]);
+ Ok(())
+}
+
+#[test]
+fn phrase_query_uses_positions() -> anyhow::Result<()> {
+ let bytes = build_index()?;
+ let mut reader = FullTextIndexReader::open(SliceReader::new(bytes))?;
+ let result = reader.search(FullTextQuery::phrase("full text", "text"), 10)?;
+
+ assert_eq!(result.row_ids, vec![10]);
+ Ok(())
+}
+
+#[test]
+fn query_json_round_trip() -> anyhow::Result<()> {
+ let query = FullTextQuery::match_query("apache paimon", "text").operator_and();
+ let json = query.to_json()?;
+ let parsed = FullTextQuery::from_json(&json)?;
+
+ assert_eq!(parsed, query);
+ Ok(())
+}
diff --git a/docs/designs/2026-07-04-paimon-full-text-index.md b/docs/designs/2026-07-04-paimon-full-text-index.md
new file mode 100644
index 0000000..aa6a51e
--- /dev/null
+++ b/docs/designs/2026-07-04-paimon-full-text-index.md
@@ -0,0 +1,445 @@
+# Paimon Full Text Index Design
+
+## Problem Statement
+
+We want to split Paimon's Tantivy-based full-text search implementation into a
+standalone repository, shaped like `apache/paimon-vector-index`, and expose a
+consistent Rust, Java, and Python API.
+
+The upstream `paimon-tantivy` code is still pre-production for this use case, so
+the standalone project does not need to read or write its existing archive
+format. The new repository should use the cleanest v1 format and API.
+
+## Chosen Approach
+
+Use a Rust-first architecture:
+
+- `core`: owns Tantivy schema, tokenizer config, query semantics, storage
+ format, writer, reader, and tests.
+- `ffi`: exposes a stable C ABI over `core`.
+- `jni`: implements Java native calls by delegating to the Rust core.
+- `java`: provides a small public Java API using Paimon-style input/output
+ callbacks.
+- `python`: provides a ctypes binding over the C ABI.
+
+This matches the proven shape of `paimon-vector-index` and avoids divergent
+Java and Python implementations.
+
+## Repository Layout
+
+```text
+paimon-full-text/
+ Cargo.toml
+ core/
+ Cargo.toml
+ src/
+ config.rs
+ document.rs
+ error.rs
+ index.rs
+ io.rs
+ query.rs
+ tokenizer.rs
+ storage.rs
+ lib.rs
+ tests/
+ ffi/
+ Cargo.toml
+ build.rs
+ cbindgen.toml
+ src/lib.rs
+ jni/
+ Cargo.toml
+ src/lib.rs
+ include/
+ paimon_ftindex.h
+ paimon_ftindex.hpp
+ java/
+ pom.xml
+ src/main/java/org/apache/paimon/index/fulltext/
+ python/
+ pyproject.toml
+ setup.py
+ paimon_ftindex/
+ c/
+ CMakeLists.txt
+ test_ftindex.c
+ cpp/
+ CMakeLists.txt
+ test_ftindex.cpp
+ docs/
+ storage-format.md
+ java-api.md
+ python-api.md
+ paimon-integration.md
+```
+
+## Storage Format
+
+The v1 index file is self-describing. Readers must be able to open a file
+without Paimon manifest metadata.
+
+```text
+magic: 8 bytes "PFTIDX01"
+version: u32 1
+header_len: u32
+header: JSON metadata and archive directory
+body: concatenated Tantivy files
+checksum: optional, future v2
+```
+
+Header JSON:
+
+```json
+{
+ "format_version": 1,
+ "metadata": {
+ "format_version": 1,
+ "config": {
+ "row_id_field": "row_id",
+ "text_field": "text",
+ "tokenizer": {
+ "tokenizer": "default",
+ "lower_case": true,
+ "max_token_length": 40,
+ "with_position": true
+ }
+ },
+ "document_count": 0,
+ "tantivy_version": "0.22.1"
+ },
+ "files": [
+ {"name": "meta.json", "offset": 0, "length": 1234}
+ ]
+}
+```
+
+File offsets are relative to the body start. The archive directory lives in the
+front header so C, Java, and Python readers only need positional `read_at`
+callbacks and do not need to know total file length.
+
+## Rust API
+
+Rust is the source of truth.
+
+```rust
+use paimon_ftindex_core::{
+ FullTextIndexConfig, FullTextIndexReader, FullTextIndexWriter,
+ FullTextQuery, TokenizerConfig,
+};
+
+let config = FullTextIndexConfig::new()
+ .tokenizer(TokenizerConfig::default())
+ .with_positions(true);
+
+let mut writer = FullTextIndexWriter::new(config)?;
+writer.add_document(1, "Apache Paimon supports full text search")?;
+writer.write(&mut output)?;
+
+let mut reader = FullTextIndexReader::open(input)?;
+reader.optimize_for_search()?;
+let result = reader.search(
+ FullTextQuery::match_query("paimon", "text").operator_or(),
+ 10,
+)?;
+```
+
+Core concepts:
+
+- `FullTextIndexConfig`: schema and tokenizer config.
+- `TokenizerConfig`: `default`, `simple`, `whitespace`, `raw`, `ngram`,
+ `jieba`, filters, stop words, stemming, positions.
+- `FullTextDocument`: row id plus named text fields.
+- `FullTextQuery`: structured query DSL.
+- `FullTextSearchResult`: row ids plus BM25 scores.
+- `SeekRead` and `SeekWrite`: callback-based I/O like vector-index.
+
+First release supports one logical text field named `text`. The file format
+already stores field names so multi-field search can be added without changing
+the public storage envelope.
+
+## Query DSL
+
+Keep the Paimon/LanceDB-style structured JSON, but implement parsing in Rust.
+
+Supported v1 queries:
+
+- `match`: tokenized term query with `OR` or `AND`.
+- `match_phrase`: phrase query when positions are enabled.
+- `boolean`: `should`, `must`, `must_not`.
+- `boost`: score demotion or promotion by composing child query scores.
+
+Deferred:
+
+- `multi_match`: depends on multi-field index support.
+- arbitrary Tantivy query strings: useful for debugging but too easy to make
+ unstable as a public API.
+
+Example:
+
+```json
+{
+ "match": {
+ "column": "text",
+ "terms": "apache paimon",
+ "operator": "And",
+ "boost": 1.0
+ }
+}
+```
+
+## C ABI
+
+The C ABI is the stable interop boundary.
+
+Naming:
+
+- library: `libpaimon_ftindex_ffi`
+- header: `include/paimon_ftindex.h`
+- prefix: `paimon_ftindex_`
+
+Main handles:
+
+- `PaimonFtindexConfigHandle`
+- `PaimonFtindexWriterHandle`
+- `PaimonFtindexReaderHandle`
+- `PaimonFtindexSearchResult`
+
+I/O callbacks:
+
+```c
+typedef struct {
+ void *ctx;
+ int (*write_fn)(void *ctx, const uint8_t *buf, size_t len);
+ int (*flush_fn)(void *ctx);
+ int64_t (*get_pos_fn)(void *ctx);
+} PaimonFtindexOutputFile;
+
+typedef struct {
+ void *ctx;
+ int (*read_at_fn)(void *ctx, uint64_t pos, uint8_t *buf, size_t len);
+} PaimonFtindexInputFile;
+```
+
+Core calls:
+
+```c
+PaimonFtindexWriterHandle *paimon_ftindex_writer_open(
+ const char **keys,
+ const char **values,
+ size_t len);
+
+int paimon_ftindex_writer_add_document(
+ PaimonFtindexWriterHandle *writer,
+ int64_t row_id,
+ const char *text);
+
+int paimon_ftindex_writer_write_index(
+ PaimonFtindexWriterHandle *writer,
+ PaimonFtindexOutputFile output);
+
+PaimonFtindexReaderHandle *paimon_ftindex_reader_open(
+ PaimonFtindexInputFile input);
+
+int paimon_ftindex_reader_search_json(
+ PaimonFtindexReaderHandle *reader,
+ const char *query_json,
+ size_t limit,
+ int64_t *row_ids,
+ float *scores,
+ size_t capacity,
+ size_t *result_len);
+```
+
+All functions return `0` on success and `-1` on error. A thread-local
+`paimon_ftindex_last_error()` exposes the error message. Native panics are
+caught at the FFI boundary.
+
+## Java API
+
+Package:
+
+```text
+org.apache.paimon.index.fulltext
+```
+
+Public classes:
+
+- `FullTextIndexInput`
+- `FullTextIndexOutput`
+- `FullTextIndexWriter`
+- `FullTextIndexReader`
+- `FullTextIndexMetadata`
+- `FullTextQuery`
+- `FullTextSearchResult`
+- `FullTextNative`
+
+Usage:
+
+```java
+Map<String, String> options = new HashMap<>();
+options.put("tokenizer", "ngram");
+options.put("ngram.min-gram", "2");
+options.put("ngram.max-gram", "2");
+
+try (FullTextIndexWriter writer = FullTextIndexWriter.create(options)) {
+ writer.addDocument(1L, "Apache Paimon supports full text search");
+ writer.writeIndex(output);
+}
+
+try (FullTextIndexReader reader = new FullTextIndexReader(input)) {
+ FullTextSearchResult result =
+ reader.search(FullTextQuery.match("paimon", "text"), 10);
+}
+```
+
+Java should not expose Tantivy classes. It should use string options so Paimon
+table and procedure options map directly into the standalone library.
+
+## Python API
+
+Package:
+
+```text
+paimon_ftindex
+```
+
+Python uses ctypes over the C ABI and mirrors Java names in Python style.
+
+```python
+from paimon_ftindex import FullTextIndexReader, FullTextIndexWriter, MatchQuery
+
+writer = FullTextIndexWriter({
+ "tokenizer": "ngram",
+ "ngram.min-gram": "2",
+ "ngram.max-gram": "2",
+})
+writer.add_document(1, "Apache Paimon supports full text search")
+writer.write(output)
+
+reader = FullTextIndexReader(input)
+ids, scores = reader.search(MatchQuery("paimon", column="text"), limit=10)
+```
+
+Python I/O protocol:
+
+- output object: `write(bytes)`, optional `flush()`, optional `tell()`.
+- input object: `pread(pos: int, length: int) -> bytes`.
+
+The Python package should not depend on `tantivy-py`; Rust core provides the
+search semantics.
+
+## Paimon Integration
+
+The Paimon core repository should eventually keep only a thin integration
+module:
+
+- `paimon-fulltext` or `paimon-full-text`
+- `GlobalIndexerFactory` identifier: `tantivy-fulltext` or `fulltext`
+- index writer delegates to `FullTextIndexWriter`
+- index reader delegates to `FullTextIndexReader`
+- `FullTextQuery` and `FullTextSearch` in `paimon-common` can stay as the query
+ API and serialize to the JSON understood by this library
+
+Recommended Paimon option namespace:
+
+```text
+fulltext.tokenizer
+fulltext.ngram.min-gram
+fulltext.ngram.max-gram
+fulltext.ngram.prefix-only
+fulltext.lower-case
+fulltext.max-token-length
+fulltext.ascii-folding
+fulltext.stem
+fulltext.language
+fulltext.remove-stop-words
+fulltext.stop-words
+fulltext.with-position
+```
+
+The standalone library should accept both unprefixed keys and `fulltext.`
+prefixed keys for caller convenience.
+
+## Tokenizer Policy
+
+Supported first implementation tokenizers:
+
+- `default`
+- `simple`
+- `whitespace`
+- `raw`
+- `ngram`
+
+Reserved follow-up tokenizers and filters:
+
+- `jieba`
+- stemming
+- built-in stop words
+- custom stop words
+
+Supported first implementation filters:
+
+- lowercase
+- max token length
+- ASCII folding
+
+No dynamic Rust tokenizer plugins in v1. They complicate packaging and native
+loading too much for an initial library.
+
+## Testing Strategy
+
+Core tests:
+
+- storage format round trip
+- metadata round trip
+- query JSON parsing
+- tokenizer config validation
+- match, phrase, boolean, boost query behavior
+- seek-based input reads only requested byte ranges where possible
+
+Cross-language tests:
+
+- Rust writes, Java reads
+- Java writes, Rust reads
+- Rust writes, Python reads
+- Python writes, Rust reads
+- Java and Python return the same row ids and scores for fixed fixtures
+
+Fixture policy:
+
+- Commit small hex fixtures for v1 format.
+- Add a test that fails on accidental format drift unless the version changes.
+
+## Release Strategy
+
+Publish artifacts independently:
+
+- Rust crates: `paimon-ftindex-core`, `paimon-ftindex-ffi`,
+ `paimon-ftindex-jni`.
+- Java artifact: `org.apache.paimon:paimon-full-text-index`.
+- Python package: `paimon-ftindex`.
+
+Linux native binaries should be built on the oldest supported glibc baseline,
+matching the caution already present in upstream `paimon-tantivy-jni`.
+
+## Out Of Scope
+
+- Reading old upstream `paimon-tantivy` archive files.
+- Multi-field indexing in the first release.
+- Highlighting/snippets.
+- Arbitrary query-string API as a stable surface.
+- Distributed index merge and compaction policy inside this repository.
+- Paimon SQL procedures and optimizer rules. Those remain Paimon repository
+ integration work.
+
+## Open Questions
+
+- Should the public index type be `fulltext` or keep `tantivy-fulltext` for
+ clarity?
+- Should Java native loading copy prebuilt libraries from resources, or should
+ this repository initially require `PAIMON_FTINDEX_LIB_PATH` like the Python
+ package?
+- Should v1 expose row-id prefiltering with serialized 64-bit Roaring bitmaps,
+ matching vector-index, or defer it until Paimon needs hybrid scalar plus
+ full-text search?
diff --git a/docs/java-api.md b/docs/java-api.md
new file mode 100644
index 0000000..73aa5d4
--- /dev/null
+++ b/docs/java-api.md
@@ -0,0 +1,35 @@
+# Java API
+
+Package:
+
+```text
+org.apache.paimon.index.fulltext
+```
+
+Main classes:
+
+- `FullTextIndexWriter`
+- `FullTextIndexReader`
+- `FullTextQuery`
+- `FullTextSearchResult`
+- `FullTextIndexInput`
+- `FullTextIndexOutput`
+
+Example:
+
+```java
+try (FullTextIndexWriter writer = FullTextIndexWriter.create(Collections.emptyMap())) {
+ writer.addDocument(1L, "Apache Paimon full text search");
+ writer.writeIndex(output);
+}
+
+try (FullTextIndexReader reader = new FullTextIndexReader(input)) {
+ FullTextSearchResult result = reader.search(FullTextQuery.match("paimon", "text"), 10);
+}
+```
+
+Native loading:
+
+- Set `PAIMON_FTINDEX_JNI_LIB_PATH` to the full path of
+ `libpaimon_ftindex_jni.dylib` / `.so`, or
+- put the library on `java.library.path` as `paimon_ftindex_jni`.
diff --git a/docs/paimon-integration.md b/docs/paimon-integration.md
new file mode 100644
index 0000000..26f82b3
--- /dev/null
+++ b/docs/paimon-integration.md
@@ -0,0 +1,32 @@
+# Paimon Integration
+
+The standalone library is intentionally independent of Paimon core. Paimon
+integration should be a thin adapter:
+
+- Keep `FullTextQuery` / `FullTextSearch` in Paimon common as query API.
+- Serialize queries to the JSON accepted by this library.
+- Implement a Paimon `GlobalIndexerFactory` that delegates to Java
+ `FullTextIndexWriter` and `FullTextIndexReader`.
+- Store produced files as global index files.
+
+Suggested index identifier:
+
+```text
+fulltext
+```
+
+Suggested option namespace:
+
+```text
+fulltext.tokenizer
+fulltext.ngram.min-gram
+fulltext.ngram.max-gram
+fulltext.ngram.prefix-only
+fulltext.lower-case
+fulltext.max-token-length
+fulltext.ascii-folding
+fulltext.with-position
+```
+
+The standalone library accepts both unprefixed keys and `fulltext.` prefixed
+keys.
diff --git a/docs/plans/2026-07-04-paimon-full-text-index-implementation.md b/docs/plans/2026-07-04-paimon-full-text-index-implementation.md
new file mode 100644
index 0000000..3ba184f
--- /dev/null
+++ b/docs/plans/2026-07-04-paimon-full-text-index-implementation.md
@@ -0,0 +1,65 @@
+# Implementation Plan: Paimon Full Text Index
+
+## Prerequisites
+
+- [x] Confirm local Rust, Python, and Maven toolchains are available.
+- [x] Confirm this repository is the standalone `paimon-full-text` workspace.
+
+## Tasks
+
+### Task 1: Create Rust workspace skeleton
+
+- **Files**: `Cargo.toml`, `core/Cargo.toml`, `ffi/Cargo.toml`, `jni/Cargo.toml`
+- **Changes**: Define a top-level Cargo workspace with `core`, `ffi`, and `jni`
+ crates. Add Tantivy and serialization dependencies to `core`.
+- **Verify**: `cargo metadata --no-deps` succeeds.
+- **Dependencies**: None.
+
+### Task 2: Implement Rust core config, query, I/O, and storage
+
+- **Files**: `core/src/*.rs`
+- **Changes**: Add tokenizer config parsing, structured query JSON parsing,
+ seek/read-write traits, v1 storage envelope, Tantivy writer, and reader.
+- **Verify**: `cargo test -p paimon-ftindex-core`.
+- **Dependencies**: Task 1.
+
+### Task 3: Add C FFI over Rust core
+
+- **Files**: `ffi/src/lib.rs`, `include/paimon_ftindex.h`
+- **Changes**: Expose writer/reader handles, I/O callbacks, result buffers,
+ thread-local errors, and panic boundaries.
+- **Verify**: `cargo test -p paimon-ftindex-ffi`.
+- **Dependencies**: Task 2.
+
+### Task 4: Add Java API and JNI bridge
+
+- **Files**: `java/**`, `jni/src/lib.rs`
+- **Changes**: Add Java wrapper classes and JNI methods backed by Rust core.
+- **Verify**: `mvn test -pl java` where possible, and `cargo test -p
+ paimon-ftindex-jni`.
+- **Dependencies**: Task 2.
+
+### Task 5: Add Python ctypes package
+
+- **Files**: `python/**`
+- **Changes**: Add ctypes binding, Python query helpers, input/output adapter,
+ and unit tests using the local native library.
+- **Verify**: `python3 -m pytest python/tests` after building FFI library.
+- **Dependencies**: Task 3.
+
+### Task 6: Add repository docs and examples
+
+- **Files**: `README.md`, `docs/storage-format.md`, `docs/java-api.md`,
+ `docs/python-api.md`, `docs/paimon-integration.md`
+- **Changes**: Document architecture, build commands, API examples, and Paimon
+ integration.
+- **Verify**: `cargo test --workspace` and basic doc review.
+- **Dependencies**: Tasks 2-5.
+
+## Post-Implementation
+
+- [ ] Run `cargo fmt --all`.
+- [ ] Run `cargo test --workspace`.
+- [ ] Run targeted Python tests if native library builds.
+- [ ] Run targeted Java compilation/tests if JNI library builds.
+- [ ] Review `git diff`.
diff --git a/docs/python-api.md b/docs/python-api.md
new file mode 100644
index 0000000..5411d83
--- /dev/null
+++ b/docs/python-api.md
@@ -0,0 +1,35 @@
+# Python API
+
+Package:
+
+```text
+paimon_ftindex
+```
+
+Example:
+
+```python
+from paimon_ftindex import FullTextIndexReader, FullTextIndexWriter, MatchQuery
+
+with FullTextIndexWriter({"tokenizer": "ngram"}) as writer:
+ writer.add_document(1, "Apache Paimon full text search")
+ writer.write(output)
+
+with FullTextIndexReader(input_) as reader:
+ ids, scores = reader.search(MatchQuery("paimon"), limit=10)
+```
+
+The output object must provide:
+
+- `write(bytes)`
+- optional `flush()`
+
+The input object must provide:
+
+- `pread(pos: int, length: int) -> bytes`
+
+Native loading:
+
+- Set `PAIMON_FTINDEX_LIB_PATH` to a library file or directory, or
+- build `paimon-ftindex-ffi` so the package can discover `target/debug` or
+ `target/release`.
diff --git a/docs/storage-format.md b/docs/storage-format.md
new file mode 100644
index 0000000..0bc798d
--- /dev/null
+++ b/docs/storage-format.md
@@ -0,0 +1,24 @@
+# Storage Format
+
+Version 1 uses a front-loaded self-describing envelope:
+
+```text
+magic: 8 bytes "PFTIDX01"
+version: u32 1
+header_len: u32
+header: JSON metadata and archive directory
+body: concatenated Tantivy files
+```
+
+The header contains:
+
+- `metadata.format_version`
+- `metadata.config`
+- `metadata.document_count`
+- `metadata.tantivy_version`
+- `files[]`: Tantivy file name, body-relative offset, and length
+
+Readers first read the 16-byte fixed prefix, then the JSON header, then the
+listed Tantivy files by positional reads. The current reader loads listed files
+into Tantivy `RamDirectory`; a future reader can replace this with a custom
+seek-on-demand `Directory` without changing the envelope.
diff --git a/ffi/Cargo.toml b/ffi/Cargo.toml
new file mode 100644
index 0000000..f552b3e
--- /dev/null
+++ b/ffi/Cargo.toml
@@ -0,0 +1,18 @@
+[package]
+name = "paimon-ftindex-ffi"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+rust-version.workspace = true
+
+[lib]
+name = "paimon_ftindex_ffi"
+crate-type = ["cdylib", "rlib"]
+
+[dependencies]
+paimon-ftindex-core = { path = "../core" }
+serde_json.workspace = true
+
+[dev-dependencies]
+anyhow.workspace = true
diff --git a/ffi/src/lib.rs b/ffi/src/lib.rs
new file mode 100644
index 0000000..4840eff
--- /dev/null
+++ b/ffi/src/lib.rs
@@ -0,0 +1,398 @@
+#![allow(clippy::missing_safety_doc)]
+
+use paimon_ftindex_core::io::{ReadRequest, SeekRead, SeekWrite};
+use paimon_ftindex_core::{
+ FullTextIndexConfig, FullTextIndexReader, FullTextIndexWriter, FullTextQuery, TokenizerConfig,
+};
+use std::cell::RefCell;
+use std::collections::HashMap;
+use std::ffi::{CStr, CString};
+use std::io;
+use std::os::raw::{c_char, c_int, c_void};
+use std::panic::{self, AssertUnwindSafe};
+use std::{ptr, slice};
+
+thread_local! {
+ static LAST_ERROR: RefCell<Option<CString>> = const { RefCell::new(None) };
+}
+
+fn set_error(msg: impl Into<String>) {
+ let msg = msg.into().replace('\0', "\\0");
+ LAST_ERROR.with(|e| {
+ *e.borrow_mut() = CString::new(msg).ok();
+ });
+}
+
+fn panic_message(e: &Box<dyn std::any::Any + Send>) -> String {
+ if let Some(s) = e.downcast_ref::<String>() {
+ format!("native panic: {s}")
+ } else if let Some(s) = e.downcast_ref::<&str>() {
+ format!("native panic: {s}")
+ } else {
+ "native panic: unknown".to_string()
+ }
+}
+
+fn ffi_status<F>(f: F) -> c_int
+where
+ F: FnOnce() -> Result<(), String>,
+{
+ match panic::catch_unwind(AssertUnwindSafe(f)) {
+ Ok(Ok(())) => 0,
+ Ok(Err(e)) => {
+ set_error(e);
+ -1
+ }
+ Err(e) => {
+ set_error(panic_message(&e));
+ -1
+ }
+ }
+}
+
+fn ffi_ptr<T, F>(f: F) -> *mut T
+where
+ F: FnOnce() -> Result<*mut T, String>,
+{
+ match panic::catch_unwind(AssertUnwindSafe(f)) {
+ Ok(Ok(value)) => value,
+ Ok(Err(e)) => {
+ set_error(e);
+ ptr::null_mut()
+ }
+ Err(e) => {
+ set_error(panic_message(&e));
+ ptr::null_mut()
+ }
+ }
+}
+
+#[no_mangle]
+pub extern "C" fn paimon_ftindex_last_error() -> *const c_char {
+ LAST_ERROR.with(|e| match &*e.borrow() {
+ Some(msg) => msg.as_ptr(),
+ None => ptr::null(),
+ })
+}
+
+#[repr(C)]
+pub struct PaimonFtindexOutputFile {
+ pub ctx: *mut c_void,
+ pub write_fn: Option<unsafe extern "C" fn(*mut c_void, *const u8, usize) -> c_int>,
+ pub flush_fn: Option<unsafe extern "C" fn(*mut c_void) -> c_int>,
+}
+
+struct FfiOutputFile {
+ raw: PaimonFtindexOutputFile,
+}
+
+unsafe impl Send for FfiOutputFile {}
+
+impl SeekWrite for FfiOutputFile {
+ fn write_all(&mut self, buf: &[u8]) -> io::Result<()> {
+ let write_fn = self
+ .raw
+ .write_fn
+ .ok_or_else(|| io::Error::other("write_fn is null"))?;
+ let status = unsafe { write_fn(self.raw.ctx, buf.as_ptr(), buf.len()) };
+ if status == 0 {
+ Ok(())
+ } else {
+ Err(io::Error::other("write callback failed"))
+ }
+ }
+
+ fn flush(&mut self) -> io::Result<()> {
+ if let Some(flush_fn) = self.raw.flush_fn {
+ let status = unsafe { flush_fn(self.raw.ctx) };
+ if status != 0 {
+ return Err(io::Error::other("flush callback failed"));
+ }
+ }
+ Ok(())
+ }
+}
+
+#[repr(C)]
+pub struct PaimonFtindexInputFile {
+ pub ctx: *mut c_void,
+ pub read_at_fn: Option<unsafe extern "C" fn(*mut c_void, u64, *mut u8, usize) -> c_int>,
+}
+
+struct FfiInputFile {
+ raw: PaimonFtindexInputFile,
+}
+
+unsafe impl Send for FfiInputFile {}
+
+impl SeekRead for FfiInputFile {
+ fn pread(&mut self, ranges: &mut [ReadRequest<'_>]) -> io::Result<()> {
+ let read_at_fn = self
+ .raw
+ .read_at_fn
+ .ok_or_else(|| io::Error::other("read_at_fn is null"))?;
+ for range in ranges {
+ let status = unsafe {
+ read_at_fn(
+ self.raw.ctx,
+ range.pos,
+ range.buf.as_mut_ptr(),
+ range.buf.len(),
+ )
+ };
+ if status != 0 {
+ return Err(io::Error::other(format!(
+ "read_at callback failed at offset {} length {}",
+ range.pos,
+ range.buf.len()
+ )));
+ }
+ }
+ Ok(())
+ }
+}
+
+pub struct PaimonFtindexWriterHandle {
+ inner: FullTextIndexWriter,
+}
+
+pub struct PaimonFtindexReaderHandle {
+ inner: FullTextIndexReader<FfiInputFile>,
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn paimon_ftindex_writer_open(
+ keys: *const *const c_char,
+ values: *const *const c_char,
+ len: usize,
+) -> *mut PaimonFtindexWriterHandle {
+ ffi_ptr(|| {
+ let options = options_from_raw(keys, values, len)?;
+ let tokenizer = TokenizerConfig::from_options(&options).map_err(|e| e.to_string())?;
+ let config = FullTextIndexConfig::new().tokenizer(tokenizer);
+ let writer = FullTextIndexWriter::new(config).map_err(|e| e.to_string())?;
+ Ok(Box::into_raw(Box::new(PaimonFtindexWriterHandle {
+ inner: writer,
+ })))
+ })
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn paimon_ftindex_writer_add_document(
+ writer: *mut PaimonFtindexWriterHandle,
+ row_id: i64,
+ text: *const c_char,
+) -> c_int {
+ ffi_status(|| {
+ let writer = require_mut(writer, "writer")?;
+ let text = cstr_to_string(text, "text")?;
+ writer
+ .inner
+ .add_document(row_id, text)
+ .map_err(|e| e.to_string())
+ })
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn paimon_ftindex_writer_write_index(
+ writer: *mut PaimonFtindexWriterHandle,
+ output: PaimonFtindexOutputFile,
+) -> c_int {
+ ffi_status(|| {
+ let writer = require_mut(writer, "writer")?;
+ let mut output = FfiOutputFile { raw: output };
+ writer.inner.write(&mut output).map_err(|e| e.to_string())
+ })
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn paimon_ftindex_writer_free(writer: *mut PaimonFtindexWriterHandle) {
+ if !writer.is_null() {
+ drop(Box::from_raw(writer));
+ }
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn paimon_ftindex_reader_open(
+ input: PaimonFtindexInputFile,
+) -> *mut PaimonFtindexReaderHandle {
+ ffi_ptr(|| {
+ let input = FfiInputFile { raw: input };
+ let reader = FullTextIndexReader::open(input).map_err(|e| e.to_string())?;
+ Ok(Box::into_raw(Box::new(PaimonFtindexReaderHandle {
+ inner: reader,
+ })))
+ })
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn paimon_ftindex_reader_search_json(
+ reader: *mut PaimonFtindexReaderHandle,
+ query_json: *const c_char,
+ limit: usize,
+ row_ids: *mut i64,
+ scores: *mut f32,
+ capacity: usize,
+ result_len: *mut usize,
+) -> c_int {
+ ffi_status(|| {
+ let reader = require_mut(reader, "reader")?;
+ if result_len.is_null() {
+ return Err("result_len is null".to_string());
+ }
+ let query_json = cstr_to_string(query_json, "query_json")?;
+ let query = FullTextQuery::from_json(&query_json).map_err(|e| e.to_string())?;
+ let result = reader
+ .inner
+ .search(query, limit)
+ .map_err(|e| e.to_string())?;
+ unsafe {
+ *result_len = result.row_ids.len();
+ }
+ if result.row_ids.len() > capacity {
+ return Err(format!(
+ "result capacity {} is smaller than result length {}",
+ capacity,
+ result.row_ids.len()
+ ));
+ }
+ if !result.row_ids.is_empty() {
+ if row_ids.is_null() {
+ return Err("row_ids is null".to_string());
+ }
+ if scores.is_null() {
+ return Err("scores is null".to_string());
+ }
+ unsafe {
+ ptr::copy_nonoverlapping(result.row_ids.as_ptr(), row_ids, result.row_ids.len());
+ ptr::copy_nonoverlapping(result.scores.as_ptr(), scores, result.scores.len());
+ }
+ }
+ Ok(())
+ })
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn paimon_ftindex_reader_free(reader: *mut PaimonFtindexReaderHandle) {
+ if !reader.is_null() {
+ drop(Box::from_raw(reader));
+ }
+}
+
+unsafe fn options_from_raw(
+ keys: *const *const c_char,
+ values: *const *const c_char,
+ len: usize,
+) -> Result<HashMap<String, String>, String> {
+ if len == 0 {
+ return Ok(HashMap::new());
+ }
+ if keys.is_null() {
+ return Err("keys is null".to_string());
+ }
+ if values.is_null() {
+ return Err("values is null".to_string());
+ }
+ let keys = slice::from_raw_parts(keys, len);
+ let values = slice::from_raw_parts(values, len);
+ let mut options = HashMap::with_capacity(len);
+ for i in 0..len {
+ let key = cstr_to_string(keys[i], "option key")?;
+ let value = cstr_to_string(values[i], "option value")?;
+ options.insert(key, value);
+ }
+ Ok(options)
+}
+
+unsafe fn require_mut<'a, T>(ptr: *mut T, name: &str) -> Result<&'a mut T, String> {
+ ptr.as_mut().ok_or_else(|| format!("{name} is null"))
+}
+
+unsafe fn cstr_to_string(ptr: *const c_char, name: &str) -> Result<String, String> {
+ if ptr.is_null() {
+ return Err(format!("{name} is null"));
+ }
+ CStr::from_ptr(ptr)
+ .to_str()
+ .map(|s| s.to_string())
+ .map_err(|e| format!("{name} is not valid UTF-8: {e}"))
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use std::ffi::CString;
+
+ unsafe extern "C" fn write_vec(ctx: *mut c_void, buf: *const u8, len: usize) -> c_int {
+ let out = &mut *(ctx as *mut Vec<u8>);
+ out.extend_from_slice(slice::from_raw_parts(buf, len));
+ 0
+ }
+
+ unsafe extern "C" fn read_vec(ctx: *mut c_void, pos: u64, buf: *mut u8, len: usize) -> c_int {
+ let input = &*(ctx as *const Vec<u8>);
+ let start = pos as usize;
+ let end = start + len;
+ if end > input.len() {
+ return -1;
+ }
+ ptr::copy_nonoverlapping(input[start..end].as_ptr(), buf, len);
+ 0
+ }
+
+ #[test]
+ fn ffi_round_trip_search_json() {
+ unsafe {
+ let writer = paimon_ftindex_writer_open(ptr::null(), ptr::null(), 0);
+ assert!(!writer.is_null());
+
+ let text = CString::new("Apache Paimon full text").unwrap();
+ assert_eq!(
+ paimon_ftindex_writer_add_document(writer, 7, text.as_ptr()),
+ 0
+ );
+
+ let mut bytes = Vec::new();
+ let output = PaimonFtindexOutputFile {
+ ctx: &mut bytes as *mut Vec<u8> as *mut c_void,
+ write_fn: Some(write_vec),
+ flush_fn: None,
+ };
+ assert_eq!(paimon_ftindex_writer_write_index(writer, output), 0);
+ paimon_ftindex_writer_free(writer);
+
+ let input = PaimonFtindexInputFile {
+ ctx: &bytes as *const Vec<u8> as *mut c_void,
+ read_at_fn: Some(read_vec),
+ };
+ let reader = paimon_ftindex_reader_open(input);
+ assert!(!reader.is_null());
+
+ let query = CString::new(
+ paimon_ftindex_core::FullTextQuery::match_query("paimon", "text")
+ .to_json()
+ .unwrap(),
+ )
+ .unwrap();
+ let mut row_ids = [0i64; 4];
+ let mut scores = [0f32; 4];
+ let mut result_len = 0usize;
+ assert_eq!(
+ paimon_ftindex_reader_search_json(
+ reader,
+ query.as_ptr(),
+ 4,
+ row_ids.as_mut_ptr(),
+ scores.as_mut_ptr(),
+ row_ids.len(),
+ &mut result_len,
+ ),
+ 0
+ );
+ assert_eq!(result_len, 1);
+ assert_eq!(row_ids[0], 7);
+ assert!(scores[0] > 0.0);
+ paimon_ftindex_reader_free(reader);
+ }
+ }
+}
diff --git a/include/paimon_ftindex.h b/include/paimon_ftindex.h
new file mode 100644
index 0000000..08c9237
--- /dev/null
+++ b/include/paimon_ftindex.h
@@ -0,0 +1,60 @@
+#ifndef PAIMON_FTINDEX_H
+#define PAIMON_FTINDEX_H
+
+#include <stdint.h>
+#include <stddef.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct PaimonFtindexWriterHandle PaimonFtindexWriterHandle;
+typedef struct PaimonFtindexReaderHandle PaimonFtindexReaderHandle;
+
+typedef struct {
+ void *ctx;
+ int (*write_fn)(void *ctx, const uint8_t *buf, size_t len);
+ int (*flush_fn)(void *ctx);
+} PaimonFtindexOutputFile;
+
+typedef struct {
+ void *ctx;
+ int (*read_at_fn)(void *ctx, uint64_t pos, uint8_t *buf, size_t len);
+} PaimonFtindexInputFile;
+
+const char *paimon_ftindex_last_error(void);
+
+PaimonFtindexWriterHandle *paimon_ftindex_writer_open(
+ const char **keys,
+ const char **values,
+ size_t len);
+
+int paimon_ftindex_writer_add_document(
+ PaimonFtindexWriterHandle *writer,
+ int64_t row_id,
+ const char *text);
+
+int paimon_ftindex_writer_write_index(
+ PaimonFtindexWriterHandle *writer,
+ PaimonFtindexOutputFile output);
+
+void paimon_ftindex_writer_free(PaimonFtindexWriterHandle *writer);
+
+PaimonFtindexReaderHandle *paimon_ftindex_reader_open(PaimonFtindexInputFile input);
+
+int paimon_ftindex_reader_search_json(
+ PaimonFtindexReaderHandle *reader,
+ const char *query_json,
+ size_t limit,
+ int64_t *row_ids,
+ float *scores,
+ size_t capacity,
+ size_t *result_len);
+
+void paimon_ftindex_reader_free(PaimonFtindexReaderHandle *reader);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/java/pom.xml b/java/pom.xml
new file mode 100644
index 0000000..7bf9eca
--- /dev/null
+++ b/java/pom.xml
@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <groupId>org.apache.paimon</groupId>
+ <artifactId>paimon-full-text-index</artifactId>
+ <version>0.1.0-SNAPSHOT</version>
+ <packaging>jar</packaging>
+
+ <properties>
+ <maven.compiler.source>8</maven.compiler.source>
+ <maven.compiler.target>8</maven.compiler.target>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ </properties>
+</project>
diff --git a/java/src/main/java/org/apache/paimon/index/fulltext/FullTextIndexInput.java b/java/src/main/java/org/apache/paimon/index/fulltext/FullTextIndexInput.java
new file mode 100644
index 0000000..ccd5452
--- /dev/null
+++ b/java/src/main/java/org/apache/paimon/index/fulltext/FullTextIndexInput.java
@@ -0,0 +1,8 @@
+package org.apache.paimon.index.fulltext;
+
+import java.io.IOException;
+
+public interface FullTextIndexInput {
+
+ void readAt(long position, byte[] buffer, int offset, int length) throws IOException;
+}
diff --git a/java/src/main/java/org/apache/paimon/index/fulltext/FullTextIndexOutput.java b/java/src/main/java/org/apache/paimon/index/fulltext/FullTextIndexOutput.java
new file mode 100644
index 0000000..c64f405
--- /dev/null
+++ b/java/src/main/java/org/apache/paimon/index/fulltext/FullTextIndexOutput.java
@@ -0,0 +1,10 @@
+package org.apache.paimon.index.fulltext;
+
+import java.io.IOException;
+
+public interface FullTextIndexOutput {
+
+ void write(byte[] buffer, int offset, int length) throws IOException;
+
+ default void flush() throws IOException {}
+}
diff --git a/java/src/main/java/org/apache/paimon/index/fulltext/FullTextIndexReader.java b/java/src/main/java/org/apache/paimon/index/fulltext/FullTextIndexReader.java
new file mode 100644
index 0000000..247051e
--- /dev/null
+++ b/java/src/main/java/org/apache/paimon/index/fulltext/FullTextIndexReader.java
@@ -0,0 +1,36 @@
+package org.apache.paimon.index.fulltext;
+
+public final class FullTextIndexReader implements AutoCloseable {
+
+ private long nativePtr;
+
+ public FullTextIndexReader(FullTextIndexInput input) {
+ if (input == null) {
+ throw new NullPointerException("input");
+ }
+ this.nativePtr = FullTextNative.openReader(input);
+ }
+
+ public FullTextSearchResult search(FullTextQuery query, int limit) {
+ if (query == null) {
+ throw new NullPointerException("query");
+ }
+ return FullTextNative.searchJson(requireOpen(), query.toJson(), limit);
+ }
+
+ @Override
+ public void close() {
+ long ptr = nativePtr;
+ nativePtr = 0;
+ if (ptr != 0) {
+ FullTextNative.freeReader(ptr);
+ }
+ }
+
+ private long requireOpen() {
+ if (nativePtr == 0) {
+ throw new IllegalStateException("FullTextIndexReader is closed");
+ }
+ return nativePtr;
+ }
+}
diff --git a/java/src/main/java/org/apache/paimon/index/fulltext/FullTextIndexWriter.java b/java/src/main/java/org/apache/paimon/index/fulltext/FullTextIndexWriter.java
new file mode 100644
index 0000000..d0fd13d
--- /dev/null
+++ b/java/src/main/java/org/apache/paimon/index/fulltext/FullTextIndexWriter.java
@@ -0,0 +1,53 @@
+package org.apache.paimon.index.fulltext;
+
+import java.util.Map;
+
+public final class FullTextIndexWriter implements AutoCloseable {
+
+ private long nativePtr;
+
+ private FullTextIndexWriter(long nativePtr) {
+ this.nativePtr = nativePtr;
+ }
+
+ public static FullTextIndexWriter create(Map<String, String> options) {
+ String[] keys = new String[options == null ? 0 : options.size()];
+ String[] values = new String[keys.length];
+ if (options != null) {
+ int i = 0;
+ for (Map.Entry<String, String> entry : options.entrySet()) {
+ keys[i] = entry.getKey();
+ values[i] = entry.getValue();
+ i++;
+ }
+ }
+ return new FullTextIndexWriter(FullTextNative.createWriter(keys, values));
+ }
+
+ public void addDocument(long rowId, String text) {
+ FullTextNative.addDocument(requireOpen(), rowId, text);
+ }
+
+ public void writeIndex(FullTextIndexOutput output) {
+ if (output == null) {
+ throw new NullPointerException("output");
+ }
+ FullTextNative.writeIndex(requireOpen(), output);
+ }
+
+ @Override
+ public void close() {
+ long ptr = nativePtr;
+ nativePtr = 0;
+ if (ptr != 0) {
+ FullTextNative.freeWriter(ptr);
+ }
+ }
+
+ private long requireOpen() {
+ if (nativePtr == 0) {
+ throw new IllegalStateException("FullTextIndexWriter is closed");
+ }
+ return nativePtr;
+ }
+}
diff --git a/java/src/main/java/org/apache/paimon/index/fulltext/FullTextNative.java b/java/src/main/java/org/apache/paimon/index/fulltext/FullTextNative.java
new file mode 100644
index 0000000..d573941
--- /dev/null
+++ b/java/src/main/java/org/apache/paimon/index/fulltext/FullTextNative.java
@@ -0,0 +1,29 @@
+package org.apache.paimon.index.fulltext;
+
+final class FullTextNative {
+
+ static {
+ String libPath = System.getenv("PAIMON_FTINDEX_JNI_LIB_PATH");
+ if (libPath != null && !libPath.isEmpty()) {
+ System.load(libPath);
+ } else {
+ System.loadLibrary("paimon_ftindex_jni");
+ }
+ }
+
+ private FullTextNative() {}
+
+ static native long createWriter(String[] keys, String[] values);
+
+ static native void addDocument(long writerPtr, long rowId, String text);
+
+ static native void writeIndex(long writerPtr, FullTextIndexOutput output);
+
+ static native void freeWriter(long writerPtr);
+
+ static native long openReader(FullTextIndexInput input);
+
+ static native FullTextSearchResult searchJson(long readerPtr, String queryJson, int limit);
+
+ static native void freeReader(long readerPtr);
+}
diff --git a/java/src/main/java/org/apache/paimon/index/fulltext/FullTextQuery.java b/java/src/main/java/org/apache/paimon/index/fulltext/FullTextQuery.java
new file mode 100644
index 0000000..30e5108
--- /dev/null
+++ b/java/src/main/java/org/apache/paimon/index/fulltext/FullTextQuery.java
@@ -0,0 +1,75 @@
+package org.apache.paimon.index.fulltext;
+
+public final class FullTextQuery {
+
+ private final String json;
+
+ private FullTextQuery(String json) {
+ this.json = json;
+ }
+
+ public static FullTextQuery match(String terms, String column) {
+ return match(terms, column, "Or");
+ }
+
+ public static FullTextQuery match(String terms, String column, String operator) {
+ return new FullTextQuery(
+ "{\"match\":{\"column\":\""
+ + escape(column)
+ + "\",\"terms\":\""
+ + escape(terms)
+ + "\",\"operator\":\""
+ + escape(operator)
+ + "\",\"boost\":1.0}}");
+ }
+
+ public static FullTextQuery phrase(String terms, String column) {
+ return new FullTextQuery(
+ "{\"match_phrase\":{\"column\":\""
+ + escape(column)
+ + "\",\"terms\":\""
+ + escape(terms)
+ + "\",\"slop\":0}}");
+ }
+
+ public static FullTextQuery json(String json) {
+ if (json == null) {
+ throw new NullPointerException("json");
+ }
+ return new FullTextQuery(json);
+ }
+
+ public String toJson() {
+ return json;
+ }
+
+ private static String escape(String value) {
+ if (value == null) {
+ throw new NullPointerException("value");
+ }
+ StringBuilder builder = new StringBuilder(value.length() + 8);
+ for (int i = 0; i < value.length(); i++) {
+ char c = value.charAt(i);
+ switch (c) {
+ case '\\':
+ builder.append("\\\\");
+ break;
+ case '"':
+ builder.append("\\\"");
+ break;
+ case '\n':
+ builder.append("\\n");
+ break;
+ case '\r':
+ builder.append("\\r");
+ break;
+ case '\t':
+ builder.append("\\t");
+ break;
+ default:
+ builder.append(c);
+ }
+ }
+ return builder.toString();
+ }
+}
diff --git a/java/src/main/java/org/apache/paimon/index/fulltext/FullTextSearchResult.java b/java/src/main/java/org/apache/paimon/index/fulltext/FullTextSearchResult.java
new file mode 100644
index 0000000..dd5dd38
--- /dev/null
+++ b/java/src/main/java/org/apache/paimon/index/fulltext/FullTextSearchResult.java
@@ -0,0 +1,24 @@
+package org.apache.paimon.index.fulltext;
+
+public final class FullTextSearchResult {
+
+ private final long[] rowIds;
+ private final float[] scores;
+
+ public FullTextSearchResult(long[] rowIds, float[] scores) {
+ this.rowIds = rowIds;
+ this.scores = scores;
+ }
+
+ public long[] rowIds() {
+ return rowIds;
+ }
+
+ public float[] scores() {
+ return scores;
+ }
+
+ public int size() {
+ return rowIds.length;
+ }
+}
diff --git a/jni/Cargo.toml b/jni/Cargo.toml
new file mode 100644
index 0000000..1ae2df2
--- /dev/null
+++ b/jni/Cargo.toml
@@ -0,0 +1,19 @@
+[package]
+name = "paimon-ftindex-jni"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+rust-version.workspace = true
+
+[lib]
+name = "paimon_ftindex_jni"
+crate-type = ["cdylib", "rlib"]
+
+[dependencies]
+jni.workspace = true
+paimon-ftindex-core = { path = "../core" }
+serde_json.workspace = true
+
+[dev-dependencies]
+anyhow.workspace = true
diff --git a/jni/src/lib.rs b/jni/src/lib.rs
new file mode 100644
index 0000000..18d93c5
--- /dev/null
+++ b/jni/src/lib.rs
@@ -0,0 +1,334 @@
+use jni::objects::{GlobalRef, JByteArray, JClass, JObject, JObjectArray, JString, JValue};
+use jni::sys::{jint, jlong, jobject};
+use jni::{JNIEnv, JavaVM};
+use paimon_ftindex_core::io::{ReadRequest, SeekRead, SeekWrite};
+use paimon_ftindex_core::{
+ FullTextIndexConfig, FullTextIndexReader, FullTextIndexWriter, FullTextQuery, TokenizerConfig,
+};
+use std::collections::HashMap;
+use std::io;
+use std::ptr;
+
+struct JavaOutput {
+ jvm: JavaVM,
+ output: GlobalRef,
+}
+
+unsafe impl Send for JavaOutput {}
+
+impl SeekWrite for JavaOutput {
+ fn write_all(&mut self, buf: &[u8]) -> io::Result<()> {
+ let mut env = self
+ .jvm
+ .attach_current_thread()
+ .map_err(|e| io::Error::other(format!("JNI attach failed: {e}")))?;
+ let array = env
+ .new_byte_array(buf.len() as i32)
+ .map_err(|e| io::Error::other(format!("new_byte_array failed: {e}")))?;
+ let signed: Vec<i8> = buf.iter().map(|b| *b as i8).collect();
+ env.set_byte_array_region(&array, 0, &signed)
+ .map_err(|e| io::Error::other(format!("set_byte_array_region failed: {e}")))?;
+ let array_obj = JObject::from(array);
+ env.call_method(
+ self.output.as_obj(),
+ "write",
+ "([BII)V",
+ &[
+ JValue::Object(&array_obj),
+ JValue::Int(0),
+ JValue::Int(buf.len() as i32),
+ ],
+ )
+ .map_err(|e| io::Error::other(format!("Java output write failed: {e}")))?;
+ Ok(())
+ }
+
+ fn flush(&mut self) -> io::Result<()> {
+ let mut env = self
+ .jvm
+ .attach_current_thread()
+ .map_err(|e| io::Error::other(format!("JNI attach failed: {e}")))?;
+ env.call_method(self.output.as_obj(), "flush", "()V", &[])
+ .map_err(|e| io::Error::other(format!("Java output flush failed: {e}")))?;
+ Ok(())
+ }
+}
+
+struct JavaInput {
+ jvm: JavaVM,
+ input: GlobalRef,
+}
+
+unsafe impl Send for JavaInput {}
+
+impl SeekRead for JavaInput {
+ fn pread(&mut self, ranges: &mut [ReadRequest<'_>]) -> io::Result<()> {
+ let mut env = self
+ .jvm
+ .attach_current_thread()
+ .map_err(|e| io::Error::other(format!("JNI attach failed: {e}")))?;
+ for range in ranges {
+ let array = env
+ .new_byte_array(range.buf.len() as i32)
+ .map_err(|e| io::Error::other(format!("new_byte_array failed: {e}")))?;
+ let array_obj = JObject::from(array);
+ env.call_method(
+ self.input.as_obj(),
+ "readAt",
+ "(J[BII)V",
+ &[
+ JValue::Long(range.pos as i64),
+ JValue::Object(&array_obj),
+ JValue::Int(0),
+ JValue::Int(range.buf.len() as i32),
+ ],
+ )
+ .map_err(|e| io::Error::other(format!("Java input readAt failed: {e}")))?;
+ let array = JByteArray::from(array_obj);
+ let mut signed = vec![0i8; range.buf.len()];
+ env.get_byte_array_region(&array, 0, &mut signed)
+ .map_err(|e| io::Error::other(format!("get_byte_array_region failed: {e}")))?;
+ for (dst, src) in range.buf.iter_mut().zip(signed) {
+ *dst = src as u8;
+ }
+ }
+ Ok(())
+ }
+}
+
+struct WriterHandle {
+ inner: FullTextIndexWriter,
+}
+
+struct ReaderHandle {
+ inner: FullTextIndexReader<JavaInput>,
+}
+
+#[no_mangle]
+pub extern "system" fn Java_org_apache_paimon_index_fulltext_FullTextNative_createWriter(
+ mut env: JNIEnv,
+ _class: JClass,
+ keys: JObjectArray,
+ values: JObjectArray,
+) -> jlong {
+ match create_writer(&mut env, keys, values) {
+ Ok(ptr) => ptr,
+ Err(e) => throw_and_return(&mut env, &e, 0),
+ }
+}
+
+#[no_mangle]
+pub extern "system" fn Java_org_apache_paimon_index_fulltext_FullTextNative_addDocument(
+ mut env: JNIEnv,
+ _class: JClass,
+ writer_ptr: jlong,
+ row_id: jlong,
+ text: JString,
+) {
+ if let Err(e) = add_document(&mut env, writer_ptr, row_id, text) {
+ throw(&mut env, &e);
+ }
+}
+
+#[no_mangle]
+pub extern "system" fn Java_org_apache_paimon_index_fulltext_FullTextNative_writeIndex(
+ mut env: JNIEnv,
+ _class: JClass,
+ writer_ptr: jlong,
+ output: JObject,
+) {
+ if let Err(e) = write_index(&mut env, writer_ptr, output) {
+ throw(&mut env, &e);
+ }
+}
+
+#[no_mangle]
+pub extern "system" fn Java_org_apache_paimon_index_fulltext_FullTextNative_freeWriter(
+ _env: JNIEnv,
+ _class: JClass,
+ writer_ptr: jlong,
+) {
+ if writer_ptr != 0 {
+ unsafe {
+ drop(Box::from_raw(writer_ptr as *mut WriterHandle));
+ }
+ }
+}
+
+#[no_mangle]
+pub extern "system" fn Java_org_apache_paimon_index_fulltext_FullTextNative_openReader(
+ mut env: JNIEnv,
+ _class: JClass,
+ input: JObject,
+) -> jlong {
+ match open_reader(&mut env, input) {
+ Ok(ptr) => ptr,
+ Err(e) => throw_and_return(&mut env, &e, 0),
+ }
+}
+
+#[no_mangle]
+pub extern "system" fn Java_org_apache_paimon_index_fulltext_FullTextNative_searchJson(
+ mut env: JNIEnv,
+ _class: JClass,
+ reader_ptr: jlong,
+ query_json: JString,
+ limit: jint,
+) -> jobject {
+ match search_json(&mut env, reader_ptr, query_json, limit) {
+ Ok(obj) => obj,
+ Err(e) => throw_and_return(&mut env, &e, ptr::null_mut()),
+ }
+}
+
+#[no_mangle]
+pub extern "system" fn Java_org_apache_paimon_index_fulltext_FullTextNative_freeReader(
+ _env: JNIEnv,
+ _class: JClass,
+ reader_ptr: jlong,
+) {
+ if reader_ptr != 0 {
+ unsafe {
+ drop(Box::from_raw(reader_ptr as *mut ReaderHandle));
+ }
+ }
+}
+
+fn create_writer(
+ env: &mut JNIEnv,
+ keys: JObjectArray,
+ values: JObjectArray,
+) -> Result<jlong, String> {
+ let options = options_from_arrays(env, keys, values)?;
+ let tokenizer = TokenizerConfig::from_options(&options).map_err(|e| e.to_string())?;
+ let config = FullTextIndexConfig::new().tokenizer(tokenizer);
+ let writer = FullTextIndexWriter::new(config).map_err(|e| e.to_string())?;
+ Ok(Box::into_raw(Box::new(WriterHandle { inner: writer })) as jlong)
+}
+
+fn add_document(
+ env: &mut JNIEnv,
+ writer_ptr: jlong,
+ row_id: jlong,
+ text: JString,
+) -> Result<(), String> {
+ let writer = handle_mut::<WriterHandle>(writer_ptr, "writer")?;
+ let text: String = env
+ .get_string(&text)
+ .map_err(|e| format!("failed to read text: {e}"))?
+ .into();
+ writer
+ .inner
+ .add_document(row_id, text)
+ .map_err(|e| e.to_string())
+}
+
+fn write_index(env: &mut JNIEnv, writer_ptr: jlong, output: JObject) -> Result<(), String> {
+ let writer = handle_mut::<WriterHandle>(writer_ptr, "writer")?;
+ let jvm = env.get_java_vm().map_err(|e| e.to_string())?;
+ let output = env.new_global_ref(output).map_err(|e| e.to_string())?;
+ let mut output = JavaOutput { jvm, output };
+ writer.inner.write(&mut output).map_err(|e| e.to_string())
+}
+
+fn open_reader(env: &mut JNIEnv, input: JObject) -> Result<jlong, String> {
+ let jvm = env.get_java_vm().map_err(|e| e.to_string())?;
+ let input = env.new_global_ref(input).map_err(|e| e.to_string())?;
+ let input = JavaInput { jvm, input };
+ let reader = FullTextIndexReader::open(input).map_err(|e| e.to_string())?;
+ Ok(Box::into_raw(Box::new(ReaderHandle { inner: reader })) as jlong)
+}
+
+fn search_json(
+ env: &mut JNIEnv,
+ reader_ptr: jlong,
+ query_json: JString,
+ limit: jint,
+) -> Result<jobject, String> {
+ let reader = handle_mut::<ReaderHandle>(reader_ptr, "reader")?;
+ let query_json: String = env
+ .get_string(&query_json)
+ .map_err(|e| format!("failed to read query json: {e}"))?
+ .into();
+ let query = FullTextQuery::from_json(&query_json).map_err(|e| e.to_string())?;
+ let result = reader
+ .inner
+ .search(query, limit as usize)
+ .map_err(|e| e.to_string())?;
+
+ let row_ids = env
+ .new_long_array(result.row_ids.len() as i32)
+ .map_err(|e| e.to_string())?;
+ env.set_long_array_region(&row_ids, 0, &result.row_ids)
+ .map_err(|e| e.to_string())?;
+ let scores = env
+ .new_float_array(result.scores.len() as i32)
+ .map_err(|e| e.to_string())?;
+ env.set_float_array_region(&scores, 0, &result.scores)
+ .map_err(|e| e.to_string())?;
+
+ let row_ids_obj = JObject::from(row_ids);
+ let scores_obj = JObject::from(scores);
+ let obj = env
+ .new_object(
+ "org/apache/paimon/index/fulltext/FullTextSearchResult",
+ "([J[F)V",
+ &[JValue::Object(&row_ids_obj), JValue::Object(&scores_obj)],
+ )
+ .map_err(|e| e.to_string())?;
+ Ok(obj.into_raw())
+}
+
+fn options_from_arrays(
+ env: &mut JNIEnv,
+ keys: JObjectArray,
+ values: JObjectArray,
+) -> Result<HashMap<String, String>, String> {
+ let key_len = env.get_array_length(&keys).map_err(|e| e.to_string())?;
+ let value_len = env.get_array_length(&values).map_err(|e| e.to_string())?;
+ if key_len != value_len {
+ return Err(format!(
+ "keys length {} does not match values length {}",
+ key_len, value_len
+ ));
+ }
+ let mut options = HashMap::with_capacity(key_len as usize);
+ for i in 0..key_len {
+ let key = env
+ .get_object_array_element(&keys, i)
+ .map_err(|e| e.to_string())?;
+ let value = env
+ .get_object_array_element(&values, i)
+ .map_err(|e| e.to_string())?;
+ let key: String = env
+ .get_string(&JString::from(key))
+ .map_err(|e| e.to_string())?
+ .into();
+ let value: String = env
+ .get_string(&JString::from(value))
+ .map_err(|e| e.to_string())?
+ .into();
+ options.insert(key, value);
+ }
+ Ok(options)
+}
+
+fn handle_mut<'a, T>(ptr: jlong, name: &str) -> Result<&'a mut T, String> {
+ if ptr == 0 {
+ return Err(format!("{name} is closed"));
+ }
+ unsafe {
+ (ptr as *mut T)
+ .as_mut()
+ .ok_or_else(|| format!("{name} is null"))
+ }
+}
+
+fn throw(env: &mut JNIEnv, message: &str) {
+ let _ = env.throw_new("java/lang/RuntimeException", message);
+}
+
+fn throw_and_return<T>(env: &mut JNIEnv, message: &str, value: T) -> T {
+ throw(env, message);
+ value
+}
diff --git a/python/paimon_ftindex/__init__.py b/python/paimon_ftindex/__init__.py
new file mode 100644
index 0000000..485f1e4
--- /dev/null
+++ b/python/paimon_ftindex/__init__.py
@@ -0,0 +1,11 @@
+from .query import FullTextQuery, MatchQuery, PhraseQuery
+from .reader import FullTextIndexReader
+from .writer import FullTextIndexWriter
+
+__all__ = [
+ "FullTextIndexReader",
+ "FullTextIndexWriter",
+ "FullTextQuery",
+ "MatchQuery",
+ "PhraseQuery",
+]
diff --git a/python/paimon_ftindex/_ffi.py b/python/paimon_ftindex/_ffi.py
new file mode 100644
index 0000000..1bb025d
--- /dev/null
+++ b/python/paimon_ftindex/_ffi.py
@@ -0,0 +1,122 @@
+import ctypes
+import os
+import platform
+from ctypes import (
+ CFUNCTYPE,
+ POINTER,
+ Structure,
+ c_char_p,
+ c_float,
+ c_int,
+ c_int64,
+ c_size_t,
+ c_uint8,
+ c_uint64,
+ c_void_p,
+)
+
+
+def _lib_name():
+ system = platform.system()
+ if system == "Darwin":
+ return "libpaimon_ftindex_ffi.dylib"
+ if system == "Windows":
+ return "paimon_ftindex_ffi.dll"
+ return "libpaimon_ftindex_ffi.so"
+
+
+def _load_library():
+ lib_name = _lib_name()
+ env_path = os.environ.get("PAIMON_FTINDEX_LIB_PATH")
+ if env_path:
+ if os.path.isfile(env_path):
+ return ctypes.CDLL(env_path)
+ candidate = os.path.join(env_path, lib_name)
+ if os.path.isfile(candidate):
+ return ctypes.CDLL(candidate)
+
+ pkg_dir = os.path.dirname(os.path.abspath(__file__))
+ candidates = [
+ os.path.join(pkg_dir, lib_name),
+ os.path.join(pkg_dir, "..", "..", "target", "debug", lib_name),
+ os.path.join(pkg_dir, "..", "..", "target", "release", lib_name),
+ os.path.join(pkg_dir, "..", "..", "..", "target", "debug", lib_name),
+ os.path.join(pkg_dir, "..", "..", "..", "target", "release", lib_name),
+ ]
+ for candidate in candidates:
+ candidate = os.path.abspath(candidate)
+ if os.path.isfile(candidate):
+ return ctypes.CDLL(candidate)
+ return ctypes.CDLL(lib_name)
+
+
+lib = _load_library()
+
+WRITE_FN = CFUNCTYPE(c_int, c_void_p, POINTER(c_uint8), c_size_t)
+FLUSH_FN = CFUNCTYPE(c_int, c_void_p)
+READ_AT_FN = CFUNCTYPE(c_int, c_void_p, c_uint64, POINTER(c_uint8), c_size_t)
+
+
+class PaimonFtindexOutputFile(Structure):
+ _fields_ = [
+ ("ctx", c_void_p),
+ ("write_fn", WRITE_FN),
+ ("flush_fn", FLUSH_FN),
+ ]
+
+
+class PaimonFtindexInputFile(Structure):
+ _fields_ = [
+ ("ctx", c_void_p),
+ ("read_at_fn", READ_AT_FN),
+ ]
+
+
+lib.paimon_ftindex_last_error.argtypes = []
+lib.paimon_ftindex_last_error.restype = c_char_p
+
+lib.paimon_ftindex_writer_open.argtypes = [POINTER(c_char_p), POINTER(c_char_p), c_size_t]
+lib.paimon_ftindex_writer_open.restype = c_void_p
+
+lib.paimon_ftindex_writer_add_document.argtypes = [c_void_p, c_int64, c_char_p]
+lib.paimon_ftindex_writer_add_document.restype = c_int
+
+lib.paimon_ftindex_writer_write_index.argtypes = [c_void_p, PaimonFtindexOutputFile]
+lib.paimon_ftindex_writer_write_index.restype = c_int
+
+lib.paimon_ftindex_writer_free.argtypes = [c_void_p]
+lib.paimon_ftindex_writer_free.restype = None
+
+lib.paimon_ftindex_reader_open.argtypes = [PaimonFtindexInputFile]
+lib.paimon_ftindex_reader_open.restype = c_void_p
+
+lib.paimon_ftindex_reader_search_json.argtypes = [
+ c_void_p,
+ c_char_p,
+ c_size_t,
+ POINTER(c_int64),
+ POINTER(c_float),
+ c_size_t,
+ POINTER(c_size_t),
+]
+lib.paimon_ftindex_reader_search_json.restype = c_int
+
+lib.paimon_ftindex_reader_free.argtypes = [c_void_p]
+lib.paimon_ftindex_reader_free.restype = None
+
+
+def check_status(status):
+ if status != 0:
+ err = lib.paimon_ftindex_last_error()
+ if err:
+ raise RuntimeError(err.decode("utf-8"))
+ raise RuntimeError("native full-text index call failed")
+
+
+def check_ptr(ptr):
+ if not ptr:
+ err = lib.paimon_ftindex_last_error()
+ if err:
+ raise RuntimeError(err.decode("utf-8"))
+ raise RuntimeError("native full-text index returned null")
+ return ptr
diff --git a/python/paimon_ftindex/query.py b/python/paimon_ftindex/query.py
new file mode 100644
index 0000000..01185ae
--- /dev/null
+++ b/python/paimon_ftindex/query.py
@@ -0,0 +1,43 @@
+import json
+
+
+class FullTextQuery:
+ def to_dict(self):
+ raise NotImplementedError
+
+ def to_json(self):
+ return json.dumps(self.to_dict(), separators=(",", ":"))
+
+
+class MatchQuery(FullTextQuery):
+ def __init__(self, terms, column="text", operator="Or", boost=1.0):
+ self.terms = str(terms)
+ self.column = str(column)
+ self.operator = str(operator)
+ self.boost = float(boost)
+
+ def to_dict(self):
+ return {
+ "match": {
+ "column": self.column,
+ "terms": self.terms,
+ "operator": self.operator,
+ "boost": self.boost,
+ }
+ }
+
+
+class PhraseQuery(FullTextQuery):
+ def __init__(self, terms, column="text", slop=0):
+ self.terms = str(terms)
+ self.column = str(column)
+ self.slop = int(slop)
+
+ def to_dict(self):
+ return {
+ "match_phrase": {
+ "column": self.column,
+ "terms": self.terms,
+ "slop": self.slop,
+ }
+ }
diff --git a/python/paimon_ftindex/reader.py b/python/paimon_ftindex/reader.py
new file mode 100644
index 0000000..5cc102b
--- /dev/null
+++ b/python/paimon_ftindex/reader.py
@@ -0,0 +1,69 @@
+import ctypes
+from ctypes import c_float, c_int64, c_size_t, c_void_p
+
+from ._ffi import (
+ READ_AT_FN,
+ PaimonFtindexInputFile,
+ check_ptr,
+ check_status,
+ lib,
+)
+
+
+class FullTextIndexReader:
+ def __init__(self, input_):
+ self._closed = False
+ self._input = input_
+
+ @READ_AT_FN
+ def read_at_fn(ctx, pos, buf, length):
+ try:
+ data = input_.pread(int(pos), int(length))
+ if len(data) != length:
+ return -1
+ ctypes.memmove(buf, data, length)
+ return 0
+ except Exception:
+ return -1
+
+ self._read_at_fn = read_at_fn
+ native_input = PaimonFtindexInputFile(c_void_p(0), read_at_fn)
+ self._ptr = check_ptr(lib.paimon_ftindex_reader_open(native_input))
+
+ def search(self, query, limit=10):
+ if self._closed:
+ raise RuntimeError("FullTextIndexReader is closed")
+ query_json = query.to_json() if hasattr(query, "to_json") else str(query)
+ capacity = int(limit)
+ row_ids = (c_int64 * capacity)()
+ scores = (c_float * capacity)()
+ result_len = c_size_t()
+ check_status(
+ lib.paimon_ftindex_reader_search_json(
+ self._ptr,
+ query_json.encode("utf-8"),
+ capacity,
+ row_ids,
+ scores,
+ capacity,
+ ctypes.byref(result_len),
+ )
+ )
+ size = result_len.value
+ return list(row_ids[:size]), list(scores[:size])
+
+ def close(self):
+ if not self._closed:
+ self._closed = True
+ if self._ptr:
+ lib.paimon_ftindex_reader_free(self._ptr)
+ self._ptr = None
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc, tb):
+ self.close()
+
+ def __del__(self):
+ self.close()
diff --git a/python/paimon_ftindex/writer.py b/python/paimon_ftindex/writer.py
new file mode 100644
index 0000000..93dd42d
--- /dev/null
+++ b/python/paimon_ftindex/writer.py
@@ -0,0 +1,81 @@
+import ctypes
+from ctypes import c_char_p, c_void_p
+
+from ._ffi import (
+ FLUSH_FN,
+ WRITE_FN,
+ PaimonFtindexOutputFile,
+ check_ptr,
+ check_status,
+ lib,
+)
+
+
+class FullTextIndexWriter:
+ def __init__(self, options=None):
+ options = options or {}
+ self._closed = False
+ self._output_refs = None
+ keys = [str(k).encode("utf-8") for k in options.keys()]
+ values = [str(v).encode("utf-8") for v in options.values()]
+ key_array = (c_char_p * len(keys))(*keys) if keys else None
+ value_array = (c_char_p * len(values))(*values) if values else None
+ self._ptr = check_ptr(
+ lib.paimon_ftindex_writer_open(key_array, value_array, len(keys))
+ )
+
+ def add_document(self, row_id, text):
+ if self._closed:
+ raise RuntimeError("FullTextIndexWriter is closed")
+ check_status(
+ lib.paimon_ftindex_writer_add_document(
+ self._ptr, int(row_id), str(text).encode("utf-8")
+ )
+ )
+
+ def write(self, output):
+ if self._closed:
+ raise RuntimeError("FullTextIndexWriter is closed")
+
+ @WRITE_FN
+ def write_fn(ctx, buf, length):
+ try:
+ data = ctypes.string_at(buf, length)
+ output.write(data)
+ return 0
+ except Exception:
+ return -1
+
+ @FLUSH_FN
+ def flush_fn(ctx):
+ try:
+ flush = getattr(output, "flush", None)
+ if flush is not None:
+ flush()
+ return 0
+ except Exception:
+ return -1
+
+ self._output_refs = (write_fn, flush_fn)
+ native_output = PaimonFtindexOutputFile(
+ c_void_p(0),
+ write_fn,
+ flush_fn,
+ )
+ check_status(lib.paimon_ftindex_writer_write_index(self._ptr, native_output))
+
+ def close(self):
+ if not self._closed:
+ self._closed = True
+ if self._ptr:
+ lib.paimon_ftindex_writer_free(self._ptr)
+ self._ptr = None
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc, tb):
+ self.close()
+
+ def __del__(self):
+ self.close()
diff --git a/python/pyproject.toml b/python/pyproject.toml
new file mode 100644
index 0000000..a941384
--- /dev/null
+++ b/python/pyproject.toml
@@ -0,0 +1,16 @@
+[build-system]
+requires = ["setuptools>=61"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "paimon-ftindex"
+version = "0.1.0"
+description = "Python bindings for Apache Paimon Full Text Index"
+requires-python = ">=3.8"
+license = { text = "Apache-2.0" }
+
+[project.optional-dependencies]
+test = ["pytest"]
+
+[tool.setuptools.packages.find]
+where = ["."]
diff --git a/python/setup.py b/python/setup.py
new file mode 100644
index 0000000..6068493
--- /dev/null
+++ b/python/setup.py
@@ -0,0 +1,3 @@
+from setuptools import setup
+
+setup()
diff --git a/python/tests/test_roundtrip.py b/python/tests/test_roundtrip.py
new file mode 100644
index 0000000..2d3e667
--- /dev/null
+++ b/python/tests/test_roundtrip.py
@@ -0,0 +1,25 @@
+from io import BytesIO
+
+from paimon_ftindex import FullTextIndexReader, FullTextIndexWriter, MatchQuery
+
+
+class BytesInput:
+ def __init__(self, data):
+ self._data = data
+
+ def pread(self, pos, length):
+ return self._data[pos:pos + length]
+
+
+def test_python_round_trip():
+ output = BytesIO()
+ with FullTextIndexWriter() as writer:
+ writer.add_document(1, "Apache Paimon full text")
+ writer.add_document(2, "Rust Tantivy search")
+ writer.write(output)
+
+ with FullTextIndexReader(BytesInput(output.getvalue())) as reader:
+ row_ids, scores = reader.search(MatchQuery("paimon"), limit=10)
+
+ assert row_ids == [1]
+ assert scores[0] > 0