You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by ho...@apache.org on 2026/04/05 13:01:44 UTC
(paimon-rust) branch main updated: feat(scan): add bucket pruning, DV/postpone filtering, and DE group pruning (#205)
This is an automated email from the ASF dual-hosted git repository.
hope pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/paimon-rust.git
The following commit(s) were added to refs/heads/main by this push:
new 5d1a930 feat(scan): add bucket pruning, DV/postpone filtering, and DE group pruning (#205)
5d1a930 is described below
commit 5d1a9303d83b660785c22b3691a14dedcf4624b4
Author: Jingsong Lee <ji...@gmail.com>
AuthorDate: Sun Apr 5 21:01:38 2026 +0800
feat(scan): add bucket pruning, DV/postpone filtering, and DE group pruning (#205)
* feat(scan): add bucket predicate filtering, DV/postpone filtering, and data evolution group pruning
---
crates/integration_tests/tests/read_tables.rs | 545 ++++++++-
.../datafusion/src/physical_plan/scan.rs | 29 +-
crates/paimon/src/spec/binary_row.rs | 771 +++++++++++++
crates/paimon/src/spec/core_options.rs | 20 +
crates/paimon/src/spec/data_file.rs | 828 +-------------
crates/paimon/src/spec/mod.rs | 4 +
crates/paimon/src/spec/murmur_hash.rs | 97 ++
crates/paimon/src/spec/objects_file.rs | 40 +-
crates/paimon/src/spec/predicate.rs | 2 +-
crates/paimon/src/spec/stats.rs | 33 +-
crates/paimon/src/table/bin_pack.rs | 4 +-
crates/paimon/src/table/bucket_filter.rs | 193 ++++
crates/paimon/src/table/mod.rs | 2 +
crates/paimon/src/table/source.rs | 2 +-
crates/paimon/src/table/stats_filter.rs | 605 ++++++++++
crates/paimon/src/table/table_scan.rs | 1192 +++++++++-----------
dev/spark/README.md | 41 +
dev/spark/provision.py | 137 +++
18 files changed, 3019 insertions(+), 1526 deletions(-)
diff --git a/crates/integration_tests/tests/read_tables.rs b/crates/integration_tests/tests/read_tables.rs
index c751c8d..008b081 100644
--- a/crates/integration_tests/tests/read_tables.rs
+++ b/crates/integration_tests/tests/read_tables.rs
@@ -792,8 +792,10 @@ async fn test_read_partitioned_table_filter_matches_no_partition() {
);
}
+/// Using an unsupported DataType in a partition predicate should fail-open:
+/// the plan succeeds and returns all partitions (no pruning).
#[tokio::test]
-async fn test_read_partitioned_table_eval_row_error_fails_plan() {
+async fn test_read_partitioned_table_eval_row_error_fails_open() {
use paimon::spec::{ArrayType, DataType, Datum, IntType, PredicateOperator};
let catalog = create_file_system_catalog();
@@ -806,6 +808,7 @@ async fn test_read_partitioned_table_eval_row_error_fails_plan() {
.expect("dt partition column should exist");
// Use an unsupported partition type so remapping succeeds but `eval_row` fails.
+ // The entry-level filter catches the error and fails open (keeps the entry).
let filter = Predicate::Leaf {
column: "dt".into(),
index: dt_index,
@@ -817,15 +820,18 @@ async fn test_read_partitioned_table_eval_row_error_fails_plan() {
let mut read_builder = table.new_read_builder();
read_builder.with_filter(filter);
- let err = read_builder
+ let plan = read_builder
.new_scan()
.plan()
.await
- .expect_err("eval_row error should fail-fast during planning");
+ .expect("Plan should succeed (fail-open on unsupported type)");
- assert!(
- matches!(&err, Error::Unsupported { message } if message.contains("extract_datum")),
- "Expected extract_datum unsupported error, got: {err:?}"
+ // All partitions should survive since the predicate evaluation fails open.
+ let seen_partitions = extract_plan_partitions(&plan);
+ assert_eq!(
+ seen_partitions,
+ HashSet::from(["2024-01-01".into(), "2024-01-02".into()]),
+ "Unsupported predicate type should fail-open and keep all partitions"
);
}
@@ -1467,3 +1473,530 @@ async fn test_read_complex_type_table() {
"Complex type table should return correct ARRAY, MAP, and STRUCT values"
);
}
+
+// ---------------------------------------------------------------------------
+// PK-without-DV and non-PK-with-DV tests
+// ---------------------------------------------------------------------------
+
+/// Reading a primary-key table without deletion vectors should return an Unsupported error.
+#[tokio::test]
+async fn test_read_pk_table_without_dv_returns_error() {
+ let catalog = create_file_system_catalog();
+ let table = get_table_from_catalog(&catalog, "simple_pk_table").await;
+
+ let read_builder = table.new_read_builder();
+ let scan = read_builder.new_scan();
+ let plan = scan.plan().await.expect("Failed to plan scan");
+ assert!(
+ !plan.splits().is_empty(),
+ "PK table should have splits to read"
+ );
+
+ let read = table.new_read_builder().new_read();
+ let result = read
+ .expect("new_read should succeed")
+ .to_arrow(plan.splits());
+ let err = result
+ .err()
+ .expect("Reading PK table without DV should fail");
+
+ assert!(
+ matches!(&err, Error::Unsupported { message } if message.contains("primary-key")),
+ "Expected Unsupported error about primary-key tables, got: {err:?}"
+ );
+}
+
+/// Reading a non-PK (append-only) table with deletion vectors enabled should work correctly.
+/// Level-0 files must NOT be filtered out since there is no PK merge.
+#[tokio::test]
+async fn test_read_non_pk_table_with_dv() {
+ let (_, batches) = scan_and_read_with_fs_catalog("simple_dv_log_table", None).await;
+ let actual = extract_id_name(&batches);
+ let expected = vec![
+ (1, "alice".to_string()),
+ (2, "bob".to_string()),
+ (3, "carol".to_string()),
+ ];
+ assert_eq!(
+ actual, expected,
+ "Non-PK table with DV enabled should return all rows (level-0 files kept)"
+ );
+}
+
+/// Postpone bucket PK table (bucket = -2): uncompacted data sits in bucket-postpone
+/// and should NOT be visible to batch readers. The plan should produce no splits.
+#[tokio::test]
+async fn test_read_postpone_bucket_pk_table_returns_empty() {
+ let catalog = create_file_system_catalog();
+ let table = get_table_from_catalog(&catalog, "postpone_bucket_pk_table").await;
+
+ let read_builder = table.new_read_builder();
+ let scan = read_builder.new_scan();
+ let plan = scan.plan().await.expect("Failed to plan scan");
+
+ assert!(
+ plan.splits().is_empty(),
+ "Postpone bucket PK table should have no visible splits before compaction"
+ );
+}
+
+// ---------------------------------------------------------------------------
+// Data evolution predicate filtering tests
+// ---------------------------------------------------------------------------
+
+/// Data evolution group-level predicate filtering: after group_by_overlapping_row_id,
+/// merged stats across files in each group should allow pruning entire groups.
+#[tokio::test]
+async fn test_data_evolution_table_with_filter() {
+ use paimon::spec::{Datum, PredicateBuilder};
+
+ let catalog = create_file_system_catalog();
+ let table = get_table_from_catalog(&catalog, "data_evolution_table").await;
+ let pb = PredicateBuilder::new(table.schema().fields());
+
+ // Filter: value > 300 should keep only groups containing rows with value > 300.
+ // Expected rows after merge: (4, 'dave', 400), (5, 'eve', 500)
+ let filter = pb
+ .greater_than("value", Datum::Int(300))
+ .expect("Failed to build predicate");
+
+ let (plan, batches) = scan_and_read_with_filter(&table, filter).await;
+
+ // The first batch (rows 1-3) was MERGE INTO'd, creating overlapping row_id groups.
+ // Their max value is 300, so the group should be pruned by value > 300.
+ // The second batch (rows 4-5) has values 400, 500 and should survive.
+ assert!(
+ !plan.splits().is_empty(),
+ "Some splits should survive the filter"
+ );
+
+ let mut rows: Vec<(i32, String, i32)> = Vec::new();
+ for batch in &batches {
+ let id = batch
+ .column_by_name("id")
+ .and_then(|c| c.as_any().downcast_ref::<Int32Array>())
+ .expect("id");
+ let name = batch
+ .column_by_name("name")
+ .and_then(|c| c.as_any().downcast_ref::<StringArray>())
+ .expect("name");
+ let value = batch
+ .column_by_name("value")
+ .and_then(|c| c.as_any().downcast_ref::<Int32Array>())
+ .expect("value");
+ for i in 0..batch.num_rows() {
+ rows.push((id.value(i), name.value(i).to_string(), value.value(i)));
+ }
+ }
+ rows.sort_by_key(|(id, _, _)| *id);
+
+ assert_eq!(
+ rows,
+ vec![(4, "dave".into(), 400), (5, "eve".into(), 500),],
+ "Data evolution group-level pruning should filter out groups where value <= 300"
+ );
+}
+
+// ---------------------------------------------------------------------------
+// Bucket predicate filtering tests
+// ---------------------------------------------------------------------------
+
+/// Bucket predicate filtering: when filtering by bucket key (primary key) with an
+/// equality predicate, only splits whose bucket matches the computed target bucket
+/// should survive. This tests the full pipeline: extract bucket predicate → compute
+/// target bucket via MurmurHash3 → filter manifest entries by bucket.
+#[tokio::test]
+async fn test_bucket_predicate_filtering() {
+ use paimon::spec::{Datum, PredicateBuilder};
+
+ let catalog = create_file_system_catalog();
+ let table = get_table_from_catalog(&catalog, "multi_bucket_pk_table").await;
+ let schema = table.schema();
+ let pb = PredicateBuilder::new(schema.fields());
+
+ // Get full plan without filter to see all buckets
+ let full_plan = plan_table(&table, None).await;
+ let all_buckets: HashSet<i32> = full_plan.splits().iter().map(|s| s.bucket()).collect();
+ assert!(
+ all_buckets.len() > 1,
+ "multi_bucket_pk_table should have data in multiple buckets, got: {all_buckets:?}"
+ );
+
+ // Filter by id = 1 (bucket key). This should compute the target bucket and
+ // only return splits from that bucket.
+ let filter = pb
+ .equal("id", Datum::Int(1))
+ .expect("Failed to build predicate");
+ let (plan, batches) = scan_and_read_with_filter(&table, filter).await;
+
+ let filtered_buckets: HashSet<i32> = plan.splits().iter().map(|s| s.bucket()).collect();
+ assert_eq!(
+ filtered_buckets.len(),
+ 1,
+ "Bucket predicate filtering should narrow to exactly one bucket, got: {filtered_buckets:?}"
+ );
+ assert!(
+ filtered_buckets.is_subset(&all_buckets),
+ "Filtered bucket should be one of the original buckets"
+ );
+
+ let actual = extract_id_name(&batches);
+ // Bucket filtering is at the bucket level, not row level. Other rows that
+ // hash to the same bucket will also be returned.
+ let ids: HashSet<i32> = actual.iter().map(|(id, _)| *id).collect();
+ assert!(
+ ids.contains(&1),
+ "Row with id=1 should be in the filtered result, got: {actual:?}"
+ );
+ // Verify we got fewer rows than the full table (8 rows)
+ assert!(
+ actual.len() < 8,
+ "Bucket filtering should return fewer rows than the full table, got: {}",
+ actual.len()
+ );
+}
+
+/// Bucket predicate filtering with IN predicate: multiple target buckets.
+#[tokio::test]
+async fn test_bucket_predicate_filtering_in() {
+ use paimon::spec::{Datum, PredicateBuilder};
+
+ let catalog = create_file_system_catalog();
+ let table = get_table_from_catalog(&catalog, "multi_bucket_pk_table").await;
+ let schema = table.schema();
+ let pb = PredicateBuilder::new(schema.fields());
+
+ // Filter by id IN (1, 5) — may hash to different buckets
+ let filter = pb
+ .is_in("id", vec![Datum::Int(1), Datum::Int(5)])
+ .expect("Failed to build predicate");
+ let (plan, batches) = scan_and_read_with_filter(&table, filter).await;
+
+ let filtered_buckets: HashSet<i32> = plan.splits().iter().map(|s| s.bucket()).collect();
+ assert!(
+ filtered_buckets.len() <= 2,
+ "IN predicate with 2 values should produce at most 2 target buckets, got: {filtered_buckets:?}"
+ );
+
+ let actual = extract_id_name(&batches);
+ // Should contain exactly id=1 and id=5
+ let ids: HashSet<i32> = actual.iter().map(|(id, _)| *id).collect();
+ assert!(
+ ids.contains(&1) && ids.contains(&5),
+ "Should return rows for id=1 and id=5, got: {actual:?}"
+ );
+}
+
+// ---------------------------------------------------------------------------
+// Time travel integration tests
+// ---------------------------------------------------------------------------
+
+/// Time travel by snapshot id: snapshot 1 should return only the first batch.
+#[tokio::test]
+async fn test_time_travel_by_snapshot_id() {
+ let catalog = create_file_system_catalog();
+ let table = get_table_from_catalog(&catalog, "time_travel_table").await;
+
+ // Snapshot 1: (1, 'alice'), (2, 'bob')
+ let table_snap1 = table.copy_with_options(HashMap::from([(
+ "scan.snapshot-id".to_string(),
+ "1".to_string(),
+ )]));
+ let rb = table_snap1.new_read_builder();
+ let plan = rb.new_scan().plan().await.expect("plan snap1");
+ let read = rb.new_read().expect("read snap1");
+ let batches: Vec<RecordBatch> = read
+ .to_arrow(plan.splits())
+ .expect("stream")
+ .try_collect()
+ .await
+ .expect("collect");
+ let actual = extract_id_name(&batches);
+ assert_eq!(
+ actual,
+ vec![(1, "alice".into()), (2, "bob".into())],
+ "Snapshot 1 should contain only the first batch"
+ );
+
+ // Snapshot 2: (1, 'alice'), (2, 'bob'), (3, 'carol'), (4, 'dave')
+ let table_snap2 = table.copy_with_options(HashMap::from([(
+ "scan.snapshot-id".to_string(),
+ "2".to_string(),
+ )]));
+ let rb2 = table_snap2.new_read_builder();
+ let plan2 = rb2.new_scan().plan().await.expect("plan snap2");
+ let read2 = rb2.new_read().expect("read snap2");
+ let batches2: Vec<RecordBatch> = read2
+ .to_arrow(plan2.splits())
+ .expect("stream")
+ .try_collect()
+ .await
+ .expect("collect");
+ let actual2 = extract_id_name(&batches2);
+ assert_eq!(
+ actual2,
+ vec![
+ (1, "alice".into()),
+ (2, "bob".into()),
+ (3, "carol".into()),
+ (4, "dave".into()),
+ ],
+ "Snapshot 2 should contain all rows"
+ );
+}
+
+/// Time travel by tag name.
+#[tokio::test]
+async fn test_time_travel_by_tag_name() {
+ let catalog = create_file_system_catalog();
+ let table = get_table_from_catalog(&catalog, "time_travel_table").await;
+
+ // Tag 'snapshot1' -> snapshot 1: (1, 'alice'), (2, 'bob')
+ let table_tag1 = table.copy_with_options(HashMap::from([(
+ "scan.tag-name".to_string(),
+ "snapshot1".to_string(),
+ )]));
+ let rb = table_tag1.new_read_builder();
+ let plan = rb.new_scan().plan().await.expect("plan tag1");
+ let read = rb.new_read().expect("read tag1");
+ let batches: Vec<RecordBatch> = read
+ .to_arrow(plan.splits())
+ .expect("stream")
+ .try_collect()
+ .await
+ .expect("collect");
+ let actual = extract_id_name(&batches);
+ assert_eq!(
+ actual,
+ vec![(1, "alice".into()), (2, "bob".into())],
+ "Tag 'snapshot1' should return snapshot 1 data"
+ );
+
+ // Tag 'snapshot2' -> snapshot 2: all 4 rows
+ let table_tag2 = table.copy_with_options(HashMap::from([(
+ "scan.tag-name".to_string(),
+ "snapshot2".to_string(),
+ )]));
+ let rb2 = table_tag2.new_read_builder();
+ let plan2 = rb2.new_scan().plan().await.expect("plan tag2");
+ let read2 = rb2.new_read().expect("read tag2");
+ let batches2: Vec<RecordBatch> = read2
+ .to_arrow(plan2.splits())
+ .expect("stream")
+ .try_collect()
+ .await
+ .expect("collect");
+ let actual2 = extract_id_name(&batches2);
+ assert_eq!(
+ actual2,
+ vec![
+ (1, "alice".into()),
+ (2, "bob".into()),
+ (3, "carol".into()),
+ (4, "dave".into()),
+ ],
+ "Tag 'snapshot2' should return all rows"
+ );
+}
+
+// ---------------------------------------------------------------------------
+// Data evolution + drop column tests
+// ---------------------------------------------------------------------------
+
+/// Data evolution + drop column: old rows that were MERGE INTO'd should have NULL
+/// for the newly added column (no file in the merge group provides it).
+#[tokio::test]
+async fn test_read_data_evolution_drop_column() {
+ let (_, batches) = scan_and_read_with_fs_catalog("data_evolution_drop_column", None).await;
+
+ let mut rows: Vec<(i32, String, i32, Option<String>)> = Vec::new();
+ for batch in &batches {
+ let id = batch
+ .column_by_name("id")
+ .and_then(|c| c.as_any().downcast_ref::<Int32Array>())
+ .expect("id");
+ let name = batch
+ .column_by_name("name")
+ .and_then(|c| c.as_any().downcast_ref::<StringArray>())
+ .expect("name");
+ let value = batch
+ .column_by_name("value")
+ .and_then(|c| c.as_any().downcast_ref::<Int32Array>())
+ .expect("value");
+ let extra = batch
+ .column_by_name("extra")
+ .and_then(|c| c.as_any().downcast_ref::<StringArray>())
+ .expect("extra");
+ for i in 0..batch.num_rows() {
+ let extra_val = if extra.is_null(i) {
+ None
+ } else {
+ Some(extra.value(i).to_string())
+ };
+ rows.push((
+ id.value(i),
+ name.value(i).to_string(),
+ value.value(i),
+ extra_val,
+ ));
+ }
+ }
+ rows.sort_by_key(|(id, _, _, _)| *id);
+
+ assert_eq!(
+ rows,
+ vec![
+ (1, "alice-v2".into(), 100, None),
+ (2, "bob".into(), 200, None),
+ (3, "carol".into(), 300, Some("new".into())),
+ ],
+ "Old rows should have NULL for 'extra' (added after MERGE INTO), new rows should have it"
+ );
+}
+
+// ---------------------------------------------------------------------------
+// Limit pushdown with data predicates test
+// ---------------------------------------------------------------------------
+
+/// Limit pushdown must be disabled when data predicates exist.
+/// Otherwise merged_row_count (pre-filter) could cause early stop, returning
+/// fewer rows than the limit after filtering.
+#[tokio::test]
+async fn test_limit_pushdown_disabled_with_data_predicates() {
+ use paimon::spec::{Datum, PredicateBuilder};
+
+ let catalog = create_file_system_catalog();
+ let table = get_table_from_catalog(&catalog, "data_evolution_table").await;
+ let pb = PredicateBuilder::new(table.schema().fields());
+
+ // Filter: value >= 100 (matches all rows). With limit=2, if limit pushdown
+ // were applied, it might stop after the first split (merged_row_count >= 2)
+ // but that split's rows might all be filtered out by a stricter predicate.
+ // Here we use a lenient predicate to verify the plan still includes enough splits.
+ let filter = pb
+ .greater_than("value", Datum::Int(0))
+ .expect("Failed to build predicate");
+
+ let mut read_builder = table.new_read_builder();
+ read_builder.with_filter(filter);
+ read_builder.with_limit(2);
+ let scan = read_builder.new_scan();
+ let plan = scan.plan().await.expect("Failed to plan scan");
+
+ // With data predicates, limit pushdown should be disabled, so we should get
+ // the same number of splits as without limit.
+ let full_plan = plan_table(&table, None).await;
+ assert_eq!(
+ plan.splits().len(),
+ full_plan.splits().len(),
+ "With data predicates, limit pushdown should be disabled — split count should match full plan"
+ );
+}
+
+// ---------------------------------------------------------------------------
+// String bucket key tests (variable-length hash compatibility with Java)
+// ---------------------------------------------------------------------------
+
+/// Helper to extract (code, value) rows from batches.
+fn extract_code_value(batches: &[RecordBatch]) -> Vec<(String, i32)> {
+ let mut rows = Vec::new();
+ for batch in batches {
+ let code = batch
+ .column_by_name("code")
+ .and_then(|c| c.as_any().downcast_ref::<StringArray>())
+ .expect("code");
+ let value = batch
+ .column_by_name("value")
+ .and_then(|c| c.as_any().downcast_ref::<Int32Array>())
+ .expect("value");
+ for i in 0..batch.num_rows() {
+ rows.push((code.value(i).to_string(), value.value(i)));
+ }
+ }
+ rows.sort_by(|a, b| a.0.cmp(&b.0));
+ rows
+}
+
+/// Bucket predicate filtering with short string keys (<=7 bytes, inline encoding).
+#[tokio::test]
+async fn test_bucket_predicate_filtering_short_string_key() {
+ use paimon::spec::{Datum, PredicateBuilder};
+
+ let catalog = create_file_system_catalog();
+ let table = get_table_from_catalog(&catalog, "string_bucket_short_key").await;
+ let pb = PredicateBuilder::new(table.schema().fields());
+
+ let full_plan = plan_table(&table, None).await;
+ let all_buckets: HashSet<i32> = full_plan.splits().iter().map(|s| s.bucket()).collect();
+ assert!(
+ all_buckets.len() > 1,
+ "string_bucket_short_key should have data in multiple buckets, got: {all_buckets:?}"
+ );
+
+ // Filter by code = 'aaa' (short string, inline BinaryRow encoding)
+ let filter = pb
+ .equal("code", Datum::String("aaa".into()))
+ .expect("Failed to build predicate");
+ let (plan, batches) = scan_and_read_with_filter(&table, filter).await;
+
+ let filtered_buckets: HashSet<i32> = plan.splits().iter().map(|s| s.bucket()).collect();
+ assert_eq!(
+ filtered_buckets.len(),
+ 1,
+ "Short string bucket filtering should narrow to one bucket, got: {filtered_buckets:?}"
+ );
+
+ let actual = extract_code_value(&batches);
+ let codes: HashSet<&str> = actual.iter().map(|(c, _)| c.as_str()).collect();
+ assert!(
+ codes.contains("aaa"),
+ "Row with code='aaa' should be in the result, got: {actual:?}"
+ );
+ assert!(
+ actual.len() < 8,
+ "Bucket filtering should return fewer rows than the full table, got: {}",
+ actual.len()
+ );
+}
+
+/// Bucket predicate filtering with long string keys (>7 bytes, variable-length encoding).
+#[tokio::test]
+async fn test_bucket_predicate_filtering_long_string_key() {
+ use paimon::spec::{Datum, PredicateBuilder};
+
+ let catalog = create_file_system_catalog();
+ let table = get_table_from_catalog(&catalog, "string_bucket_long_key").await;
+ let pb = PredicateBuilder::new(table.schema().fields());
+
+ let full_plan = plan_table(&table, None).await;
+ let all_buckets: HashSet<i32> = full_plan.splits().iter().map(|s| s.bucket()).collect();
+ assert!(
+ all_buckets.len() > 1,
+ "string_bucket_long_key should have data in multiple buckets, got: {all_buckets:?}"
+ );
+
+ // Filter by code = 'alpha-long-key' (>7 bytes, var-length BinaryRow encoding with 8-byte padding)
+ let filter = pb
+ .equal("code", Datum::String("alpha-long-key".into()))
+ .expect("Failed to build predicate");
+ let (plan, batches) = scan_and_read_with_filter(&table, filter).await;
+
+ let filtered_buckets: HashSet<i32> = plan.splits().iter().map(|s| s.bucket()).collect();
+ assert_eq!(
+ filtered_buckets.len(),
+ 1,
+ "Long string bucket filtering should narrow to one bucket, got: {filtered_buckets:?}"
+ );
+
+ let actual = extract_code_value(&batches);
+ let codes: HashSet<&str> = actual.iter().map(|(c, _)| c.as_str()).collect();
+ assert!(
+ codes.contains("alpha-long-key"),
+ "Row with code='alpha-long-key' should be in the result, got: {actual:?}"
+ );
+ assert!(
+ actual.len() < 8,
+ "Bucket filtering should return fewer rows than the full table, got: {}",
+ actual.len()
+ );
+}
diff --git a/crates/integrations/datafusion/src/physical_plan/scan.rs b/crates/integrations/datafusion/src/physical_plan/scan.rs
index b047d3a..a8c9aea 100644
--- a/crates/integrations/datafusion/src/physical_plan/scan.rs
+++ b/crates/integrations/datafusion/src/physical_plan/scan.rs
@@ -16,10 +16,8 @@
// under the License.
use std::any::Any;
-use std::pin::Pin;
use std::sync::Arc;
-use datafusion::arrow::array::RecordBatch;
use datafusion::arrow::datatypes::SchemaRef as ArrowSchemaRef;
use datafusion::error::Result as DFResult;
use datafusion::execution::{SendableRecordBatchStream, TaskContext};
@@ -27,7 +25,7 @@ use datafusion::physical_expr::EquivalenceProperties;
use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType};
use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
use datafusion::physical_plan::{DisplayAs, ExecutionPlan, Partitioning, PlanProperties};
-use futures::{Stream, StreamExt, TryStreamExt};
+use futures::{StreamExt, TryStreamExt};
use paimon::table::Table;
use paimon::DataSplit;
@@ -128,7 +126,6 @@ impl ExecutionPlan for PaimonTableScan {
let table = self.table.clone();
let schema = self.schema();
let projected_columns = self.projected_columns.clone();
- let limit = self.limit;
let fut = async move {
let mut read_builder = table.new_read_builder();
@@ -148,31 +145,9 @@ impl ExecutionPlan for PaimonTableScan {
))
};
- let stream = futures::stream::once(fut).try_flatten();
-
- // Enforce the final LIMIT at the DataFusion execution layer.
- let limited_stream: Pin<Box<dyn Stream<Item = DFResult<RecordBatch>> + Send>> =
- if let Some(limit) = limit {
- let mut remaining = limit;
- Box::pin(stream.try_filter_map(move |batch| {
- futures::future::ready(if remaining == 0 {
- Ok(None)
- } else if batch.num_rows() <= remaining {
- remaining -= batch.num_rows();
- Ok(Some(batch))
- } else {
- let limited_batch = batch.slice(0, remaining);
- remaining = 0;
- Ok(Some(limited_batch))
- })
- }))
- } else {
- Box::pin(stream)
- };
-
Ok(Box::pin(RecordBatchStreamAdapter::new(
self.schema(),
- limited_stream,
+ futures::stream::once(fut).try_flatten(),
)))
}
}
diff --git a/crates/paimon/src/spec/binary_row.rs b/crates/paimon/src/spec/binary_row.rs
new file mode 100644
index 0000000..1621476
--- /dev/null
+++ b/crates/paimon/src/spec/binary_row.rs
@@ -0,0 +1,771 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! BinaryRow: an implementation of InternalRow backed by raw binary bytes,
+//! and BinaryRowBuilder for constructing BinaryRow instances.
+
+use crate::spec::murmur_hash::hash_by_words;
+use serde::{Deserialize, Serialize};
+
+pub const EMPTY_BINARY_ROW: BinaryRow = BinaryRow::new(0);
+
+/// Highest bit mask for detecting inline vs variable-length encoding.
+const HIGHEST_FIRST_BIT: u64 = 0x80 << 56;
+
+/// Mask to extract the 7-bit length from an inline-encoded value.
+const HIGHEST_SECOND_TO_EIGHTH_BIT: u64 = 0x7F << 56;
+
+/// An implementation of InternalRow backed by raw binary bytes.
+///
+/// Binary layout (little-endian):
+/// ```text
+/// | header (8 bytes) | null bit set (8-byte aligned) | fixed-length (8B per field) | variable-length |
+/// ```
+///
+/// Impl Reference: <https://github.com/apache/paimon/blob/release-0.8.2/paimon-common/src/main/java/org/apache/paimon/data/BinaryRow.java>
+#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct BinaryRow {
+ arity: i32,
+ null_bits_size_in_bytes: i32,
+
+ #[serde(with = "serde_bytes")]
+ data: Vec<u8>,
+}
+
+impl BinaryRow {
+ pub const HEADER_SIZE_IN_BYTES: i32 = 8;
+
+ pub const fn cal_bit_set_width_in_bytes(arity: i32) -> i32 {
+ ((arity + 63 + Self::HEADER_SIZE_IN_BYTES) / 64) * 8
+ }
+
+ pub const fn cal_fix_part_size_in_bytes(arity: i32) -> i32 {
+ Self::cal_bit_set_width_in_bytes(arity) + 8 * arity
+ }
+
+ pub const fn new(arity: i32) -> Self {
+ Self {
+ arity,
+ null_bits_size_in_bytes: Self::cal_bit_set_width_in_bytes(arity),
+ data: Vec::new(),
+ }
+ }
+
+ pub fn from_bytes(arity: i32, data: Vec<u8>) -> Self {
+ let null_bits_size_in_bytes = Self::cal_bit_set_width_in_bytes(arity);
+ Self {
+ arity,
+ null_bits_size_in_bytes,
+ data,
+ }
+ }
+
+ pub fn from_serialized_bytes(data: &[u8]) -> crate::Result<Self> {
+ if data.len() < 4 {
+ return Err(crate::Error::UnexpectedError {
+ message: format!(
+ "BinaryRow: serialized data too short for arity prefix: {} bytes",
+ data.len()
+ ),
+ source: None,
+ });
+ }
+ let arity = i32::from_be_bytes([data[0], data[1], data[2], data[3]]);
+ Ok(Self::from_bytes(arity, data[4..].to_vec()))
+ }
+
+ pub fn arity(&self) -> i32 {
+ self.arity
+ }
+
+ pub fn is_empty(&self) -> bool {
+ self.data.is_empty()
+ }
+
+ pub fn data(&self) -> &[u8] {
+ &self.data
+ }
+
+ pub fn is_null_at(&self, pos: usize) -> bool {
+ let bit_index = pos + Self::HEADER_SIZE_IN_BYTES as usize;
+ let byte_index = bit_index / 8;
+ let bit_offset = bit_index % 8;
+ (self.data[byte_index] & (1 << bit_offset)) != 0
+ }
+
+ fn field_offset(&self, pos: usize) -> usize {
+ self.null_bits_size_in_bytes as usize + pos * 8
+ }
+
+ fn read_slice<const N: usize>(&self, offset: usize) -> crate::Result<[u8; N]> {
+ self.data
+ .get(offset..offset + N)
+ .and_then(|s| s.try_into().ok())
+ .ok_or_else(|| crate::Error::UnexpectedError {
+ message: format!(
+ "BinaryRow: read {N} bytes at offset {offset} exceeds data length {}",
+ self.data.len()
+ ),
+ source: None,
+ })
+ }
+
+ fn read_byte_at(&self, offset: usize) -> crate::Result<u8> {
+ self.data
+ .get(offset)
+ .copied()
+ .ok_or_else(|| crate::Error::UnexpectedError {
+ message: format!(
+ "BinaryRow: read 1 byte at offset {offset} exceeds data length {}",
+ self.data.len()
+ ),
+ source: None,
+ })
+ }
+
+ fn read_i64_at(&self, offset: usize) -> crate::Result<i64> {
+ self.read_slice::<8>(offset).map(i64::from_le_bytes)
+ }
+
+ fn read_i32_at(&self, offset: usize) -> crate::Result<i32> {
+ self.read_slice::<4>(offset).map(i32::from_le_bytes)
+ }
+
+ pub fn get_boolean(&self, pos: usize) -> crate::Result<bool> {
+ self.read_byte_at(self.field_offset(pos)).map(|b| b != 0)
+ }
+
+ pub fn get_byte(&self, pos: usize) -> crate::Result<i8> {
+ self.read_byte_at(self.field_offset(pos)).map(|b| b as i8)
+ }
+
+ pub fn get_short(&self, pos: usize) -> crate::Result<i16> {
+ self.read_slice::<2>(self.field_offset(pos))
+ .map(i16::from_le_bytes)
+ }
+
+ pub fn get_int(&self, pos: usize) -> crate::Result<i32> {
+ self.read_i32_at(self.field_offset(pos))
+ }
+
+ pub fn get_long(&self, pos: usize) -> crate::Result<i64> {
+ self.read_i64_at(self.field_offset(pos))
+ }
+
+ pub fn get_float(&self, pos: usize) -> crate::Result<f32> {
+ self.read_slice::<4>(self.field_offset(pos))
+ .map(f32::from_le_bytes)
+ }
+
+ pub fn get_double(&self, pos: usize) -> crate::Result<f64> {
+ self.read_slice::<8>(self.field_offset(pos))
+ .map(f64::from_le_bytes)
+ }
+
+ fn resolve_var_length_field(&self, pos: usize) -> crate::Result<(usize, usize)> {
+ let field_off = self.field_offset(pos);
+ let raw = self.read_i64_at(field_off)? as u64;
+
+ let (start, len) = if raw & HIGHEST_FIRST_BIT == 0 {
+ let offset = (raw >> 32) as usize;
+ let len = (raw & 0xFFFF_FFFF) as usize;
+ (offset, len)
+ } else {
+ let len = ((raw & HIGHEST_SECOND_TO_EIGHTH_BIT) >> 56) as usize;
+ (field_off, len)
+ };
+
+ let end = start
+ .checked_add(len)
+ .ok_or_else(|| crate::Error::UnexpectedError {
+ message: format!(
+ "BinaryRow: var-len field at pos {pos}: offset {start} + len {len} overflows"
+ ),
+ source: None,
+ })?;
+ if end > self.data.len() {
+ return Err(crate::Error::UnexpectedError {
+ message: format!(
+ "BinaryRow: var-len field at pos {pos}: range [{start}..{end}) exceeds data length {}",
+ self.data.len()
+ ),
+ source: None,
+ });
+ }
+ Ok((start, len))
+ }
+
+ pub fn get_binary(&self, pos: usize) -> crate::Result<&[u8]> {
+ let (start, len) = self.resolve_var_length_field(pos)?;
+ Ok(&self.data[start..start + len])
+ }
+
+ pub fn get_string(&self, pos: usize) -> crate::Result<&str> {
+ let bytes = self.get_binary(pos)?;
+ std::str::from_utf8(bytes).map_err(|e| crate::Error::UnexpectedError {
+ message: format!("BinaryRow: invalid UTF-8 in string field at pos {pos}: {e}"),
+ source: Some(Box::new(e)),
+ })
+ }
+
+ pub(crate) fn get_decimal_unscaled(&self, pos: usize, precision: u32) -> crate::Result<i128> {
+ if precision <= 18 {
+ Ok(self.get_long(pos)? as i128)
+ } else {
+ let bytes = self.get_binary(pos)?;
+ if bytes.is_empty() {
+ return Err(crate::Error::UnexpectedError {
+ message: format!("BinaryRow: empty bytes for non-compact Decimal at pos {pos}"),
+ source: None,
+ });
+ }
+ let negative = bytes[0] & 0x80 != 0;
+ let mut val: i128 = if negative { -1 } else { 0 };
+ for &b in bytes {
+ val = (val << 8) | (b as i128);
+ }
+ Ok(val)
+ }
+ }
+
+ pub(crate) fn get_timestamp_raw(
+ &self,
+ pos: usize,
+ precision: u32,
+ ) -> crate::Result<(i64, i32)> {
+ if precision <= 3 {
+ Ok((self.get_long(pos)?, 0))
+ } else {
+ let field_off = self.field_offset(pos);
+ let offset_and_nano = self.read_i64_at(field_off)? as u64;
+ let offset = (offset_and_nano >> 32) as usize;
+ let nano_of_milli = offset_and_nano as i32;
+
+ if offset + 8 > self.data.len() {
+ return Err(crate::Error::UnexpectedError {
+ message: format!(
+ "BinaryRow: non-compact Timestamp at pos {pos}: offset {offset} + 8 exceeds data length {}",
+ self.data.len()
+ ),
+ source: None,
+ });
+ }
+ let millis = i64::from_le_bytes(self.read_slice::<8>(offset)?);
+ Ok((millis, nano_of_milli))
+ }
+ }
+
+ pub fn hash_code(&self) -> i32 {
+ hash_by_words(&self.data)
+ }
+
+ /// Build a BinaryRow from typed Datum values using `BinaryRowBuilder`.
+ pub fn from_datums(datums: &[(&crate::spec::Datum, &crate::spec::DataType)]) -> Option<Self> {
+ let arity = datums.len() as i32;
+ let mut builder = BinaryRowBuilder::new(arity);
+
+ for (pos, (datum, data_type)) in datums.iter().enumerate() {
+ match datum {
+ crate::spec::Datum::Bool(v) => builder.write_boolean(pos, *v),
+ crate::spec::Datum::TinyInt(v) => builder.write_byte(pos, *v),
+ crate::spec::Datum::SmallInt(v) => builder.write_short(pos, *v),
+ crate::spec::Datum::Int(v)
+ | crate::spec::Datum::Date(v)
+ | crate::spec::Datum::Time(v) => builder.write_int(pos, *v),
+ crate::spec::Datum::Long(v) => builder.write_long(pos, *v),
+ crate::spec::Datum::Float(v) => builder.write_float(pos, *v),
+ crate::spec::Datum::Double(v) => builder.write_double(pos, *v),
+ crate::spec::Datum::Timestamp { millis, nanos } => {
+ let precision = match data_type {
+ crate::spec::DataType::Timestamp(ts) => ts.precision(),
+ _ => 3,
+ };
+ if precision <= 3 {
+ builder.write_timestamp_compact(pos, *millis);
+ } else {
+ builder.write_timestamp_non_compact(pos, *millis, *nanos);
+ }
+ }
+ crate::spec::Datum::LocalZonedTimestamp { millis, nanos } => {
+ let precision = match data_type {
+ crate::spec::DataType::LocalZonedTimestamp(ts) => ts.precision(),
+ _ => 3,
+ };
+ if precision <= 3 {
+ builder.write_timestamp_compact(pos, *millis);
+ } else {
+ builder.write_timestamp_non_compact(pos, *millis, *nanos);
+ }
+ }
+ crate::spec::Datum::Decimal {
+ unscaled,
+ precision,
+ ..
+ } => {
+ if *precision <= 18 {
+ builder.write_decimal_compact(pos, *unscaled as i64);
+ } else {
+ builder.write_decimal_var_len(pos, *unscaled);
+ }
+ }
+ crate::spec::Datum::String(s) => {
+ if s.len() <= 7 {
+ builder.write_string_inline(pos, s);
+ } else {
+ builder.write_string(pos, s);
+ }
+ }
+ crate::spec::Datum::Bytes(b) => {
+ if b.len() <= 7 {
+ builder.write_binary_inline(pos, b);
+ } else {
+ builder.write_binary(pos, b);
+ }
+ }
+ }
+ }
+
+ let row = builder.build();
+ Some(row)
+ }
+
+ pub fn compute_bucket_from_datums(
+ datums: &[(&crate::spec::Datum, &crate::spec::DataType)],
+ total_buckets: i32,
+ ) -> Option<i32> {
+ let row = Self::from_datums(datums)?;
+ let hash = row.hash_code();
+ Some((hash % total_buckets).abs())
+ }
+}
+
+/// Builder for constructing BinaryRow instances matching Java's BinaryRowWriter layout.
+///
+/// Layout: header (8 bytes) | null bit set (aligned) | fixed-length (8B per field) | var-length
+pub(crate) struct BinaryRowBuilder {
+ arity: i32,
+ null_bits_size: usize,
+ data: Vec<u8>,
+}
+
+#[allow(dead_code)]
+impl BinaryRowBuilder {
+ pub fn new(arity: i32) -> Self {
+ let null_bits_size = BinaryRow::cal_bit_set_width_in_bytes(arity) as usize;
+ let fixed_part_size = null_bits_size + (arity as usize) * 8;
+ Self {
+ arity,
+ null_bits_size,
+ data: vec![0u8; fixed_part_size],
+ }
+ }
+
+ fn field_offset(&self, pos: usize) -> usize {
+ self.null_bits_size + pos * 8
+ }
+
+ pub fn set_null_at(&mut self, pos: usize) {
+ let bit_index = pos + BinaryRow::HEADER_SIZE_IN_BYTES as usize;
+ let byte_index = bit_index / 8;
+ let bit_offset = bit_index % 8;
+ self.data[byte_index] |= 1 << bit_offset;
+ let offset = self.field_offset(pos);
+ self.data[offset..offset + 8].fill(0);
+ }
+
+ pub fn write_boolean(&mut self, pos: usize, value: bool) {
+ let offset = self.field_offset(pos);
+ self.data[offset] = u8::from(value);
+ }
+
+ pub fn write_byte(&mut self, pos: usize, value: i8) {
+ let offset = self.field_offset(pos);
+ self.data[offset] = value as u8;
+ }
+
+ pub fn write_short(&mut self, pos: usize, value: i16) {
+ let offset = self.field_offset(pos);
+ self.data[offset..offset + 2].copy_from_slice(&value.to_le_bytes());
+ }
+
+ pub fn write_int(&mut self, pos: usize, value: i32) {
+ let offset = self.field_offset(pos);
+ self.data[offset..offset + 4].copy_from_slice(&value.to_le_bytes());
+ }
+
+ pub fn write_long(&mut self, pos: usize, value: i64) {
+ let offset = self.field_offset(pos);
+ self.data[offset..offset + 8].copy_from_slice(&value.to_le_bytes());
+ }
+
+ pub fn write_float(&mut self, pos: usize, value: f32) {
+ let offset = self.field_offset(pos);
+ self.data[offset..offset + 4].copy_from_slice(&value.to_le_bytes());
+ }
+
+ pub fn write_double(&mut self, pos: usize, value: f64) {
+ let offset = self.field_offset(pos);
+ self.data[offset..offset + 8].copy_from_slice(&value.to_le_bytes());
+ }
+
+ /// Write a string to the variable-length part and store offset+length in the fixed part.
+ pub fn write_string(&mut self, pos: usize, value: &str) {
+ self.write_binary(pos, value.as_bytes());
+ }
+
+ /// Write a short string (len <= 7) inline into the fixed part.
+ pub fn write_string_inline(&mut self, pos: usize, value: &str) {
+ assert!(
+ value.len() <= 7,
+ "inline string must be <= 7 bytes, got {}",
+ value.len()
+ );
+ self.write_binary_inline(pos, value.as_bytes());
+ }
+
+ /// Write binary data to the variable-length part (8-byte aligned, matching Java BinaryRowWriter).
+ pub fn write_binary(&mut self, pos: usize, value: &[u8]) {
+ let var_offset = self.data.len();
+ self.data.extend_from_slice(value);
+ // Pad to 8-byte word boundary (Java: roundNumberOfBytesToNearestWord)
+ let padding = (8 - (value.len() % 8)) % 8;
+ self.data.extend(std::iter::repeat_n(0u8, padding));
+ let encoded = ((var_offset as u64) << 32) | (value.len() as u64);
+ let offset = self.field_offset(pos);
+ self.data[offset..offset + 8].copy_from_slice(&encoded.to_le_bytes());
+ }
+
+ /// Write short binary data (len <= 7) inline into the fixed part.
+ pub fn write_binary_inline(&mut self, pos: usize, value: &[u8]) {
+ assert!(
+ value.len() <= 7,
+ "inline binary must be <= 7 bytes, got {}",
+ value.len()
+ );
+ let offset = self.field_offset(pos);
+ self.data[offset..offset + 8].fill(0);
+ self.data[offset..offset + value.len()].copy_from_slice(value);
+ self.data[offset + 7] = 0x80 | (value.len() as u8);
+ }
+
+ /// Write a compact Decimal (precision <= 18) as its unscaled i64 value.
+ pub fn write_decimal_compact(&mut self, pos: usize, unscaled: i64) {
+ self.write_long(pos, unscaled);
+ }
+
+ /// Write a non-compact Decimal (precision > 18) as big-endian two's complement bytes (8-byte aligned).
+ pub fn write_decimal_var_len(&mut self, pos: usize, unscaled: i128) {
+ let be_bytes = unscaled.to_be_bytes();
+ let mut start = 0;
+ while start < 15 {
+ let b = be_bytes[start];
+ let next = be_bytes[start + 1];
+ if (b == 0x00 && next & 0x80 == 0) || (b == 0xFF && next & 0x80 != 0) {
+ start += 1;
+ } else {
+ break;
+ }
+ }
+ let minimal = &be_bytes[start..];
+
+ let var_offset = self.data.len();
+ self.data.extend_from_slice(minimal);
+ let padding = (8 - (minimal.len() % 8)) % 8;
+ self.data.extend(std::iter::repeat_n(0u8, padding));
+ let len = minimal.len();
+ let encoded = ((var_offset as u64) << 32) | (len as u64);
+ let offset = self.field_offset(pos);
+ self.data[offset..offset + 8].copy_from_slice(&encoded.to_le_bytes());
+ }
+
+ /// Write a compact Timestamp (precision <= 3) as epoch millis.
+ pub fn write_timestamp_compact(&mut self, pos: usize, epoch_millis: i64) {
+ self.write_long(pos, epoch_millis);
+ }
+
+ /// Write a non-compact Timestamp (precision > 3).
+ pub fn write_timestamp_non_compact(
+ &mut self,
+ pos: usize,
+ epoch_millis: i64,
+ nano_of_milli: i32,
+ ) {
+ let var_offset = self.data.len();
+ self.data.extend_from_slice(&epoch_millis.to_le_bytes());
+ let encoded = ((var_offset as u64) << 32) | (nano_of_milli as u32 as u64);
+ let offset = self.field_offset(pos);
+ self.data[offset..offset + 8].copy_from_slice(&encoded.to_le_bytes());
+ }
+
+ pub fn build(self) -> BinaryRow {
+ BinaryRow::from_bytes(self.arity, self.data)
+ }
+
+ /// Build as Paimon's serialized format: 4-byte BE arity prefix + raw data.
+ pub fn build_serialized(self) -> Vec<u8> {
+ let mut serialized = Vec::with_capacity(4 + self.data.len());
+ serialized.extend_from_slice(&self.arity.to_be_bytes());
+ serialized.extend_from_slice(&self.data);
+ serialized
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn test_empty_binary_row() {
+ let row = BinaryRow::new(0);
+ assert_eq!(row.arity(), 0);
+ assert!(row.is_empty());
+ assert_eq!(row.data(), &[] as &[u8]);
+ }
+
+ #[test]
+ fn test_binary_row_constants() {
+ assert_eq!(BinaryRow::cal_bit_set_width_in_bytes(0), 8);
+ assert_eq!(BinaryRow::cal_bit_set_width_in_bytes(1), 8);
+ assert_eq!(BinaryRow::cal_bit_set_width_in_bytes(56), 8);
+ assert_eq!(BinaryRow::cal_bit_set_width_in_bytes(57), 16);
+ }
+
+ #[test]
+ fn test_from_serialized_bytes() {
+ let mut builder = BinaryRowBuilder::new(1);
+ builder.write_int(0, 42);
+ let serialized = builder.build_serialized();
+
+ let row = BinaryRow::from_serialized_bytes(&serialized).unwrap();
+ assert_eq!(row.arity(), 1);
+ assert!(!row.is_null_at(0));
+ assert_eq!(row.get_int(0).unwrap(), 42);
+ }
+
+ #[test]
+ fn test_from_serialized_bytes_too_short() {
+ assert!(BinaryRow::from_serialized_bytes(&[0, 0]).is_err());
+ }
+
+ #[test]
+ fn test_get_int() {
+ let mut builder = BinaryRowBuilder::new(2);
+ builder.write_int(0, 42);
+ builder.write_int(1, -100);
+ let row = builder.build();
+
+ assert!(!row.is_empty());
+ assert_eq!(row.arity(), 2);
+ assert_eq!(row.get_int(0).unwrap(), 42);
+ assert_eq!(row.get_int(1).unwrap(), -100);
+ }
+
+ #[test]
+ fn test_get_long() {
+ let mut builder = BinaryRowBuilder::new(1);
+ builder.write_long(0, i64::MAX);
+ let row = builder.build();
+ assert_eq!(row.get_long(0).unwrap(), i64::MAX);
+ }
+
+ #[test]
+ fn test_get_short_byte_boolean() {
+ let mut builder = BinaryRowBuilder::new(3);
+ builder.write_short(0, -32768);
+ builder.write_byte(1, -1);
+ builder.write_boolean(2, true);
+ let row = builder.build();
+
+ assert_eq!(row.get_short(0).unwrap(), -32768);
+ assert_eq!(row.get_byte(1).unwrap(), -1);
+ assert!(row.get_boolean(2).unwrap());
+ }
+
+ #[test]
+ fn test_get_float_double() {
+ let mut builder = BinaryRowBuilder::new(2);
+ builder.write_float(0, 1.5_f32);
+ builder.write_double(1, std::f64::consts::PI);
+ let row = builder.build();
+
+ assert!((row.get_float(0).unwrap() - 1.5_f32).abs() < f32::EPSILON);
+ assert!((row.get_double(1).unwrap() - std::f64::consts::PI).abs() < f64::EPSILON);
+ }
+
+ #[test]
+ fn test_null_handling() {
+ let mut builder = BinaryRowBuilder::new(3);
+ builder.write_int(0, 42);
+ builder.set_null_at(1);
+ builder.write_int(2, 99);
+ let row = builder.build();
+
+ assert!(!row.is_null_at(0));
+ assert!(row.is_null_at(1));
+ assert!(!row.is_null_at(2));
+ assert_eq!(row.get_int(0).unwrap(), 42);
+ assert_eq!(row.get_int(2).unwrap(), 99);
+ }
+
+ #[test]
+ fn test_get_string_variable_length() {
+ let mut builder = BinaryRowBuilder::new(2);
+ builder.write_string(0, "hello");
+ builder.write_string(1, "world!");
+ let row = builder.build();
+
+ assert_eq!(row.get_string(0).unwrap(), "hello");
+ assert_eq!(row.get_string(1).unwrap(), "world!");
+ }
+
+ #[test]
+ fn test_get_binary_variable_length() {
+ let mut builder = BinaryRowBuilder::new(1);
+ builder.write_binary(0, b"\x00\x01\x02\x03");
+ let row = builder.build();
+
+ assert_eq!(row.get_binary(0).unwrap(), &[0x00, 0x01, 0x02, 0x03]);
+ }
+
+ #[test]
+ fn test_mixed_types_partition_row() {
+ let mut builder = BinaryRowBuilder::new(2);
+ builder.write_string(0, "2024-01-01");
+ builder.write_int(1, 12);
+ let row = builder.build();
+
+ assert_eq!(row.get_string(0).unwrap(), "2024-01-01");
+ assert_eq!(row.get_int(1).unwrap(), 12);
+ }
+
+ #[test]
+ fn test_serde_roundtrip_empty() {
+ let row = BinaryRow::new(0);
+ let json = serde_json::to_string(&row).unwrap();
+ let deserialized: BinaryRow = serde_json::from_str(&json).unwrap();
+ assert_eq!(deserialized.arity(), 0);
+ assert!(deserialized.is_empty());
+ }
+
+ #[test]
+ fn test_serde_roundtrip_populated() {
+ let mut builder = BinaryRowBuilder::new(2);
+ builder.write_int(0, 42);
+ builder.write_string(1, "hello");
+ let row = builder.build();
+
+ let json = serde_json::to_string(&row).unwrap();
+ let deserialized: BinaryRow = serde_json::from_str(&json).unwrap();
+ assert_eq!(deserialized.arity(), row.arity());
+ assert_eq!(deserialized.data(), row.data());
+ assert_eq!(deserialized.get_int(0).unwrap(), 42);
+ assert_eq!(deserialized.get_string(1).unwrap(), "hello");
+ }
+
+ #[test]
+ fn test_from_bytes_arity_zero() {
+ let data = vec![0u8; 8];
+ let row = BinaryRow::from_bytes(0, data);
+ assert_eq!(row.arity(), 0);
+ assert!(!row.is_empty());
+ }
+
+ #[test]
+ fn test_new_and_from_bytes_null_bits_size_consistent() {
+ for arity in [0, 1, 2, 10, 56, 57, 100] {
+ let stub = BinaryRow::new(arity);
+ let data = vec![0u8; BinaryRow::cal_fix_part_size_in_bytes(arity) as usize];
+ let real = BinaryRow::from_bytes(arity, data);
+ assert_eq!(
+ stub.null_bits_size_in_bytes, real.null_bits_size_in_bytes,
+ "null_bits_size_in_bytes mismatch for arity={arity}"
+ );
+ }
+ }
+
+ #[test]
+ fn test_get_string_inline() {
+ let mut builder = BinaryRowBuilder::new(2);
+ builder.write_string_inline(0, "hi");
+ builder.write_string_inline(1, "7_bytes");
+ let row = builder.build();
+
+ assert_eq!(row.get_string(0).unwrap(), "hi");
+ assert_eq!(row.get_string(1).unwrap(), "7_bytes");
+ }
+
+ #[test]
+ fn test_get_binary_inline() {
+ let mut builder = BinaryRowBuilder::new(1);
+ builder.write_binary_inline(0, &[0xDE, 0xAD]);
+ let row = builder.build();
+
+ assert_eq!(row.get_binary(0).unwrap(), &[0xDE, 0xAD]);
+ }
+
+ #[test]
+ fn test_get_decimal_compact() {
+ let mut builder = BinaryRowBuilder::new(3);
+ builder.write_decimal_compact(0, 12345);
+ builder.write_decimal_compact(1, -100);
+ builder.write_decimal_compact(2, 0);
+ let row = builder.build();
+
+ assert_eq!(row.get_decimal_unscaled(0, 10).unwrap(), 12345);
+ assert_eq!(row.get_decimal_unscaled(1, 10).unwrap(), -100);
+ assert_eq!(row.get_decimal_unscaled(2, 10).unwrap(), 0);
+ }
+
+ #[test]
+ fn test_get_decimal_var_len() {
+ let mut builder = BinaryRowBuilder::new(2);
+ let large_pos: i128 = 10_000_000_000_000_000_000;
+ builder.write_decimal_var_len(0, large_pos);
+ let large_neg: i128 = -10_000_000_000_000_000_000;
+ builder.write_decimal_var_len(1, large_neg);
+ let row = builder.build();
+
+ assert_eq!(row.get_decimal_unscaled(0, 20).unwrap(), large_pos);
+ assert_eq!(row.get_decimal_unscaled(1, 20).unwrap(), large_neg);
+ }
+
+ #[test]
+ fn test_get_timestamp_compact() {
+ let epoch_millis: i64 = 1_704_067_200_000;
+ let mut builder = BinaryRowBuilder::new(1);
+ builder.write_timestamp_compact(0, epoch_millis);
+ let row = builder.build();
+
+ let (millis, nano) = row.get_timestamp_raw(0, 3).unwrap();
+ assert_eq!(millis, epoch_millis);
+ assert_eq!(nano, 0);
+ }
+
+ #[test]
+ fn test_get_timestamp_non_compact() {
+ let epoch_millis: i64 = 1_704_067_200_123;
+ let nano_of_milli: i32 = 456_000;
+ let mut builder = BinaryRowBuilder::new(1);
+ builder.write_timestamp_non_compact(0, epoch_millis, nano_of_milli);
+ let row = builder.build();
+
+ let (millis, nano) = row.get_timestamp_raw(0, 6).unwrap();
+ assert_eq!(millis, epoch_millis);
+ assert_eq!(nano, nano_of_milli);
+ }
+}
diff --git a/crates/paimon/src/spec/core_options.rs b/crates/paimon/src/spec/core_options.rs
index 975f56a..94f35d6 100644
--- a/crates/paimon/src/spec/core_options.rs
+++ b/crates/paimon/src/spec/core_options.rs
@@ -23,6 +23,8 @@ const SOURCE_SPLIT_TARGET_SIZE_OPTION: &str = "source.split.target-size";
const SOURCE_SPLIT_OPEN_FILE_COST_OPTION: &str = "source.split.open-file-cost";
const PARTITION_DEFAULT_NAME_OPTION: &str = "partition.default-name";
const PARTITION_LEGACY_NAME_OPTION: &str = "partition.legacy-name";
+const BUCKET_KEY_OPTION: &str = "bucket-key";
+const BUCKET_FUNCTION_TYPE_OPTION: &str = "bucket-function.type";
pub const SCAN_SNAPSHOT_ID_OPTION: &str = "scan.snapshot-id";
pub const SCAN_TIMESTAMP_MILLIS_OPTION: &str = "scan.timestamp-millis";
pub const SCAN_TAG_NAME_OPTION: &str = "scan.tag-name";
@@ -110,6 +112,24 @@ impl<'a> CoreOptions<'a> {
pub fn scan_tag_name(&self) -> Option<&str> {
self.options.get(SCAN_TAG_NAME_OPTION).map(String::as_str)
}
+
+ /// Explicit bucket key columns. If not set, defaults to primary keys for PK tables.
+ pub fn bucket_key(&self) -> Option<Vec<String>> {
+ self.options
+ .get(BUCKET_KEY_OPTION)
+ .map(|v| v.split(',').map(|s| s.trim().to_string()).collect())
+ }
+
+ /// Whether the bucket function type is the default hash-based function.
+ ///
+ /// Only the default function (`Math.abs(hash % numBuckets)`) is supported
+ /// for bucket predicate pruning. `mod` and `hive` use different algorithms.
+ pub fn is_default_bucket_function(&self) -> bool {
+ self.options
+ .get(BUCKET_FUNCTION_TYPE_OPTION)
+ .map(|v| v.eq_ignore_ascii_case("default"))
+ .unwrap_or(true)
+ }
}
/// Parse a memory size string to bytes using binary (1024-based) semantics.
diff --git a/crates/paimon/src/spec/data_file.rs b/crates/paimon/src/spec/data_file.rs
index e8cea72..7fa1e8b 100644
--- a/crates/paimon/src/spec/data_file.rs
+++ b/crates/paimon/src/spec/data_file.rs
@@ -16,343 +16,12 @@
// under the License.
use crate::spec::stats::BinaryTableStats;
-use chrono::serde::ts_milliseconds::deserialize as from_millis;
-use chrono::serde::ts_milliseconds::serialize as to_millis;
+use chrono::serde::ts_milliseconds_option::deserialize as from_millis_opt;
+use chrono::serde::ts_milliseconds_option::serialize as to_millis_opt;
use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use std::fmt::{Display, Formatter};
-pub const EMPTY_BINARY_ROW: BinaryRow = BinaryRow::new(0);
-
-/// Highest bit mask for detecting inline vs variable-length encoding.
-///
-/// If the highest bit of the 8-byte fixed-part value is 1, the data is stored
-/// inline (≤7 bytes). If 0, the data is in the variable-length part.
-///
-/// Reference: `BinarySection.HIGHEST_FIRST_BIT` in Java Paimon.
-const HIGHEST_FIRST_BIT: u64 = 0x80 << 56;
-
-/// Mask to extract the 7-bit length from an inline-encoded value.
-///
-/// Reference: `BinarySection.HIGHEST_SECOND_TO_EIGHTH_BIT` in Java Paimon.
-const HIGHEST_SECOND_TO_EIGHTH_BIT: u64 = 0x7F << 56;
-
-/// An implementation of InternalRow backed by raw binary bytes.
-///
-/// Binary layout (little-endian):
-/// ```text
-/// | header (8 bytes) | null bit set (8-byte aligned) | fixed-length (8B per field) | variable-length |
-/// ```
-///
-/// - **Header**: byte 0 = RowKind, bytes 1-7 reserved.
-/// - **Null bit set**: starts at bit index 8 (skip header bits), 1 bit per field, padded to 8-byte boundary.
-/// - **Fixed-length part**: 8 bytes per field. Primitives stored directly; variable-length types store offset+length.
-/// - **Variable-length part**: String / Binary data referenced by offset+length from fixed part.
-///
-/// Impl Reference: <https://github.com/apache/paimon/blob/release-0.8.2/paimon-common/src/main/java/org/apache/paimon/data/BinaryRow.java>
-#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub struct BinaryRow {
- arity: i32,
- null_bits_size_in_bytes: i32,
-
- /// Raw binary data backing this row. Empty when constructed via `new()`.
- /// Populated via `from_bytes()` for partition data from manifest entries.
- #[serde(with = "serde_bytes")]
- data: Vec<u8>,
-}
-
-impl BinaryRow {
- pub const HEADER_SIZE_IN_BYTES: i32 = 8;
-
- pub const fn cal_bit_set_width_in_bytes(arity: i32) -> i32 {
- ((arity + 63 + Self::HEADER_SIZE_IN_BYTES) / 64) * 8
- }
-
- pub const fn cal_fix_part_size_in_bytes(arity: i32) -> i32 {
- Self::cal_bit_set_width_in_bytes(arity) + 8 * arity
- }
-
- /// Create a BinaryRow stub without backing data.
- /// Use `from_bytes()` to create a BinaryRow with actual data.
- pub const fn new(arity: i32) -> Self {
- Self {
- arity,
- null_bits_size_in_bytes: Self::cal_bit_set_width_in_bytes(arity),
- data: Vec::new(),
- }
- }
-
- /// Create a BinaryRow from raw binary bytes.
- ///
- /// The `data` must contain the full binary row content:
- /// header + null bit set + fixed-length part + variable-length part.
- /// Does NOT include the 4-byte arity prefix (use `from_serialized_bytes` for that).
- pub fn from_bytes(arity: i32, data: Vec<u8>) -> Self {
- let null_bits_size_in_bytes = Self::cal_bit_set_width_in_bytes(arity);
- Self {
- arity,
- null_bits_size_in_bytes,
- data,
- }
- }
-
- /// Create a BinaryRow from Paimon's serialized format (e.g. `ManifestEntry._PARTITION`).
- ///
- /// Java `SerializationUtils.serializeBinaryRow()` prepends a 4-byte big-endian arity
- /// before the raw BinaryRow content. This method reads that prefix and strips it,
- /// matching Java `SerializationUtils.deserializeBinaryRow()`.
- ///
- /// Variable-length field offsets inside the BinaryRow content are relative to the
- /// BinaryRow base, so they remain valid after stripping the 4-byte prefix.
- pub fn from_serialized_bytes(data: &[u8]) -> crate::Result<Self> {
- if data.len() < 4 {
- return Err(crate::Error::UnexpectedError {
- message: format!(
- "BinaryRow: serialized data too short for arity prefix: {} bytes",
- data.len()
- ),
- source: None,
- });
- }
- let arity = i32::from_be_bytes([data[0], data[1], data[2], data[3]]);
- Ok(Self::from_bytes(arity, data[4..].to_vec()))
- }
-
- /// Number of fields in this row.
- pub fn arity(&self) -> i32 {
- self.arity
- }
-
- /// Returns `true` if this row has no backing data.
- pub fn is_empty(&self) -> bool {
- self.data.is_empty()
- }
-
- /// Returns the raw backing data bytes.
- pub fn data(&self) -> &[u8] {
- &self.data
- }
-
- // ======================== Null check ========================
-
- /// Check if the field at `pos` is null.
- ///
- /// Null bits start at bit index `HEADER_SIZE_IN_BYTES` (= 8) from the base offset.
- /// Bit layout: byte_index = (bit_index) / 8, bit_within_byte = (bit_index) % 8.
- pub fn is_null_at(&self, pos: usize) -> bool {
- let bit_index = pos + Self::HEADER_SIZE_IN_BYTES as usize;
- let byte_index = bit_index / 8;
- let bit_offset = bit_index % 8;
- (self.data[byte_index] & (1 << bit_offset)) != 0
- }
-
- // ======================== Internal read helpers ========================
-
- /// Byte offset of the field value at position `pos` within `self.data`.
- fn field_offset(&self, pos: usize) -> usize {
- self.null_bits_size_in_bytes as usize + pos * 8
- }
-
- /// Bounds-checked slice read. Returns exactly `N` bytes starting at `offset`.
- fn read_slice<const N: usize>(&self, offset: usize) -> crate::Result<[u8; N]> {
- self.data
- .get(offset..offset + N)
- .and_then(|s| s.try_into().ok())
- .ok_or_else(|| crate::Error::UnexpectedError {
- message: format!(
- "BinaryRow: read {N} bytes at offset {offset} exceeds data length {}",
- self.data.len()
- ),
- source: None,
- })
- }
-
- /// Bounds-checked single byte read.
- fn read_byte_at(&self, offset: usize) -> crate::Result<u8> {
- self.data
- .get(offset)
- .copied()
- .ok_or_else(|| crate::Error::UnexpectedError {
- message: format!(
- "BinaryRow: read 1 byte at offset {offset} exceeds data length {}",
- self.data.len()
- ),
- source: None,
- })
- }
-
- /// Read a little-endian i64 from `self.data` at the given byte offset.
- fn read_i64_at(&self, offset: usize) -> crate::Result<i64> {
- self.read_slice::<8>(offset).map(i64::from_le_bytes)
- }
-
- /// Read a little-endian i32 from `self.data` at the given byte offset.
- fn read_i32_at(&self, offset: usize) -> crate::Result<i32> {
- self.read_slice::<4>(offset).map(i32::from_le_bytes)
- }
-
- // ======================== Fixed-length getters ========================
-
- pub fn get_boolean(&self, pos: usize) -> crate::Result<bool> {
- self.read_byte_at(self.field_offset(pos)).map(|b| b != 0)
- }
-
- pub fn get_byte(&self, pos: usize) -> crate::Result<i8> {
- self.read_byte_at(self.field_offset(pos)).map(|b| b as i8)
- }
-
- pub fn get_short(&self, pos: usize) -> crate::Result<i16> {
- self.read_slice::<2>(self.field_offset(pos))
- .map(i16::from_le_bytes)
- }
-
- pub fn get_int(&self, pos: usize) -> crate::Result<i32> {
- self.read_i32_at(self.field_offset(pos))
- }
-
- pub fn get_long(&self, pos: usize) -> crate::Result<i64> {
- self.read_i64_at(self.field_offset(pos))
- }
-
- pub fn get_float(&self, pos: usize) -> crate::Result<f32> {
- self.read_slice::<4>(self.field_offset(pos))
- .map(f32::from_le_bytes)
- }
-
- pub fn get_double(&self, pos: usize) -> crate::Result<f64> {
- self.read_slice::<8>(self.field_offset(pos))
- .map(f64::from_le_bytes)
- }
-
- // ======================== Variable-length getters ========================
-
- /// Resolve the (start, length) byte range for a variable-length field.
- ///
- /// Encoding in the fixed part (8-byte i64):
- /// - Highest bit = 0: data in variable-length part. offset = upper 32 bits, len = lower 32 bits.
- /// Actual position = base_offset(0) + offset.
- /// - Highest bit = 1: data inline in fixed part (≤7 bytes). len = bits[62:56].
- /// Data starts at `field_offset` (little-endian: right after the length/mark byte).
- ///
- /// Returns `Err` if the decoded byte range falls outside the backing data.
- fn resolve_var_length_field(&self, pos: usize) -> crate::Result<(usize, usize)> {
- let field_off = self.field_offset(pos);
- let raw = self.read_i64_at(field_off)? as u64;
-
- let (start, len) = if raw & HIGHEST_FIRST_BIT == 0 {
- // Variable-length part: offset in upper 32 bits, length in lower 32 bits.
- let offset = (raw >> 32) as usize;
- let len = (raw & 0xFFFF_FFFF) as usize;
- (offset, len)
- } else {
- // Inline: length in bits [62:56], data starts at field_offset (LE).
- let len = ((raw & HIGHEST_SECOND_TO_EIGHTH_BIT) >> 56) as usize;
- (field_off, len)
- };
-
- let end = start
- .checked_add(len)
- .ok_or_else(|| crate::Error::UnexpectedError {
- message: format!(
- "BinaryRow: var-len field at pos {pos}: offset {start} + len {len} overflows"
- ),
- source: None,
- })?;
- if end > self.data.len() {
- return Err(crate::Error::UnexpectedError {
- message: format!(
- "BinaryRow: var-len field at pos {pos}: range [{start}..{end}) exceeds data length {}",
- self.data.len()
- ),
- source: None,
- });
- }
- Ok((start, len))
- }
-
- pub fn get_binary(&self, pos: usize) -> crate::Result<&[u8]> {
- let (start, len) = self.resolve_var_length_field(pos)?;
- Ok(&self.data[start..start + len])
- }
-
- pub fn get_string(&self, pos: usize) -> crate::Result<&str> {
- let bytes = self.get_binary(pos)?;
- std::str::from_utf8(bytes).map_err(|e| crate::Error::UnexpectedError {
- message: format!("BinaryRow: invalid UTF-8 in string field at pos {pos}: {e}"),
- source: Some(Box::new(e)),
- })
- }
-
- /// Read the unscaled value of a Decimal field as `i128`.
- ///
- /// - `precision <= 18` (compact): stored as `i64` in the fixed part.
- /// - `precision > 18`: stored as big-endian two's complement bytes in the variable-length part.
- ///
- /// Reference: `BinaryRow.getDecimal` and `DecimalData.isCompact` in Java Paimon.
- pub(crate) fn get_decimal_unscaled(&self, pos: usize, precision: u32) -> crate::Result<i128> {
- if precision <= 18 {
- Ok(self.get_long(pos)? as i128)
- } else {
- let bytes = self.get_binary(pos)?;
- if bytes.is_empty() {
- return Err(crate::Error::UnexpectedError {
- message: format!("BinaryRow: empty bytes for non-compact Decimal at pos {pos}"),
- source: None,
- });
- }
- // Big-endian two's complement, same as Java BigInteger.toByteArray().
- let negative = bytes[0] & 0x80 != 0;
- let mut val: i128 = if negative { -1 } else { 0 };
- for &b in bytes {
- val = (val << 8) | (b as i128);
- }
- Ok(val)
- }
- }
-
- /// Read the raw components of a Timestamp / LocalZonedTimestamp field.
- ///
- /// Returns `(epoch_millis, nano_of_milli)`.
- ///
- /// - `precision <= 3` (compact): stored as epoch millis `i64` in the fixed part,
- /// `nano_of_milli` is 0.
- /// - `precision > 3` (non-compact): the fixed 8-byte slot stores
- /// `(offset << 32) | nanoOfMillisecond`; the variable area at `offset` contains
- /// an 8-byte `i64` millisecond value.
- ///
- /// Reference: `AbstractBinaryWriter.writeTimestamp` and
- /// `MemorySegmentUtils.readTimestampData` in Java Paimon.
- pub(crate) fn get_timestamp_raw(
- &self,
- pos: usize,
- precision: u32,
- ) -> crate::Result<(i64, i32)> {
- if precision <= 3 {
- Ok((self.get_long(pos)?, 0))
- } else {
- // Read the raw 8-byte fixed slot: high 32 bits = offset, low 32 bits = nanoOfMillisecond.
- let field_off = self.field_offset(pos);
- let offset_and_nano = self.read_i64_at(field_off)? as u64;
- let offset = (offset_and_nano >> 32) as usize;
- let nano_of_milli = offset_and_nano as i32;
-
- // Read the 8-byte millisecond value from the variable area.
- if offset + 8 > self.data.len() {
- return Err(crate::Error::UnexpectedError {
- message: format!(
- "BinaryRow: non-compact Timestamp at pos {pos}: offset {offset} + 8 exceeds data length {}",
- self.data.len()
- ),
- source: None,
- });
- }
- let millis = i64::from_le_bytes(self.read_slice::<8>(offset)?);
- Ok((millis, nano_of_milli))
- }
- }
-}
-
/// Metadata of a data file.
///
/// Impl References: <https://github.com/apache/paimon/blob/release-0.8.2/paimon-core/src/main/java/org/apache/paimon/io/DataFileMeta.java>
@@ -386,10 +55,11 @@ pub struct DataFileMeta {
pub extra_files: Vec<String>,
#[serde(
rename = "_CREATION_TIME",
- serialize_with = "to_millis",
- deserialize_with = "from_millis"
+ serialize_with = "to_millis_opt",
+ deserialize_with = "from_millis_opt",
+ default
)]
- pub creation_time: DateTime<Utc>,
+ pub creation_time: Option<DateTime<Utc>>,
#[serde(rename = "_DELETE_ROW_COUNT")]
// rowCount = add_row_count + delete_row_count.
pub delete_row_count: Option<i64>,
@@ -397,21 +67,21 @@ pub struct DataFileMeta {
#[serde(rename = "_EMBEDDED_FILE_INDEX", with = "serde_bytes")]
pub embedded_index: Option<Vec<u8>>,
- /// The starting row ID for this file's data (used in data evolution mode).
+ /// File source identifier (e.g. APPEND, COMPACT).
#[serde(
- rename = "_FIRST_ROW_ID",
+ rename = "_FILE_SOURCE",
default,
skip_serializing_if = "Option::is_none"
)]
- pub first_row_id: Option<i64>,
+ pub file_source: Option<i32>,
- /// Which table columns this file contains (used in data evolution mode).
+ /// Column names covered by `_VALUE_STATS` when stats are written in dense mode.
#[serde(
- rename = "_WRITE_COLS",
+ rename = "_VALUE_STATS_COLS",
default,
skip_serializing_if = "Option::is_none"
)]
- pub write_cols: Option<Vec<String>>,
+ pub value_stats_cols: Option<Vec<String>>,
/// External path for the data file (e.g. when data is stored outside the table directory).
#[serde(
@@ -420,6 +90,22 @@ pub struct DataFileMeta {
skip_serializing_if = "Option::is_none"
)]
pub external_path: Option<String>,
+
+ /// The starting row ID for this file's data (used in data evolution mode).
+ #[serde(
+ rename = "_FIRST_ROW_ID",
+ default,
+ skip_serializing_if = "Option::is_none"
+ )]
+ pub first_row_id: Option<i64>,
+
+ /// Which table columns this file contains (used in data evolution mode).
+ #[serde(
+ rename = "_WRITE_COLS",
+ default,
+ skip_serializing_if = "Option::is_none"
+ )]
+ pub write_cols: Option<Vec<String>>,
}
impl Display for DataFileMeta {
@@ -434,461 +120,3 @@ impl DataFileMeta {
self.first_row_id.map(|fid| (fid, fid + self.row_count - 1))
}
}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- /// Helper to build a BinaryRow byte buffer matching Java's BinaryRow memory layout.
- ///
- /// Layout: header (8 bytes) | null bit set (aligned) | fixed-length (8B per field) | var-length
- ///
- /// This mimics what Java `BinaryRowWriter` produces.
- struct BinaryRowBuilder {
- arity: i32,
- null_bits_size: usize,
- data: Vec<u8>,
- }
-
- impl BinaryRowBuilder {
- fn new(arity: i32) -> Self {
- let null_bits_size = BinaryRow::cal_bit_set_width_in_bytes(arity) as usize;
- let fixed_part_size = null_bits_size + (arity as usize) * 8;
- let data = vec![0u8; fixed_part_size];
- Self {
- arity,
- null_bits_size,
- data,
- }
- }
-
- fn field_offset(&self, pos: usize) -> usize {
- self.null_bits_size + pos * 8
- }
-
- fn set_null_at(&mut self, pos: usize) {
- let bit_index = pos + BinaryRow::HEADER_SIZE_IN_BYTES as usize;
- let byte_index = bit_index / 8;
- let bit_offset = bit_index % 8;
- self.data[byte_index] |= 1 << bit_offset;
- // Zero out the field value (Java does this too).
- let offset = self.field_offset(pos);
- self.data[offset..offset + 8].fill(0);
- }
-
- fn write_int(&mut self, pos: usize, value: i32) {
- let offset = self.field_offset(pos);
- let bytes = value.to_le_bytes();
- self.data[offset..offset + 4].copy_from_slice(&bytes);
- }
-
- fn write_long(&mut self, pos: usize, value: i64) {
- let offset = self.field_offset(pos);
- let bytes = value.to_le_bytes();
- self.data[offset..offset + 8].copy_from_slice(&bytes);
- }
-
- fn write_short(&mut self, pos: usize, value: i16) {
- let offset = self.field_offset(pos);
- let bytes = value.to_le_bytes();
- self.data[offset..offset + 2].copy_from_slice(&bytes);
- }
-
- fn write_byte(&mut self, pos: usize, value: i8) {
- let offset = self.field_offset(pos);
- self.data[offset] = value as u8;
- }
-
- fn write_boolean(&mut self, pos: usize, value: bool) {
- let offset = self.field_offset(pos);
- self.data[offset] = if value { 1 } else { 0 };
- }
-
- fn write_float(&mut self, pos: usize, value: f32) {
- let offset = self.field_offset(pos);
- let bytes = value.to_le_bytes();
- self.data[offset..offset + 4].copy_from_slice(&bytes);
- }
-
- fn write_double(&mut self, pos: usize, value: f64) {
- let offset = self.field_offset(pos);
- let bytes = value.to_le_bytes();
- self.data[offset..offset + 8].copy_from_slice(&bytes);
- }
-
- /// Write a string to the variable-length part and store offset+length in the fixed part.
- fn write_string(&mut self, pos: usize, value: &str) {
- let var_offset = self.data.len();
- self.data.extend_from_slice(value.as_bytes());
- let len = value.len();
-
- // Encode as: upper 32 bits = offset, lower 32 bits = length.
- let encoded = ((var_offset as u64) << 32) | (len as u64);
- let offset = self.field_offset(pos);
- let bytes = encoded.to_le_bytes();
- self.data[offset..offset + 8].copy_from_slice(&bytes);
- }
-
- /// Write a short string (len <= 7) inline into the fixed part.
- ///
- /// Encoding (little-endian):
- /// - Byte 7 (highest byte in LE i64): mark bit (0x80) | length (7 bits)
- /// - Bytes 0..len: the actual string data
- ///
- /// This matches Java `AbstractBinaryWriter.writeBinaryToFixLenPart`.
- fn write_string_inline(&mut self, pos: usize, value: &str) {
- assert!(
- value.len() <= 7,
- "inline string must be <= 7 bytes, got {}",
- value.len()
- );
- self.write_binary_inline(pos, value.as_bytes());
- }
-
- /// Write short binary data (len <= 7) inline into the fixed part.
- fn write_binary_inline(&mut self, pos: usize, value: &[u8]) {
- assert!(
- value.len() <= 7,
- "inline binary must be <= 7 bytes, got {}",
- value.len()
- );
- let offset = self.field_offset(pos);
- // Zero out the 8-byte slot first.
- self.data[offset..offset + 8].fill(0);
- // Write data into lower bytes (LE layout).
- self.data[offset..offset + value.len()].copy_from_slice(value);
- // Write mark + length into the highest byte (byte 7 in LE = offset+7).
- self.data[offset + 7] = 0x80 | (value.len() as u8);
- }
-
- /// Write a compact Decimal (precision <= 18) as its unscaled i64 value.
- fn write_decimal_compact(&mut self, pos: usize, unscaled: i64) {
- self.write_long(pos, unscaled);
- }
-
- /// Write a non-compact Decimal (precision > 18) as big-endian two's complement bytes.
- fn write_decimal_var_len(&mut self, pos: usize, unscaled: i128) {
- // Convert i128 to minimal big-endian two's complement.
- let be_bytes = unscaled.to_be_bytes();
- // Find the first significant byte (skip redundant sign-extension bytes).
- let mut start = 0;
- while start < 15 {
- let b = be_bytes[start];
- let next = be_bytes[start + 1];
- // Safe to skip if byte is pure sign extension.
- if (b == 0x00 && next & 0x80 == 0) || (b == 0xFF && next & 0x80 != 0) {
- start += 1;
- } else {
- break;
- }
- }
- let minimal = &be_bytes[start..];
-
- let var_offset = self.data.len();
- self.data.extend_from_slice(minimal);
- let len = minimal.len();
- let encoded = ((var_offset as u64) << 32) | (len as u64);
- let offset = self.field_offset(pos);
- self.data[offset..offset + 8].copy_from_slice(&encoded.to_le_bytes());
- }
-
- /// Write a compact Timestamp (precision <= 3) as epoch millis.
- fn write_timestamp_compact(&mut self, pos: usize, epoch_millis: i64) {
- self.write_long(pos, epoch_millis);
- }
-
- /// Write a non-compact Timestamp (precision > 3).
- ///
- /// Matches Java `AbstractBinaryWriter.writeTimestamp`:
- /// - Fixed slot: `(offset << 32) | nanoOfMillisecond`
- /// - Variable area: 8-byte `millisecond` (LE)
- fn write_timestamp_non_compact(
- &mut self,
- pos: usize,
- epoch_millis: i64,
- nano_of_milli: i32,
- ) {
- let var_offset = self.data.len();
- // Variable area: only the 8-byte millis value.
- self.data.extend_from_slice(&epoch_millis.to_le_bytes());
- // Fixed slot: (offset << 32) | nano_of_milli
- let encoded = ((var_offset as u64) << 32) | (nano_of_milli as u32 as u64);
- let offset = self.field_offset(pos);
- self.data[offset..offset + 8].copy_from_slice(&encoded.to_le_bytes());
- }
-
- fn build(self) -> BinaryRow {
- BinaryRow::from_bytes(self.arity, self.data)
- }
- }
-
- #[test]
- fn test_empty_binary_row() {
- let row = BinaryRow::new(0);
- assert_eq!(row.arity(), 0);
- assert!(row.is_empty());
- assert_eq!(row.data(), &[] as &[u8]);
- }
-
- #[test]
- fn test_binary_row_constants() {
- // arity=0: null_bits_size = ((0 + 63 + 8) / 64) * 8 = (71/64)*8 = 1*8 = 8
- assert_eq!(BinaryRow::cal_bit_set_width_in_bytes(0), 8);
- // arity=1: ((1 + 63 + 8) / 64) * 8 = (72/64)*8 = 1*8 = 8
- assert_eq!(BinaryRow::cal_bit_set_width_in_bytes(1), 8);
- // arity=56: ((56 + 63 + 8) / 64) * 8 = (127/64)*8 = 1*8 = 8
- assert_eq!(BinaryRow::cal_bit_set_width_in_bytes(56), 8);
- // arity=57: ((57 + 63 + 8) / 64) * 8 = (128/64)*8 = 2*8 = 16
- assert_eq!(BinaryRow::cal_bit_set_width_in_bytes(57), 16);
- }
-
- #[test]
- fn test_from_serialized_bytes() {
- // Build a raw BinaryRow with arity=1, int value 42, then prepend 4-byte BE arity prefix
- // to simulate Java SerializationUtils.serializeBinaryRow() format.
- let mut builder = BinaryRowBuilder::new(1);
- builder.write_int(0, 42);
- let raw_row = builder.build();
- let raw_data = raw_row.data();
-
- let mut serialized = Vec::with_capacity(4 + raw_data.len());
- serialized.extend_from_slice(&1_i32.to_be_bytes());
- serialized.extend_from_slice(raw_data);
-
- let row = BinaryRow::from_serialized_bytes(&serialized).unwrap();
- assert_eq!(row.arity(), 1);
- assert!(!row.is_null_at(0));
- assert_eq!(row.get_int(0).unwrap(), 42);
- }
-
- #[test]
- fn test_from_serialized_bytes_too_short() {
- assert!(BinaryRow::from_serialized_bytes(&[0, 0]).is_err());
- }
-
- #[test]
- fn test_get_int() {
- let mut builder = BinaryRowBuilder::new(2);
- builder.write_int(0, 42);
- builder.write_int(1, -100);
- let row = builder.build();
-
- assert!(!row.is_empty());
- assert_eq!(row.arity(), 2);
- assert!(!row.is_null_at(0));
- assert!(!row.is_null_at(1));
- assert_eq!(row.get_int(0).unwrap(), 42);
- assert_eq!(row.get_int(1).unwrap(), -100);
- }
-
- #[test]
- fn test_get_long() {
- let mut builder = BinaryRowBuilder::new(1);
- builder.write_long(0, i64::MAX);
- let row = builder.build();
-
- assert_eq!(row.get_long(0).unwrap(), i64::MAX);
- }
-
- #[test]
- fn test_get_short_byte_boolean() {
- let mut builder = BinaryRowBuilder::new(3);
- builder.write_short(0, -32768);
- builder.write_byte(1, -1);
- builder.write_boolean(2, true);
- let row = builder.build();
-
- assert_eq!(row.get_short(0).unwrap(), -32768);
- assert_eq!(row.get_byte(1).unwrap(), -1);
- assert!(row.get_boolean(2).unwrap());
- }
-
- #[test]
- fn test_get_float_double() {
- let mut builder = BinaryRowBuilder::new(2);
- builder.write_float(0, 1.5_f32);
- builder.write_double(1, std::f64::consts::PI);
- let row = builder.build();
-
- assert!((row.get_float(0).unwrap() - 1.5_f32).abs() < f32::EPSILON);
- assert!((row.get_double(1).unwrap() - std::f64::consts::PI).abs() < f64::EPSILON);
- }
-
- #[test]
- fn test_null_handling() {
- let mut builder = BinaryRowBuilder::new(3);
- builder.write_int(0, 42);
- builder.set_null_at(1);
- builder.write_int(2, 99);
- let row = builder.build();
-
- assert!(!row.is_null_at(0));
- assert!(row.is_null_at(1));
- assert!(!row.is_null_at(2));
- assert_eq!(row.get_int(0).unwrap(), 42);
- assert_eq!(row.get_int(2).unwrap(), 99);
- }
-
- #[test]
- fn test_get_string_variable_length() {
- let mut builder = BinaryRowBuilder::new(2);
- builder.write_string(0, "hello");
- builder.write_string(1, "world!");
- let row = builder.build();
-
- assert_eq!(row.get_string(0).unwrap(), "hello");
- assert_eq!(row.get_string(1).unwrap(), "world!");
- }
-
- #[test]
- fn test_get_binary_variable_length() {
- let mut builder = BinaryRowBuilder::new(1);
- builder.write_string(0, "\x00\x01\x02\x03");
- let row = builder.build();
-
- assert_eq!(row.get_binary(0).unwrap(), &[0x00, 0x01, 0x02, 0x03]);
- }
-
- #[test]
- fn test_mixed_types_partition_row() {
- // Simulate a typical partition row: (dt: String, hr: Int)
- let mut builder = BinaryRowBuilder::new(2);
- builder.write_string(0, "2024-01-01");
- builder.write_int(1, 12);
- let row = builder.build();
-
- assert_eq!(row.get_string(0).unwrap(), "2024-01-01");
- assert_eq!(row.get_int(1).unwrap(), 12);
- }
-
- #[test]
- fn test_serde_roundtrip_empty() {
- // Verify empty BinaryRow serde roundtrip is stable.
- let row = BinaryRow::new(0);
- let json = serde_json::to_string(&row).unwrap();
- let deserialized: BinaryRow = serde_json::from_str(&json).unwrap();
- assert_eq!(deserialized.arity(), 0);
- assert!(deserialized.is_empty());
- }
-
- #[test]
- fn test_serde_roundtrip_populated() {
- // Verify a populated BinaryRow roundtrips correctly with data intact.
- let mut builder = BinaryRowBuilder::new(2);
- builder.write_int(0, 42);
- builder.write_string(1, "hello");
- let row = builder.build();
-
- let json = serde_json::to_string(&row).unwrap();
- let deserialized: BinaryRow = serde_json::from_str(&json).unwrap();
- assert_eq!(deserialized.arity(), row.arity());
- assert_eq!(deserialized.data(), row.data());
- assert_eq!(deserialized.get_int(0).unwrap(), 42);
- assert_eq!(deserialized.get_string(1).unwrap(), "hello");
- }
-
- #[test]
- fn test_from_bytes_arity_zero() {
- // Empty row with header only (8 bytes of zeros + 0 fields).
- let data = vec![0u8; 8]; // header only, null_bits_size_in_bytes = 8
- let row = BinaryRow::from_bytes(0, data);
- assert_eq!(row.arity(), 0);
- assert!(!row.is_empty());
- }
-
- #[test]
- fn test_new_and_from_bytes_null_bits_size_consistent() {
- // Verify that new() and from_bytes() produce the same null_bits_size_in_bytes.
- for arity in [0, 1, 2, 10, 56, 57, 100] {
- let stub = BinaryRow::new(arity);
- let data = vec![0u8; BinaryRow::cal_fix_part_size_in_bytes(arity) as usize];
- let real = BinaryRow::from_bytes(arity, data);
- assert_eq!(
- stub.null_bits_size_in_bytes, real.null_bits_size_in_bytes,
- "null_bits_size_in_bytes mismatch for arity={arity}"
- );
- }
- }
-
- #[test]
- fn test_get_string_inline() {
- // Test the inline encoding path (len <= 7).
- // Java BinaryRowWriter inlines short strings: highest bit = 1, len in bits[62:56],
- // data occupies the lower bytes of the 8-byte fixed-part slot.
- let mut builder = BinaryRowBuilder::new(2);
- builder.write_string_inline(0, "hi");
- builder.write_string_inline(1, "7_bytes"); // exactly 7 bytes
- let row = builder.build();
-
- assert_eq!(row.get_string(0).unwrap(), "hi");
- assert_eq!(row.get_string(1).unwrap(), "7_bytes");
- }
-
- #[test]
- fn test_get_binary_inline() {
- // Test inline binary (len <= 7).
- let mut builder = BinaryRowBuilder::new(1);
- builder.write_binary_inline(0, &[0xDE, 0xAD]);
- let row = builder.build();
-
- assert_eq!(row.get_binary(0).unwrap(), &[0xDE, 0xAD]);
- }
-
- #[test]
- fn test_get_decimal_compact() {
- // precision <= 18: stored as i64 unscaled value.
- let mut builder = BinaryRowBuilder::new(3);
- builder.write_decimal_compact(0, 12345); // 12.345 with scale=3
- builder.write_decimal_compact(1, -100); // -0.100 with scale=3
- builder.write_decimal_compact(2, 0); // 0.000 with scale=3
- let row = builder.build();
-
- assert_eq!(row.get_decimal_unscaled(0, 10).unwrap(), 12345);
- assert_eq!(row.get_decimal_unscaled(1, 10).unwrap(), -100);
- assert_eq!(row.get_decimal_unscaled(2, 10).unwrap(), 0);
- }
-
- #[test]
- fn test_get_decimal_var_len() {
- // precision > 18: stored as big-endian two's complement in var-len part.
- let mut builder = BinaryRowBuilder::new(2);
- // Large positive: 10^19 = 10_000_000_000_000_000_000
- let large_pos: i128 = 10_000_000_000_000_000_000;
- builder.write_decimal_var_len(0, large_pos);
- // Large negative
- let large_neg: i128 = -10_000_000_000_000_000_000;
- builder.write_decimal_var_len(1, large_neg);
- let row = builder.build();
-
- assert_eq!(row.get_decimal_unscaled(0, 20).unwrap(), large_pos);
- assert_eq!(row.get_decimal_unscaled(1, 20).unwrap(), large_neg);
- }
-
- #[test]
- fn test_get_timestamp_compact() {
- // precision <= 3: stored as epoch millis i64.
- let epoch_millis: i64 = 1_704_067_200_000; // 2024-01-01 00:00:00 UTC
- let mut builder = BinaryRowBuilder::new(1);
- builder.write_timestamp_compact(0, epoch_millis);
- let row = builder.build();
-
- let (millis, nano) = row.get_timestamp_raw(0, 3).unwrap();
- assert_eq!(millis, epoch_millis);
- assert_eq!(nano, 0);
- }
-
- #[test]
- fn test_get_timestamp_non_compact() {
- // precision > 3: fixed slot = (offset << 32 | nano_of_milli),
- // variable area = 8 bytes millis.
- let epoch_millis: i64 = 1_704_067_200_123;
- let nano_of_milli: i32 = 456_000;
- let mut builder = BinaryRowBuilder::new(1);
- builder.write_timestamp_non_compact(0, epoch_millis, nano_of_milli);
- let row = builder.build();
-
- let (millis, nano) = row.get_timestamp_raw(0, 6).unwrap();
- assert_eq!(millis, epoch_millis);
- assert_eq!(nano, nano_of_milli);
- }
-}
diff --git a/crates/paimon/src/spec/mod.rs b/crates/paimon/src/spec/mod.rs
index 8891c43..2cddd82 100644
--- a/crates/paimon/src/spec/mod.rs
+++ b/crates/paimon/src/spec/mod.rs
@@ -19,6 +19,9 @@
//!
//! All paimon specs types are defined here.
+mod binary_row;
+pub use binary_row::*;
+
mod data_file;
pub use data_file::*;
@@ -62,3 +65,4 @@ pub(crate) use predicate::extract_datum;
pub use predicate::{
field_idx_to_partition_idx, Datum, Predicate, PredicateBuilder, PredicateOperator,
};
+pub(crate) mod murmur_hash;
diff --git a/crates/paimon/src/spec/murmur_hash.rs b/crates/paimon/src/spec/murmur_hash.rs
new file mode 100644
index 0000000..bf7b1a2
--- /dev/null
+++ b/crates/paimon/src/spec/murmur_hash.rs
@@ -0,0 +1,97 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Murmur3 32-bit hash compatible with Java Paimon's `MurmurHashUtils`.
+//!
+//! Reference: <https://github.com/apache/paimon/blob/master/paimon-common/src/main/java/org/apache/paimon/utils/MurmurHashUtils.java>
+
+const C1: u32 = 0xcc9e2d51;
+const C2: u32 = 0x1b873593;
+const DEFAULT_SEED: u32 = 42;
+
+fn mix_k1(mut k1: u32) -> u32 {
+ k1 = k1.wrapping_mul(C1);
+ k1 = k1.rotate_left(15);
+ k1 = k1.wrapping_mul(C2);
+ k1
+}
+
+fn mix_h1(mut h1: u32, k1: u32) -> u32 {
+ h1 ^= k1;
+ h1 = h1.rotate_left(13);
+ h1 = h1.wrapping_mul(5).wrapping_add(0xe6546b64);
+ h1
+}
+
+fn fmix(mut h: u32) -> u32 {
+ h ^= h >> 16;
+ h = h.wrapping_mul(0x85ebca6b);
+ h ^= h >> 13;
+ h = h.wrapping_mul(0xc2b2ae35);
+ h ^= h >> 16;
+ h
+}
+
+/// Murmur3 32-bit hash over word-aligned data (length must be a multiple of 4).
+///
+/// This matches Java Paimon's `MurmurHashUtils.hashBytesByWords` with `DEFAULT_SEED = 42`.
+/// Java's `BinaryRow.hashCode()` calls `hashByWords(segments, offset, sizeInBytes)`.
+///
+/// Note: Java reads ints in native (little-endian on x86) byte order via `Unsafe.getInt`.
+/// We use `i32::from_le_bytes` to match.
+pub fn hash_by_words(data: &[u8]) -> i32 {
+ assert!(
+ data.len().is_multiple_of(4),
+ "hash_by_words: data length must be word-aligned (multiple of 4), got {}",
+ data.len()
+ );
+ let mut h1 = DEFAULT_SEED;
+ for chunk in data.chunks_exact(4) {
+ let word = u32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]]);
+ let k1 = mix_k1(word);
+ h1 = mix_h1(h1, k1);
+ }
+ fmix(h1 ^ data.len() as u32) as i32
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn test_hash_empty() {
+ let h = hash_by_words(&[]);
+ let expected = fmix(DEFAULT_SEED) as i32;
+ assert_eq!(h, expected);
+ }
+
+ #[test]
+ fn test_hash_single_int() {
+ let data = 1_i32.to_le_bytes();
+ let h = hash_by_words(&data);
+ let k1 = mix_k1(1);
+ let h1 = mix_h1(DEFAULT_SEED, k1);
+ let expected = fmix(h1 ^ 4) as i32;
+ assert_eq!(h, expected);
+ }
+
+ #[test]
+ #[should_panic(expected = "word-aligned")]
+ fn test_hash_non_aligned_panics() {
+ hash_by_words(&[1, 2, 3]);
+ }
+}
diff --git a/crates/paimon/src/spec/objects_file.rs b/crates/paimon/src/spec/objects_file.rs
index af10d26..171de1d 100644
--- a/crates/paimon/src/spec/objects_file.rs
+++ b/crates/paimon/src/spec/objects_file.rs
@@ -56,7 +56,11 @@ mod tests {
10,
10,
10,
- BinaryTableStats::new(value_bytes.clone(), value_bytes.clone(), vec![1, 2]),
+ BinaryTableStats::new(
+ value_bytes.clone(),
+ value_bytes.clone(),
+ vec![Some(1), Some(2)]
+ ),
1
),
ManifestFileMeta::new(
@@ -64,7 +68,11 @@ mod tests {
11,
0,
10,
- BinaryTableStats::new(value_bytes.clone(), value_bytes.clone(), vec![1, 2]),
+ BinaryTableStats::new(
+ value_bytes.clone(),
+ value_bytes.clone(),
+ vec![Some(1), Some(2)]
+ ),
2
)
],
@@ -101,26 +109,30 @@ mod tests {
key_stats: BinaryTableStats::new(
value_bytes.clone(),
value_bytes.clone(),
- vec![1, 2]
+ vec![Some(1), Some(2)]
),
value_stats: BinaryTableStats::new(
value_bytes.clone(),
value_bytes.clone(),
- vec![1, 2]
+ vec![Some(1), Some(2)]
),
min_sequence_number: 1,
max_sequence_number: 100,
schema_id: 0,
level: 1,
extra_files: vec![],
- creation_time: "2024-09-06T07:45:55.039+00:00"
- .parse::<DateTime<Utc>>()
- .unwrap(),
+ creation_time: Some(
+ "2024-09-06T07:45:55.039+00:00"
+ .parse::<DateTime<Utc>>()
+ .unwrap()
+ ),
delete_row_count: Some(0),
embedded_index: None,
first_row_id: None,
write_cols: None,
external_path: None,
+ file_source: None,
+ value_stats_cols: None,
},
2
),
@@ -138,26 +150,30 @@ mod tests {
key_stats: BinaryTableStats::new(
value_bytes.clone(),
value_bytes.clone(),
- vec![1, 2]
+ vec![Some(1), Some(2)]
),
value_stats: BinaryTableStats::new(
value_bytes.clone(),
value_bytes.clone(),
- vec![1, 2]
+ vec![Some(1), Some(2)]
),
min_sequence_number: 1,
max_sequence_number: 100,
schema_id: 0,
level: 1,
extra_files: vec![],
- creation_time: "2024-09-06T07:45:55.039+00:00"
- .parse::<DateTime<Utc>>()
- .unwrap(),
+ creation_time: Some(
+ "2024-09-06T07:45:55.039+00:00"
+ .parse::<DateTime<Utc>>()
+ .unwrap()
+ ),
delete_row_count: Some(1),
embedded_index: None,
first_row_id: None,
write_cols: None,
external_path: None,
+ file_source: None,
+ value_stats_cols: None,
},
2
),
diff --git a/crates/paimon/src/spec/predicate.rs b/crates/paimon/src/spec/predicate.rs
index 953b92c..8135c7d 100644
--- a/crates/paimon/src/spec/predicate.rs
+++ b/crates/paimon/src/spec/predicate.rs
@@ -24,7 +24,7 @@
//! - Java `PredicateBuilder` / `LeafPredicate` / `CompoundPredicate`
use crate::error::*;
-use crate::spec::data_file::BinaryRow;
+use crate::spec::binary_row::BinaryRow;
use crate::spec::types::DataType;
use crate::spec::DataField;
use std::cmp::Ordering;
diff --git a/crates/paimon/src/spec/stats.rs b/crates/paimon/src/spec/stats.rs
index 98923ce..adc3202 100644
--- a/crates/paimon/src/spec/stats.rs
+++ b/crates/paimon/src/spec/stats.rs
@@ -15,9 +15,28 @@
// specific language governing permissions and limitations
// under the License.
-use serde::{Deserialize, Serialize};
+use serde::{Deserialize, Deserializer, Serialize, Serializer};
use std::fmt::{Display, Formatter};
+/// Deserialize `_NULL_COUNTS` which in Avro is `["null", {"type":"array","items":["null","long"]}]`.
+/// Preserves null items as `None` (meaning "unknown") rather than collapsing to 0.
+fn deserialize_null_counts<'de, D>(deserializer: D) -> Result<Vec<Option<i64>>, D::Error>
+where
+ D: Deserializer<'de>,
+{
+ let opt: Option<Vec<Option<i64>>> = Option::deserialize(deserializer)?;
+ Ok(opt.unwrap_or_default())
+}
+
+fn serialize_null_counts<S>(value: &[Option<i64>], serializer: S) -> Result<S::Ok, S::Error>
+where
+ S: Serializer,
+{
+ // Serialize as Option<Vec<Option<i64>>> to match the Avro union schema.
+ let wrapped: Option<&[Option<i64>]> = Some(value);
+ wrapped.serialize(serializer)
+}
+
/// The statistics for columns, supports the following stats.
///
/// All statistics are stored in the form of a Binary, which can significantly reduce its memory consumption, but the cost is that the column type needs to be known when getting.
@@ -34,8 +53,12 @@ pub struct BinaryTableStats {
max_values: Vec<u8>,
/// the number of nulls of the columns
- #[serde(rename = "_NULL_COUNTS")]
- null_counts: Vec<i64>,
+ #[serde(
+ rename = "_NULL_COUNTS",
+ deserialize_with = "deserialize_null_counts",
+ serialize_with = "serialize_null_counts"
+ )]
+ null_counts: Vec<Option<i64>>,
}
impl BinaryTableStats {
@@ -53,14 +76,14 @@ impl BinaryTableStats {
/// Get the number of nulls of the columns
#[inline]
- pub fn null_counts(&self) -> &Vec<i64> {
+ pub fn null_counts(&self) -> &Vec<Option<i64>> {
&self.null_counts
}
pub fn new(
min_values: Vec<u8>,
max_values: Vec<u8>,
- null_counts: Vec<i64>,
+ null_counts: Vec<Option<i64>>,
) -> BinaryTableStats {
Self {
min_values,
diff --git a/crates/paimon/src/table/bin_pack.rs b/crates/paimon/src/table/bin_pack.rs
index 841154e..b4dc178 100644
--- a/crates/paimon/src/table/bin_pack.rs
+++ b/crates/paimon/src/table/bin_pack.rs
@@ -93,12 +93,14 @@ mod tests {
schema_id: 0,
level: 0,
extra_files: Vec::new(),
- creation_time: DateTime::<Utc>::from_timestamp(0, 0).unwrap(),
+ creation_time: DateTime::<Utc>::from_timestamp(0, 0),
delete_row_count: None,
embedded_index: None,
first_row_id: None,
write_cols: None,
external_path: None,
+ file_source: None,
+ value_stats_cols: None,
}
}
diff --git a/crates/paimon/src/table/bucket_filter.rs b/crates/paimon/src/table/bucket_filter.rs
new file mode 100644
index 0000000..ea300df
--- /dev/null
+++ b/crates/paimon/src/table/bucket_filter.rs
@@ -0,0 +1,193 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Bucket and partition predicate extraction and bucket hash pruning.
+
+use crate::spec::{
+ field_idx_to_partition_idx, BinaryRow, DataField, DataType, Datum, Predicate, PredicateOperator,
+};
+use std::collections::HashSet;
+
+pub(super) fn split_partition_and_data_predicates(
+ filter: Predicate,
+ fields: &[DataField],
+ partition_keys: &[String],
+) -> (Option<Predicate>, Vec<Predicate>) {
+ let mapping = field_idx_to_partition_idx(fields, partition_keys);
+ let mut partition_predicates = Vec::new();
+ let mut data_predicates = Vec::new();
+
+ for conjunct in filter.split_and() {
+ let strict_partition_only = conjunct.references_only_mapped_fields(&mapping);
+
+ if let Some(projected) = conjunct.project_field_index_inclusive(&mapping) {
+ partition_predicates.push(projected);
+ }
+
+ // Keep any conjunct that is not fully partition-only for data-level
+ // stats pruning, even if part of it contributed to partition pruning.
+ if !strict_partition_only {
+ data_predicates.push(conjunct);
+ }
+ }
+
+ let partition_predicate = if partition_predicates.is_empty() {
+ None
+ } else {
+ Some(Predicate::and(partition_predicates))
+ };
+
+ (partition_predicate, data_predicates)
+}
+
+/// Extract a predicate projected onto the given key columns.
+///
+/// This is a generic utility that works for both partition keys and bucket keys.
+/// Returns `None` if no conjuncts reference the given keys.
+pub(super) fn extract_predicate_for_keys(
+ filter: &Predicate,
+ fields: &[DataField],
+ keys: &[String],
+) -> Option<Predicate> {
+ if keys.is_empty() {
+ return None;
+ }
+ let mapping = field_idx_to_partition_idx(fields, keys);
+ let projected: Vec<Predicate> = filter
+ .clone()
+ .split_and()
+ .into_iter()
+ .filter_map(|conjunct| conjunct.project_field_index_inclusive(&mapping))
+ .collect();
+ if projected.is_empty() {
+ None
+ } else {
+ Some(Predicate::and(projected))
+ }
+}
+
+/// Compute the set of target buckets from a bucket predicate.
+///
+/// Extracts equal-value literals for each bucket key field from the predicate,
+/// builds a BinaryRow, hashes it, and returns the target bucket(s).
+///
+/// Supports:
+/// - `key = value` (single bucket)
+/// - `key IN (v1, v2, ...)` (multiple buckets)
+/// - AND of the above for composite bucket keys
+///
+/// Returns `None` if the predicate cannot determine target buckets (fail-open).
+pub(super) fn compute_target_buckets(
+ bucket_predicate: &Predicate,
+ bucket_key_fields: &[DataField],
+ total_buckets: i32,
+) -> Option<HashSet<i32>> {
+ if total_buckets <= 0 || bucket_key_fields.is_empty() {
+ return None;
+ }
+
+ // Collect equal-value candidates per bucket key field (by projected index).
+ // Each field can have one value (Eq) or multiple values (In).
+ let num_keys = bucket_key_fields.len();
+ let mut field_candidates: Vec<Option<Vec<&Datum>>> = vec![None; num_keys];
+
+ collect_eq_candidates(bucket_predicate, &mut field_candidates);
+
+ // All bucket key fields must have candidates.
+ let candidates: Vec<&Vec<&Datum>> =
+ field_candidates.iter().filter_map(|c| c.as_ref()).collect();
+ if candidates.len() != num_keys {
+ return None;
+ }
+
+ // Compute cartesian product of candidates and hash each combination.
+ let mut buckets = HashSet::new();
+ let mut combo: Vec<usize> = vec![0; num_keys];
+ loop {
+ let datums: Vec<(&Datum, &DataType)> = (0..num_keys)
+ .map(|i| {
+ let vals = field_candidates[i].as_ref().unwrap();
+ (vals[combo[i]], bucket_key_fields[i].data_type())
+ })
+ .collect();
+
+ if let Some(bucket) = BinaryRow::compute_bucket_from_datums(&datums, total_buckets) {
+ buckets.insert(bucket);
+ } else {
+ return None;
+ }
+
+ // Advance the combination counter (rightmost first).
+ let mut carry = true;
+ for i in (0..num_keys).rev() {
+ if carry {
+ combo[i] += 1;
+ if combo[i] < field_candidates[i].as_ref().unwrap().len() {
+ carry = false;
+ } else {
+ combo[i] = 0;
+ }
+ }
+ }
+ if carry {
+ break;
+ }
+ }
+
+ if buckets.is_empty() {
+ None
+ } else {
+ Some(buckets)
+ }
+}
+
+/// Recursively collect Eq/In literal candidates from a predicate for each bucket key field.
+fn collect_eq_candidates<'a>(
+ predicate: &'a Predicate,
+ field_candidates: &mut Vec<Option<Vec<&'a Datum>>>,
+) {
+ match predicate {
+ Predicate::And(children) => {
+ for child in children {
+ collect_eq_candidates(child, field_candidates);
+ }
+ }
+ Predicate::Leaf {
+ index,
+ op,
+ literals,
+ ..
+ } => {
+ if *index < field_candidates.len() {
+ match op {
+ PredicateOperator::Eq => {
+ if let Some(lit) = literals.first() {
+ field_candidates[*index] = Some(vec![lit]);
+ }
+ }
+ PredicateOperator::In => {
+ if !literals.is_empty() {
+ field_candidates[*index] = Some(literals.iter().collect());
+ }
+ }
+ _ => {}
+ }
+ }
+ }
+ _ => {}
+ }
+}
diff --git a/crates/paimon/src/table/mod.rs b/crates/paimon/src/table/mod.rs
index ae553d7..481c757 100644
--- a/crates/paimon/src/table/mod.rs
+++ b/crates/paimon/src/table/mod.rs
@@ -18,10 +18,12 @@
//! Table API for Apache Paimon
pub(crate) mod bin_pack;
+mod bucket_filter;
mod read_builder;
pub(crate) mod schema_manager;
mod snapshot_manager;
mod source;
+mod stats_filter;
mod table_scan;
mod tag_manager;
diff --git a/crates/paimon/src/table/source.rs b/crates/paimon/src/table/source.rs
index 532179a..a137a9c 100644
--- a/crates/paimon/src/table/source.rs
+++ b/crates/paimon/src/table/source.rs
@@ -20,7 +20,7 @@
//! Reference: [org.apache.paimon.table.source](https://github.com/apache/paimon/blob/master/paimon-core/src/main/java/org/apache/paimon/table/source/).
use crate::spec::{BinaryRow, DataFileMeta};
-use crate::table::table_scan::group_by_overlapping_row_id;
+use crate::table::stats_filter::group_by_overlapping_row_id;
use serde::{Deserialize, Serialize};
// ======================= DeletionFile ===============================
diff --git a/crates/paimon/src/table/stats_filter.rs b/crates/paimon/src/table/stats_filter.rs
new file mode 100644
index 0000000..171b00d
--- /dev/null
+++ b/crates/paimon/src/table/stats_filter.rs
@@ -0,0 +1,605 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Data-level stats predicate filtering for manifest entries and data evolution groups.
+
+use super::Table;
+use crate::arrow::schema_evolution::create_index_mapping;
+use crate::spec::{
+ extract_datum, BinaryRow, DataField, DataFileMeta, DataType, Datum, Predicate,
+ PredicateOperator,
+};
+use std::cmp::Ordering;
+use std::collections::HashMap;
+use std::sync::Arc;
+
+#[derive(Debug, Clone)]
+pub(super) struct FileStatsRows {
+ pub(super) row_count: i64,
+ min_values: Option<BinaryRow>,
+ max_values: Option<BinaryRow>,
+ null_counts: Vec<Option<i64>>,
+ /// Maps schema field index → stats index. `None` means identity mapping
+ /// (stats cover all schema fields in order). `Some` is used when
+ /// `value_stats_cols` or `write_cols` is present (dense mode).
+ stats_col_mapping: Option<Vec<Option<usize>>>,
+}
+
+impl FileStatsRows {
+ /// Build a `FileStatsRows` for manifest-level partition stats (no column mapping).
+ pub(super) fn for_manifest_partition(
+ row_count: i64,
+ min_values: Option<BinaryRow>,
+ max_values: Option<BinaryRow>,
+ null_counts: Vec<Option<i64>>,
+ ) -> Self {
+ Self {
+ row_count,
+ min_values,
+ max_values,
+ null_counts,
+ stats_col_mapping: None,
+ }
+ }
+
+ /// Build file stats from a data file, respecting `value_stats_cols`.
+ ///
+ /// When `value_stats_cols` is `None`, stats cover all fields in `schema_fields` order.
+ /// When `value_stats_cols` is `Some`, stats are in dense mode — only covering those
+ /// columns, and the mapping from schema field index to stats index is built by name.
+ pub(super) fn from_data_file(file: &DataFileMeta, schema_fields: &[DataField]) -> Self {
+ // Determine which columns the stats cover and build the mapping.
+ // Priority: value_stats_cols > write_cols > all schema fields.
+ let stats_col_mapping = if let Some(cols) = &file.value_stats_cols {
+ let mapping: Vec<Option<usize>> = schema_fields
+ .iter()
+ .map(|field| cols.iter().position(|c| c == field.name()))
+ .collect();
+ Some(mapping)
+ } else if let Some(cols) = &file.write_cols {
+ let mapping: Vec<Option<usize>> = schema_fields
+ .iter()
+ .map(|field| cols.iter().position(|c| c == field.name()))
+ .collect();
+ Some(mapping)
+ } else {
+ None
+ };
+
+ Self {
+ row_count: file.row_count,
+ min_values: BinaryRow::from_serialized_bytes(file.value_stats.min_values()).ok(),
+ max_values: BinaryRow::from_serialized_bytes(file.value_stats.max_values()).ok(),
+ null_counts: file.value_stats.null_counts().clone(),
+ stats_col_mapping,
+ }
+ }
+
+ /// Resolve a schema field index to the corresponding stats index.
+ fn stats_index(&self, schema_index: usize) -> Option<usize> {
+ match &self.stats_col_mapping {
+ None => Some(schema_index),
+ Some(mapping) => mapping.get(schema_index).copied().flatten(),
+ }
+ }
+
+ fn null_count(&self, stats_index: usize) -> Option<i64> {
+ self.null_counts.get(stats_index).copied().flatten()
+ }
+}
+
+#[derive(Debug)]
+pub(super) struct ResolvedStatsSchema {
+ file_fields: Vec<DataField>,
+ field_mapping: Vec<Option<usize>>,
+}
+
+fn identity_field_mapping(num_fields: usize) -> Vec<Option<usize>> {
+ (0..num_fields).map(Some).collect()
+}
+
+fn normalize_field_mapping(mapping: Option<Vec<i32>>, num_fields: usize) -> Vec<Option<usize>> {
+ mapping
+ .map(|field_mapping| {
+ field_mapping
+ .into_iter()
+ .map(|index| usize::try_from(index).ok())
+ .collect()
+ })
+ .unwrap_or_else(|| identity_field_mapping(num_fields))
+}
+
+/// Check whether a data file *may* contain rows matching all `predicates`.
+///
+/// Pruning is evaluated per file and fails open when stats cannot be
+/// interpreted safely, including schema mismatches, incompatible stats arity,
+/// and missing or corrupted stats.
+pub(super) fn data_file_matches_predicates(
+ file: &DataFileMeta,
+ predicates: &[Predicate],
+ current_schema_id: i64,
+ schema_fields: &[DataField],
+) -> bool {
+ if predicates.is_empty() {
+ return true;
+ }
+
+ if predicates
+ .iter()
+ .any(|p| matches!(p, Predicate::AlwaysFalse))
+ {
+ return false;
+ }
+ if predicates
+ .iter()
+ .all(|p| matches!(p, Predicate::AlwaysTrue))
+ {
+ return true;
+ }
+
+ if file.schema_id != current_schema_id {
+ return true;
+ }
+
+ let stats = FileStatsRows::from_data_file(file, schema_fields);
+
+ predicates
+ .iter()
+ .all(|predicate| data_predicate_may_match(predicate, &stats))
+}
+
+async fn resolve_stats_schema(
+ table: &Table,
+ file_schema_id: i64,
+ schema_cache: &mut HashMap<i64, Option<Arc<ResolvedStatsSchema>>>,
+) -> Option<Arc<ResolvedStatsSchema>> {
+ if let Some(cached) = schema_cache.get(&file_schema_id) {
+ return cached.clone();
+ }
+
+ let table_schema = table.schema();
+ let current_fields = table_schema.fields();
+ let resolved = if file_schema_id == table_schema.id() {
+ Some(Arc::new(ResolvedStatsSchema {
+ file_fields: current_fields.to_vec(),
+ field_mapping: identity_field_mapping(current_fields.len()),
+ }))
+ } else {
+ let file_schema = table.schema_manager().schema(file_schema_id).await.ok()?;
+ let file_fields = file_schema.fields().to_vec();
+ Some(Arc::new(ResolvedStatsSchema {
+ field_mapping: normalize_field_mapping(
+ create_index_mapping(current_fields, &file_fields),
+ current_fields.len(),
+ ),
+ file_fields,
+ }))
+ };
+
+ schema_cache.insert(file_schema_id, resolved.clone());
+ resolved
+}
+
+pub(super) async fn data_file_matches_predicates_for_table(
+ table: &Table,
+ file: &DataFileMeta,
+ predicates: &[Predicate],
+ schema_cache: &mut HashMap<i64, Option<Arc<ResolvedStatsSchema>>>,
+) -> bool {
+ if predicates.is_empty() {
+ return true;
+ }
+
+ if file.schema_id == table.schema().id() {
+ return data_file_matches_predicates(
+ file,
+ predicates,
+ table.schema().id(),
+ table.schema().fields(),
+ );
+ }
+
+ let Some(resolved) = resolve_stats_schema(table, file.schema_id, schema_cache).await else {
+ return true;
+ };
+
+ let stats = FileStatsRows::from_data_file(file, &resolved.file_fields);
+
+ predicates.iter().all(|predicate| {
+ data_predicate_may_match_with_schema(
+ predicate,
+ &stats,
+ &resolved.field_mapping,
+ &resolved.file_fields,
+ )
+ })
+}
+
+fn data_predicate_may_match(predicate: &Predicate, stats: &FileStatsRows) -> bool {
+ match predicate {
+ Predicate::AlwaysTrue => true,
+ Predicate::AlwaysFalse => false,
+ Predicate::And(children) => children
+ .iter()
+ .all(|child| data_predicate_may_match(child, stats)),
+ Predicate::Or(_) | Predicate::Not(_) => true,
+ Predicate::Leaf {
+ index,
+ data_type,
+ op,
+ literals,
+ ..
+ } => {
+ let Some(stats_idx) = stats.stats_index(*index) else {
+ return true;
+ };
+ data_leaf_may_match(stats_idx, data_type, data_type, *op, literals, stats)
+ }
+ }
+}
+
+fn data_predicate_may_match_with_schema(
+ predicate: &Predicate,
+ stats: &FileStatsRows,
+ field_mapping: &[Option<usize>],
+ file_fields: &[DataField],
+) -> bool {
+ match predicate {
+ Predicate::AlwaysTrue => true,
+ Predicate::AlwaysFalse => false,
+ Predicate::And(children) => children.iter().all(|child| {
+ data_predicate_may_match_with_schema(child, stats, field_mapping, file_fields)
+ }),
+ Predicate::Or(_) | Predicate::Not(_) => true,
+ Predicate::Leaf {
+ index,
+ data_type,
+ op,
+ literals,
+ ..
+ } => match field_mapping.get(*index).copied().flatten() {
+ Some(file_index) => {
+ let Some(file_field) = file_fields.get(file_index) else {
+ return true;
+ };
+ let Some(stats_idx) = stats.stats_index(file_index) else {
+ return true;
+ };
+ data_leaf_may_match(
+ stats_idx,
+ file_field.data_type(),
+ data_type,
+ *op,
+ literals,
+ stats,
+ )
+ }
+ None => missing_field_may_match(*op, stats.row_count),
+ },
+ }
+}
+
+pub(super) fn data_leaf_may_match(
+ index: usize,
+ stats_data_type: &DataType,
+ predicate_data_type: &DataType,
+ op: PredicateOperator,
+ literals: &[Datum],
+ stats: &FileStatsRows,
+) -> bool {
+ let row_count = stats.row_count;
+ if row_count <= 0 {
+ return false;
+ }
+
+ let null_count = stats.null_count(index);
+ let all_null = null_count.map(|count| count == row_count);
+
+ match op {
+ PredicateOperator::IsNull => {
+ return null_count.is_none_or(|count| count > 0);
+ }
+ PredicateOperator::IsNotNull => {
+ return all_null != Some(true);
+ }
+ PredicateOperator::In | PredicateOperator::NotIn => {
+ return true;
+ }
+ PredicateOperator::Eq
+ | PredicateOperator::NotEq
+ | PredicateOperator::Lt
+ | PredicateOperator::LtEq
+ | PredicateOperator::Gt
+ | PredicateOperator::GtEq => {}
+ }
+
+ if all_null == Some(true) {
+ return false;
+ }
+
+ let literal = match literals.first() {
+ Some(literal) => literal,
+ None => return true,
+ };
+
+ let min_value = match stats
+ .min_values
+ .as_ref()
+ .and_then(|row| extract_stats_datum(row, index, stats_data_type))
+ .and_then(|datum| coerce_stats_datum_for_predicate(datum, predicate_data_type))
+ {
+ Some(value) => value,
+ None => return true,
+ };
+ let max_value = match stats
+ .max_values
+ .as_ref()
+ .and_then(|row| extract_stats_datum(row, index, stats_data_type))
+ .and_then(|datum| coerce_stats_datum_for_predicate(datum, predicate_data_type))
+ {
+ Some(value) => value,
+ None => return true,
+ };
+
+ match op {
+ PredicateOperator::Eq => {
+ !matches!(literal.partial_cmp(&min_value), Some(Ordering::Less))
+ && !matches!(literal.partial_cmp(&max_value), Some(Ordering::Greater))
+ }
+ PredicateOperator::NotEq => !(min_value == *literal && max_value == *literal),
+ PredicateOperator::Lt => !matches!(
+ min_value.partial_cmp(literal),
+ Some(Ordering::Greater | Ordering::Equal)
+ ),
+ PredicateOperator::LtEq => {
+ !matches!(min_value.partial_cmp(literal), Some(Ordering::Greater))
+ }
+ PredicateOperator::Gt => !matches!(
+ max_value.partial_cmp(literal),
+ Some(Ordering::Less | Ordering::Equal)
+ ),
+ PredicateOperator::GtEq => !matches!(max_value.partial_cmp(literal), Some(Ordering::Less)),
+ PredicateOperator::IsNull
+ | PredicateOperator::IsNotNull
+ | PredicateOperator::In
+ | PredicateOperator::NotIn => true,
+ }
+}
+
+fn missing_field_may_match(op: PredicateOperator, row_count: i64) -> bool {
+ if row_count <= 0 {
+ return false;
+ }
+
+ matches!(op, PredicateOperator::IsNull)
+}
+
+fn coerce_stats_datum_for_predicate(datum: Datum, predicate_data_type: &DataType) -> Option<Datum> {
+ match (datum, predicate_data_type) {
+ (datum @ Datum::Bool(_), DataType::Boolean(_))
+ | (datum @ Datum::TinyInt(_), DataType::TinyInt(_))
+ | (datum @ Datum::SmallInt(_), DataType::SmallInt(_))
+ | (datum @ Datum::Int(_), DataType::Int(_))
+ | (datum @ Datum::Long(_), DataType::BigInt(_))
+ | (datum @ Datum::Float(_), DataType::Float(_))
+ | (datum @ Datum::Double(_), DataType::Double(_))
+ | (datum @ Datum::String(_), DataType::VarChar(_))
+ | (datum @ Datum::String(_), DataType::Char(_))
+ | (datum @ Datum::Bytes(_), DataType::Binary(_))
+ | (datum @ Datum::Bytes(_), DataType::VarBinary(_))
+ | (datum @ Datum::Date(_), DataType::Date(_))
+ | (datum @ Datum::Time(_), DataType::Time(_))
+ | (datum @ Datum::Timestamp { .. }, DataType::Timestamp(_))
+ | (datum @ Datum::LocalZonedTimestamp { .. }, DataType::LocalZonedTimestamp(_))
+ | (datum @ Datum::Decimal { .. }, DataType::Decimal(_)) => Some(datum),
+ (Datum::TinyInt(value), DataType::SmallInt(_)) => Some(Datum::SmallInt(value as i16)),
+ (Datum::TinyInt(value), DataType::Int(_)) => Some(Datum::Int(value as i32)),
+ (Datum::TinyInt(value), DataType::BigInt(_)) => Some(Datum::Long(value as i64)),
+ (Datum::SmallInt(value), DataType::Int(_)) => Some(Datum::Int(value as i32)),
+ (Datum::SmallInt(value), DataType::BigInt(_)) => Some(Datum::Long(value as i64)),
+ (Datum::Int(value), DataType::BigInt(_)) => Some(Datum::Long(value as i64)),
+ (Datum::Float(value), DataType::Double(_)) => Some(Datum::Double(value as f64)),
+ _ => None,
+ }
+}
+
+fn extract_stats_datum(row: &BinaryRow, index: usize, data_type: &DataType) -> Option<Datum> {
+ let min_row_len = BinaryRow::cal_fix_part_size_in_bytes(row.arity()) as usize;
+ if index >= row.arity() as usize || row.data().len() < min_row_len {
+ return None;
+ }
+
+ match extract_datum(row, index, data_type) {
+ Ok(Some(datum)) => Some(datum),
+ Ok(None) | Err(_) => None,
+ }
+}
+
+/// Check whether a data-evolution file group *may* contain rows matching all `predicates`.
+///
+/// In data evolution mode, a logical row can be spread across multiple files with
+/// different column sets. After `group_by_overlapping_row_id`, each group contains
+/// files covering the same row ID range. Stats for each field come from the file
+/// with the highest `max_sequence_number` that actually contains that field.
+///
+/// Reference: [DataEvolutionFileStoreScan.evolutionStats](https://github.com/apache/paimon/blob/master/paimon-core/src/main/java/org/apache/paimon/operation/DataEvolutionFileStoreScan.java)
+pub(super) fn data_evolution_group_matches_predicates(
+ group: &[DataFileMeta],
+ predicates: &[Predicate],
+ table_fields: &[DataField],
+) -> bool {
+ if predicates.is_empty() || group.is_empty() {
+ return true;
+ }
+
+ if predicates
+ .iter()
+ .any(|p| matches!(p, Predicate::AlwaysFalse))
+ {
+ return false;
+ }
+ if predicates
+ .iter()
+ .all(|p| matches!(p, Predicate::AlwaysTrue))
+ {
+ return true;
+ }
+
+ // Sort files by max_sequence_number descending so the highest-seq file wins per field.
+ let mut sorted_files: Vec<&DataFileMeta> = group.iter().collect();
+ sorted_files.sort_by(|a, b| b.max_sequence_number.cmp(&a.max_sequence_number));
+
+ // For each table field, find which file (index in sorted_files) provides it,
+ // and the field's offset within that file's stats.
+ let field_sources: Vec<Option<(usize, usize)>> = table_fields
+ .iter()
+ .map(|field| {
+ for (file_idx, file) in sorted_files.iter().enumerate() {
+ let file_columns = file_stats_columns(file, table_fields);
+ for (stats_idx, col_name) in file_columns.iter().enumerate() {
+ if *col_name == field.name() {
+ return Some((file_idx, stats_idx));
+ }
+ }
+ }
+ None
+ })
+ .collect();
+
+ // Build per-file stats without arity validation — data evolution files
+ // may have fewer columns than the current table schema.
+ let file_stats: Vec<FileStatsRows> = sorted_files
+ .iter()
+ .map(|file| FileStatsRows::from_data_file(file, table_fields))
+ .collect();
+
+ // row_count is the max across the group (overlapping row ranges).
+ let row_count = group.iter().map(|f| f.row_count).max().unwrap_or(0);
+
+ predicates.iter().all(|predicate| {
+ data_evolution_predicate_may_match(
+ predicate,
+ table_fields,
+ &field_sources,
+ &file_stats,
+ row_count,
+ )
+ })
+}
+
+/// Resolve which columns a file's value stats cover.
+/// If `value_stats_cols` is set, those are the stats columns. Otherwise, the file's stats
+/// cover all table fields (or `write_cols` if present).
+fn file_stats_columns<'a>(file: &'a DataFileMeta, table_fields: &'a [DataField]) -> Vec<&'a str> {
+ if let Some(cols) = &file.value_stats_cols {
+ return cols.iter().map(|s| s.as_str()).collect();
+ }
+ match &file.write_cols {
+ Some(cols) => cols.iter().map(|s| s.as_str()).collect(),
+ None => table_fields.iter().map(|f| f.name()).collect(),
+ }
+}
+
+fn data_evolution_predicate_may_match(
+ predicate: &Predicate,
+ table_fields: &[DataField],
+ field_sources: &[Option<(usize, usize)>],
+ file_stats: &[FileStatsRows],
+ row_count: i64,
+) -> bool {
+ match predicate {
+ Predicate::AlwaysTrue => true,
+ Predicate::AlwaysFalse => false,
+ Predicate::And(children) => children.iter().all(|child| {
+ data_evolution_predicate_may_match(
+ child,
+ table_fields,
+ field_sources,
+ file_stats,
+ row_count,
+ )
+ }),
+ Predicate::Or(_) | Predicate::Not(_) => true,
+ Predicate::Leaf {
+ index,
+ data_type,
+ op,
+ literals,
+ ..
+ } => {
+ let Some(source) = field_sources.get(*index).copied().flatten() else {
+ return missing_field_may_match(*op, row_count);
+ };
+ let (file_idx, stats_idx) = source;
+ let stats = &file_stats[file_idx];
+ let stats_data_type = table_fields
+ .get(*index)
+ .map(|f| f.data_type())
+ .unwrap_or(data_type);
+ data_leaf_may_match(stats_idx, stats_data_type, data_type, *op, literals, stats)
+ }
+ }
+}
+
+/// Groups data files by overlapping `row_id_range` for data evolution.
+///
+/// Files are sorted by `(first_row_id, -max_sequence_number)`. Files whose row ID ranges
+/// overlap are merged into the same group (they contain different columns for the same rows).
+/// Files without `first_row_id` become their own group.
+///
+/// Reference: [DataEvolutionSplitGenerator](https://github.com/apache/paimon/blob/master/paimon-core/src/main/java/org/apache/paimon/table/source/splitread/DataEvolutionSplitGenerator.java)
+pub(crate) fn group_by_overlapping_row_id(mut files: Vec<DataFileMeta>) -> Vec<Vec<DataFileMeta>> {
+ files.sort_by(|a, b| {
+ let a_row_id = a.first_row_id.unwrap_or(i64::MIN);
+ let b_row_id = b.first_row_id.unwrap_or(i64::MIN);
+ a_row_id
+ .cmp(&b_row_id)
+ .then_with(|| b.max_sequence_number.cmp(&a.max_sequence_number))
+ });
+
+ let mut result: Vec<Vec<DataFileMeta>> = Vec::new();
+ let mut current_group: Vec<DataFileMeta> = Vec::new();
+ let mut current_range_end: i64 = i64::MIN;
+
+ for file in files {
+ match file.row_id_range() {
+ None => {
+ if !current_group.is_empty() {
+ result.push(std::mem::take(&mut current_group));
+ current_range_end = i64::MIN;
+ }
+ result.push(vec![file]);
+ }
+ Some((start, end)) => {
+ if current_group.is_empty() || start <= current_range_end {
+ if end > current_range_end {
+ current_range_end = end;
+ }
+ current_group.push(file);
+ } else {
+ result.push(std::mem::take(&mut current_group));
+ current_range_end = end;
+ current_group.push(file);
+ }
+ }
+ }
+ }
+ if !current_group.is_empty() {
+ result.push(current_group);
+ }
+ result
+}
diff --git a/crates/paimon/src/table/table_scan.rs b/crates/paimon/src/table/table_scan.rs
index ba3e071..9f2ebf9 100644
--- a/crates/paimon/src/table/table_scan.rs
+++ b/crates/paimon/src/table/table_scan.rs
@@ -20,13 +20,19 @@
//! Reference: [pypaimon.read.table_scan.TableScan](https://github.com/apache/paimon/blob/release-1.3/paimon-python/pypaimon/read/table_scan.py)
//! and [FullStartingScanner](https://github.com/apache/paimon/blob/release-1.3/paimon-python/pypaimon/read/scanner/full_starting_scanner.py).
+use super::bucket_filter::{
+ compute_target_buckets, extract_predicate_for_keys, split_partition_and_data_predicates,
+};
+use super::stats_filter::{
+ data_evolution_group_matches_predicates, data_file_matches_predicates,
+ data_file_matches_predicates_for_table, data_leaf_may_match, group_by_overlapping_row_id,
+ FileStatsRows, ResolvedStatsSchema,
+};
use super::Table;
-use crate::arrow::schema_evolution::create_index_mapping;
use crate::io::FileIO;
use crate::spec::{
- eval_row, extract_datum, field_idx_to_partition_idx, BinaryRow, CoreOptions, DataField,
- DataFileMeta, DataType, Datum, FileKind, IndexManifest, ManifestEntry, PartitionComputer,
- Predicate, PredicateOperator, Snapshot,
+ eval_row, BinaryRow, CoreOptions, DataField, DataFileMeta, FileKind, IndexManifest,
+ ManifestEntry, ManifestFileMeta, PartitionComputer, Predicate, Snapshot,
};
use crate::table::bin_pack::split_for_batch;
use crate::table::source::{DataSplit, DataSplitBuilder, DeletionFile, PartitionBucket, Plan};
@@ -34,7 +40,6 @@ use crate::table::SnapshotManager;
use crate::table::TagManager;
use crate::Error;
use futures::{StreamExt, TryStreamExt};
-use std::cmp::Ordering;
use std::collections::{HashMap, HashSet};
use std::sync::Arc;
@@ -63,22 +68,145 @@ async fn read_manifest_list(
crate::spec::from_avro_bytes::<crate::spec::ManifestFileMeta>(&bytes)
}
+/// Check whether a manifest file *may* contain entries matching the partition predicate,
+/// using the manifest-level partition stats (min/max over all entries in the manifest).
+///
+/// The `predicate` must already be projected to partition-field indices.
+fn manifest_file_matches_partition_predicate(
+ meta: &ManifestFileMeta,
+ predicate: &Predicate,
+ partition_fields: &[DataField],
+) -> bool {
+ let stats = meta.partition_stats();
+
+ let min_values = BinaryRow::from_serialized_bytes(stats.min_values()).ok();
+ let max_values = BinaryRow::from_serialized_bytes(stats.max_values()).ok();
+ let null_counts = stats.null_counts().clone();
+
+ let file_stats = FileStatsRows::for_manifest_partition(
+ meta.num_added_files() + meta.num_deleted_files(),
+ min_values,
+ max_values,
+ null_counts,
+ );
+
+ manifest_partition_predicate_may_match(predicate, &file_stats, partition_fields)
+}
+
+fn manifest_partition_predicate_may_match(
+ predicate: &Predicate,
+ stats: &FileStatsRows,
+ partition_fields: &[DataField],
+) -> bool {
+ match predicate {
+ Predicate::AlwaysTrue => true,
+ Predicate::AlwaysFalse => false,
+ Predicate::And(children) => children
+ .iter()
+ .all(|child| manifest_partition_predicate_may_match(child, stats, partition_fields)),
+ Predicate::Or(_) | Predicate::Not(_) => true,
+ Predicate::Leaf {
+ index,
+ data_type,
+ op,
+ literals,
+ ..
+ } => {
+ let stats_data_type = match partition_fields.get(*index) {
+ Some(f) => f.data_type(),
+ None => return true,
+ };
+ data_leaf_may_match(*index, stats_data_type, data_type, *op, literals, stats)
+ }
+ }
+}
+
/// Reads all manifest entries for a snapshot (base + delta manifest lists, then each manifest file).
+/// Applies filters during concurrent manifest reading to reduce entries early:
+/// - Manifest-file-level partition stats pruning (skip entire manifest files)
+/// - DV level-0 filtering per entry
+/// - Partition predicate filtering per entry
+/// - Data-level stats pruning per entry (current schema only, cross-schema fail-open)
+#[allow(clippy::too_many_arguments)]
async fn read_all_manifest_entries(
file_io: &FileIO,
table_path: &str,
snapshot: &Snapshot,
+ deletion_vectors_enabled: bool,
+ has_primary_keys: bool,
+ partition_predicate: Option<&Predicate>,
+ partition_fields: &[DataField],
+ data_predicates: &[Predicate],
+ current_schema_id: i64,
+ schema_fields: &[DataField],
+ bucket_predicate: Option<&Predicate>,
+ bucket_key_fields: &[DataField],
) -> crate::Result<Vec<ManifestEntry>> {
let mut manifest_files =
read_manifest_list(file_io, table_path, snapshot.base_manifest_list()).await?;
let delta = read_manifest_list(file_io, table_path, snapshot.delta_manifest_list()).await?;
manifest_files.extend(delta);
+ // Manifest-file-level partition stats pruning: skip entire manifest files
+ // whose partition range doesn't overlap the partition predicate.
+ if let Some(pred) = partition_predicate {
+ if !partition_fields.is_empty() {
+ manifest_files.retain(|meta| {
+ manifest_file_matches_partition_predicate(meta, pred, partition_fields)
+ });
+ }
+ }
+
let manifest_path_prefix = format!("{}/{}", table_path.trim_end_matches('/'), MANIFEST_DIR);
let all_entries: Vec<ManifestEntry> = futures::stream::iter(manifest_files)
.map(|meta| {
let path = format!("{}/{}", manifest_path_prefix, meta.file_name());
- async move { crate::spec::Manifest::read(file_io, &path).await }
+ async move {
+ let entries = crate::spec::Manifest::read(file_io, &path).await?;
+ // Per-task bucket cache (few distinct total_buckets values per manifest).
+ let mut bucket_cache: HashMap<i32, Option<HashSet<i32>>> = HashMap::new();
+ let filtered: Vec<ManifestEntry> = entries
+ .into_iter()
+ .filter(|entry| {
+ if deletion_vectors_enabled && has_primary_keys && entry.file().level == 0 {
+ return false;
+ }
+ if has_primary_keys && entry.bucket() < 0 {
+ return false;
+ }
+ if let Some(pred) = bucket_predicate {
+ let total = entry.total_buckets();
+ let targets = bucket_cache.entry(total).or_insert_with(|| {
+ compute_target_buckets(pred, bucket_key_fields, total)
+ });
+ if let Some(targets) = targets {
+ if !targets.contains(&entry.bucket()) {
+ return false;
+ }
+ }
+ }
+ if let Some(pred) = partition_predicate {
+ match partition_matches_predicate(entry.partition(), pred) {
+ Ok(false) => return false,
+ Ok(true) => {}
+ Err(_) => {}
+ }
+ }
+ if !data_predicates.is_empty()
+ && !data_file_matches_predicates(
+ entry.file(),
+ data_predicates,
+ current_schema_id,
+ schema_fields,
+ )
+ {
+ return false;
+ }
+ true
+ })
+ .collect();
+ Ok::<_, crate::Error>(filtered)
+ }
})
.buffered(64)
.try_collect::<Vec<_>>()
@@ -89,16 +217,6 @@ async fn read_all_manifest_entries(
Ok(all_entries)
}
-fn filter_manifest_entries(
- entries: Vec<ManifestEntry>,
- deletion_vectors_enabled: bool,
-) -> Vec<ManifestEntry> {
- entries
- .into_iter()
- .filter(|entry| !(deletion_vectors_enabled && entry.file().level == 0))
- .collect()
-}
-
/// Builds a map from (partition, bucket) to (data_file_name -> DeletionFile) from index manifest entries.
/// Only considers ADD entries with index_type "DELETION_VECTORS" and their deletion_vectors_ranges.
fn build_deletion_files_map(
@@ -175,464 +293,6 @@ fn partition_matches_predicate(
}
}
-/// Groups data files by overlapping `row_id_range` for data evolution.
-///
-/// Files are sorted by `(first_row_id, -max_sequence_number)`. Files whose row ID ranges
-/// overlap are merged into the same group (they contain different columns for the same rows).
-/// Files without `first_row_id` become their own group.
-///
-/// Reference: [DataEvolutionSplitGenerator](https://github.com/apache/paimon/blob/master/paimon-core/src/main/java/org/apache/paimon/table/source/splitread/DataEvolutionSplitGenerator.java)
-pub(crate) fn group_by_overlapping_row_id(mut files: Vec<DataFileMeta>) -> Vec<Vec<DataFileMeta>> {
- files.sort_by(|a, b| {
- let a_row_id = a.first_row_id.unwrap_or(i64::MIN);
- let b_row_id = b.first_row_id.unwrap_or(i64::MIN);
- a_row_id
- .cmp(&b_row_id)
- .then_with(|| b.max_sequence_number.cmp(&a.max_sequence_number))
- });
-
- let mut result: Vec<Vec<DataFileMeta>> = Vec::new();
- let mut current_group: Vec<DataFileMeta> = Vec::new();
- // Track the end of the current merged row_id range.
- let mut current_range_end: i64 = i64::MIN;
-
- for file in files {
- match file.row_id_range() {
- None => {
- // Files without first_row_id become their own group.
- if !current_group.is_empty() {
- result.push(std::mem::take(&mut current_group));
- current_range_end = i64::MIN;
- }
- result.push(vec![file]);
- }
- Some((start, end)) => {
- if current_group.is_empty() || start <= current_range_end {
- // Overlaps with current range — merge into current group.
- if end > current_range_end {
- current_range_end = end;
- }
- current_group.push(file);
- } else {
- // No overlap — start a new group.
- result.push(std::mem::take(&mut current_group));
- current_range_end = end;
- current_group.push(file);
- }
- }
- }
- }
- if !current_group.is_empty() {
- result.push(current_group);
- }
- result
-}
-
-#[derive(Debug, Clone)]
-struct FileStatsRows {
- row_count: i64,
- min_values: Option<BinaryRow>,
- max_values: Option<BinaryRow>,
- null_counts: Vec<i64>,
-}
-
-impl FileStatsRows {
- /// Build file stats only when they are compatible with the expected file schema.
- fn try_from_data_file(file: &DataFileMeta, expected_fields: usize) -> Option<Self> {
- let stats = Self {
- row_count: file.row_count,
- min_values: BinaryRow::from_serialized_bytes(file.value_stats.min_values()).ok(),
- max_values: BinaryRow::from_serialized_bytes(file.value_stats.max_values()).ok(),
- null_counts: file.value_stats.null_counts().clone(),
- };
-
- stats.arity_matches(expected_fields).then_some(stats)
- }
-
- fn null_count(&self, index: usize) -> Option<i64> {
- self.null_counts.get(index).copied()
- }
-
- /// Check whether the stats rows have the expected number of fields.
- ///
- /// If either min or max BinaryRow has an arity different from
- /// `expected_fields`, the stats were likely written in dense mode or
- /// under a different schema — making index-based access unsafe.
- fn arity_matches(&self, expected_fields: usize) -> bool {
- let min_ok = self
- .min_values
- .as_ref()
- .is_none_or(|r| r.arity() as usize == expected_fields);
- let max_ok = self
- .max_values
- .as_ref()
- .is_none_or(|r| r.arity() as usize == expected_fields);
- let null_ok = self.null_counts.is_empty() || self.null_counts.len() == expected_fields;
- min_ok && max_ok && null_ok
- }
-}
-
-#[derive(Debug)]
-struct ResolvedStatsSchema {
- file_fields: Vec<DataField>,
- field_mapping: Vec<Option<usize>>,
-}
-
-fn identity_field_mapping(num_fields: usize) -> Vec<Option<usize>> {
- (0..num_fields).map(Some).collect()
-}
-
-fn normalize_field_mapping(mapping: Option<Vec<i32>>, num_fields: usize) -> Vec<Option<usize>> {
- mapping
- .map(|field_mapping| {
- field_mapping
- .into_iter()
- .map(|index| usize::try_from(index).ok())
- .collect()
- })
- .unwrap_or_else(|| identity_field_mapping(num_fields))
-}
-
-fn split_partition_and_data_predicates(
- filter: Predicate,
- fields: &[DataField],
- partition_keys: &[String],
-) -> (Option<Predicate>, Vec<Predicate>) {
- let mapping = field_idx_to_partition_idx(fields, partition_keys);
- let mut partition_predicates = Vec::new();
- let mut data_predicates = Vec::new();
-
- for conjunct in filter.split_and() {
- let strict_partition_only = conjunct.references_only_mapped_fields(&mapping);
-
- if let Some(projected) = conjunct.project_field_index_inclusive(&mapping) {
- partition_predicates.push(projected);
- }
-
- // Keep any conjunct that is not fully partition-only for data-level
- // stats pruning, even if part of it contributed to partition pruning.
- if !strict_partition_only {
- data_predicates.push(conjunct);
- }
- }
-
- let partition_predicate = if partition_predicates.is_empty() {
- None
- } else {
- Some(Predicate::and(partition_predicates))
- };
-
- (partition_predicate, data_predicates)
-}
-
-/// Check whether a data file *may* contain rows matching all `predicates`.
-///
-/// Pruning is evaluated per file and fails open when stats cannot be
-/// interpreted safely, including schema mismatches, incompatible stats arity,
-/// and missing or corrupted stats. Mixed-schema tables can still prune files
-/// written with the current schema; unsupported or inconclusive predicates are
-/// conservatively kept.
-fn data_file_matches_predicates(
- file: &DataFileMeta,
- predicates: &[Predicate],
- current_schema_id: i64,
- num_fields: usize,
-) -> bool {
- if predicates.is_empty() {
- return true;
- }
-
- // Evaluate constant predicates before consulting stats.
- if predicates
- .iter()
- .any(|p| matches!(p, Predicate::AlwaysFalse))
- {
- return false;
- }
- if predicates
- .iter()
- .all(|p| matches!(p, Predicate::AlwaysTrue))
- {
- return true;
- }
-
- if file.schema_id != current_schema_id {
- return true;
- }
-
- // Fail open if schema evolution or stats layout make index-based access unsafe.
- let Some(stats) = FileStatsRows::try_from_data_file(file, num_fields) else {
- return true;
- };
-
- predicates
- .iter()
- .all(|predicate| data_predicate_may_match(predicate, &stats))
-}
-
-async fn resolve_stats_schema(
- table: &Table,
- file_schema_id: i64,
- schema_cache: &mut HashMap<i64, Option<Arc<ResolvedStatsSchema>>>,
-) -> Option<Arc<ResolvedStatsSchema>> {
- if let Some(cached) = schema_cache.get(&file_schema_id) {
- return cached.clone();
- }
-
- let table_schema = table.schema();
- let current_fields = table_schema.fields();
- let resolved = if file_schema_id == table_schema.id() {
- Some(Arc::new(ResolvedStatsSchema {
- file_fields: current_fields.to_vec(),
- field_mapping: identity_field_mapping(current_fields.len()),
- }))
- } else {
- let file_schema = table.schema_manager().schema(file_schema_id).await.ok()?;
- let file_fields = file_schema.fields().to_vec();
- Some(Arc::new(ResolvedStatsSchema {
- field_mapping: normalize_field_mapping(
- create_index_mapping(current_fields, &file_fields),
- current_fields.len(),
- ),
- file_fields,
- }))
- };
-
- schema_cache.insert(file_schema_id, resolved.clone());
- resolved
-}
-
-async fn data_file_matches_predicates_for_table(
- table: &Table,
- file: &DataFileMeta,
- predicates: &[Predicate],
- schema_cache: &mut HashMap<i64, Option<Arc<ResolvedStatsSchema>>>,
-) -> bool {
- if predicates.is_empty() {
- return true;
- }
-
- if file.schema_id == table.schema().id() {
- return data_file_matches_predicates(
- file,
- predicates,
- table.schema().id(),
- table.schema().fields().len(),
- );
- }
-
- let Some(resolved) = resolve_stats_schema(table, file.schema_id, schema_cache).await else {
- return true;
- };
-
- let Some(stats) = FileStatsRows::try_from_data_file(file, resolved.file_fields.len()) else {
- return true;
- };
-
- predicates.iter().all(|predicate| {
- data_predicate_may_match_with_schema(
- predicate,
- &stats,
- &resolved.field_mapping,
- &resolved.file_fields,
- )
- })
-}
-
-fn data_predicate_may_match(predicate: &Predicate, stats: &FileStatsRows) -> bool {
- match predicate {
- Predicate::AlwaysTrue => true,
- Predicate::AlwaysFalse => false,
- Predicate::And(children) => children
- .iter()
- .all(|child| data_predicate_may_match(child, stats)),
- // Keep the first version conservative: only prune simple leaves and conjunctions.
- Predicate::Or(_) | Predicate::Not(_) => true,
- Predicate::Leaf {
- index,
- data_type,
- op,
- literals,
- ..
- } => data_leaf_may_match(*index, data_type, data_type, *op, literals, stats),
- }
-}
-
-fn data_predicate_may_match_with_schema(
- predicate: &Predicate,
- stats: &FileStatsRows,
- field_mapping: &[Option<usize>],
- file_fields: &[DataField],
-) -> bool {
- match predicate {
- Predicate::AlwaysTrue => true,
- Predicate::AlwaysFalse => false,
- Predicate::And(children) => children.iter().all(|child| {
- data_predicate_may_match_with_schema(child, stats, field_mapping, file_fields)
- }),
- // Keep the first version conservative: only prune simple leaves and conjunctions.
- Predicate::Or(_) | Predicate::Not(_) => true,
- Predicate::Leaf {
- index,
- data_type,
- op,
- literals,
- ..
- } => match field_mapping.get(*index).copied().flatten() {
- Some(file_index) => {
- let Some(file_field) = file_fields.get(file_index) else {
- return true;
- };
- data_leaf_may_match(
- file_index,
- file_field.data_type(),
- data_type,
- *op,
- literals,
- stats,
- )
- }
- None => missing_field_may_match(*op, stats.row_count),
- },
- }
-}
-
-fn data_leaf_may_match(
- index: usize,
- stats_data_type: &DataType,
- predicate_data_type: &DataType,
- op: PredicateOperator,
- literals: &[Datum],
- stats: &FileStatsRows,
-) -> bool {
- let row_count = stats.row_count;
- if row_count <= 0 {
- return false;
- }
-
- let null_count = stats.null_count(index);
- let all_null = null_count.map(|count| count == row_count);
-
- match op {
- PredicateOperator::IsNull => {
- return null_count.is_none_or(|count| count > 0);
- }
- PredicateOperator::IsNotNull => {
- return all_null != Some(true);
- }
- PredicateOperator::In | PredicateOperator::NotIn => {
- return true;
- }
- PredicateOperator::Eq
- | PredicateOperator::NotEq
- | PredicateOperator::Lt
- | PredicateOperator::LtEq
- | PredicateOperator::Gt
- | PredicateOperator::GtEq => {}
- }
-
- if all_null == Some(true) {
- return false;
- }
-
- let literal = match literals.first() {
- Some(literal) => literal,
- None => return true,
- };
-
- let min_value = match stats
- .min_values
- .as_ref()
- .and_then(|row| extract_stats_datum(row, index, stats_data_type))
- .and_then(|datum| coerce_stats_datum_for_predicate(datum, predicate_data_type))
- {
- Some(value) => value,
- None => return true,
- };
- let max_value = match stats
- .max_values
- .as_ref()
- .and_then(|row| extract_stats_datum(row, index, stats_data_type))
- .and_then(|datum| coerce_stats_datum_for_predicate(datum, predicate_data_type))
- {
- Some(value) => value,
- None => return true,
- };
-
- match op {
- PredicateOperator::Eq => {
- !matches!(literal.partial_cmp(&min_value), Some(Ordering::Less))
- && !matches!(literal.partial_cmp(&max_value), Some(Ordering::Greater))
- }
- PredicateOperator::NotEq => !(min_value == *literal && max_value == *literal),
- PredicateOperator::Lt => !matches!(
- min_value.partial_cmp(literal),
- Some(Ordering::Greater | Ordering::Equal)
- ),
- PredicateOperator::LtEq => {
- !matches!(min_value.partial_cmp(literal), Some(Ordering::Greater))
- }
- PredicateOperator::Gt => !matches!(
- max_value.partial_cmp(literal),
- Some(Ordering::Less | Ordering::Equal)
- ),
- PredicateOperator::GtEq => !matches!(max_value.partial_cmp(literal), Some(Ordering::Less)),
- PredicateOperator::IsNull
- | PredicateOperator::IsNotNull
- | PredicateOperator::In
- | PredicateOperator::NotIn => true,
- }
-}
-
-fn missing_field_may_match(op: PredicateOperator, row_count: i64) -> bool {
- if row_count <= 0 {
- return false;
- }
-
- matches!(op, PredicateOperator::IsNull)
-}
-
-fn coerce_stats_datum_for_predicate(datum: Datum, predicate_data_type: &DataType) -> Option<Datum> {
- match (datum, predicate_data_type) {
- (datum @ Datum::Bool(_), DataType::Boolean(_))
- | (datum @ Datum::TinyInt(_), DataType::TinyInt(_))
- | (datum @ Datum::SmallInt(_), DataType::SmallInt(_))
- | (datum @ Datum::Int(_), DataType::Int(_))
- | (datum @ Datum::Long(_), DataType::BigInt(_))
- | (datum @ Datum::Float(_), DataType::Float(_))
- | (datum @ Datum::Double(_), DataType::Double(_))
- | (datum @ Datum::String(_), DataType::VarChar(_))
- | (datum @ Datum::String(_), DataType::Char(_))
- | (datum @ Datum::Bytes(_), DataType::Binary(_))
- | (datum @ Datum::Bytes(_), DataType::VarBinary(_))
- | (datum @ Datum::Date(_), DataType::Date(_))
- | (datum @ Datum::Time(_), DataType::Time(_))
- | (datum @ Datum::Timestamp { .. }, DataType::Timestamp(_))
- | (datum @ Datum::LocalZonedTimestamp { .. }, DataType::LocalZonedTimestamp(_))
- | (datum @ Datum::Decimal { .. }, DataType::Decimal(_)) => Some(datum),
- (Datum::TinyInt(value), DataType::SmallInt(_)) => Some(Datum::SmallInt(value as i16)),
- (Datum::TinyInt(value), DataType::Int(_)) => Some(Datum::Int(value as i32)),
- (Datum::TinyInt(value), DataType::BigInt(_)) => Some(Datum::Long(value as i64)),
- (Datum::SmallInt(value), DataType::Int(_)) => Some(Datum::Int(value as i32)),
- (Datum::SmallInt(value), DataType::BigInt(_)) => Some(Datum::Long(value as i64)),
- (Datum::Int(value), DataType::BigInt(_)) => Some(Datum::Long(value as i64)),
- (Datum::Float(value), DataType::Double(_)) => Some(Datum::Double(value as f64)),
- _ => None,
- }
-}
-
-fn extract_stats_datum(row: &BinaryRow, index: usize, data_type: &DataType) -> Option<Datum> {
- let min_row_len = BinaryRow::cal_fix_part_size_in_bytes(row.arity()) as usize;
- if index >= row.arity() as usize || row.data().len() < min_row_len {
- return None;
- }
-
- match extract_datum(row, index, data_type) {
- Ok(Some(datum)) => Some(datum),
- Ok(None) | Err(_) => None,
- }
-}
-
/// TableScan for full table scan (no incremental, no predicate).
///
/// Reference: [pypaimon.read.table_scan.TableScan](https://github.com/apache/paimon/blob/master/paimon-python/pypaimon/read/table_scan.py)
@@ -755,13 +415,8 @@ impl<'a> TableScan<'a> {
let data_evolution_enabled = core_options.data_evolution_enabled();
let target_split_size = core_options.source_split_target_size();
let open_file_cost = core_options.source_split_open_file_cost();
- let entries = read_all_manifest_entries(file_io, table_path, &snapshot).await?;
- let entries = filter_manifest_entries(entries, deletion_vectors_enabled);
- let entries = merge_manifest_entries(entries);
- if entries.is_empty() {
- return Ok(Plan::new(Vec::new()));
- }
+ // Compute predicates before reading manifests so they can be pushed down.
let partition_keys = self.table.schema().partition_keys();
let (partition_predicate, data_predicates) = if let Some(filter) = self.filter.clone() {
if partition_keys.is_empty() {
@@ -777,52 +432,125 @@ impl<'a> TableScan<'a> {
(None, Vec::new())
};
- let entries = if let Some(ref pred) = partition_predicate {
- let mut kept = Vec::with_capacity(entries.len());
- let mut cache: HashMap<Vec<u8>, bool> = HashMap::new();
- for e in entries {
- let accept = match cache.get(e.partition()) {
- Some(&cached) => cached,
- None => {
- let partition_bytes = e.partition();
- let accept = partition_matches_predicate(partition_bytes, pred)?;
- cache.insert(partition_bytes.to_vec(), accept);
- accept
- }
- };
- if accept {
- kept.push(e);
- }
- }
- kept
+ // Resolve partition fields for manifest-file-level stats pruning.
+ let partition_fields: Vec<DataField> = partition_keys
+ .iter()
+ .filter_map(|key| {
+ self.table
+ .schema()
+ .fields()
+ .iter()
+ .find(|f| f.name() == key)
+ .cloned()
+ })
+ .collect();
+
+ // Data-evolution tables must not prune data files independently.
+ let pushdown_data_predicates = if data_evolution_enabled {
+ &[][..]
} else {
- entries
+ &data_predicates
};
+
+ let has_primary_keys = !self.table.schema().primary_keys().is_empty();
+
+ // Compute bucket predicate and key fields for per-entry bucket pruning.
+ // Only supported for the default bucket function (MurmurHash3-based).
+ let (bucket_predicate, bucket_key_fields): (Option<Predicate>, Vec<DataField>) =
+ if !core_options.is_default_bucket_function() {
+ (None, Vec::new())
+ } else if let Some(filter) = &self.filter {
+ let bucket_keys = core_options.bucket_key().unwrap_or_else(|| {
+ if has_primary_keys {
+ self.table
+ .schema()
+ .primary_keys()
+ .iter()
+ .map(|s| s.to_string())
+ .collect()
+ } else {
+ Vec::new()
+ }
+ });
+ if bucket_keys.is_empty() {
+ (None, Vec::new())
+ } else {
+ let fields: Vec<DataField> = bucket_keys
+ .iter()
+ .filter_map(|key| {
+ self.table
+ .schema()
+ .fields()
+ .iter()
+ .find(|f| f.name() == key)
+ .cloned()
+ })
+ .collect();
+ if fields.len() == bucket_keys.len() {
+ let pred = extract_predicate_for_keys(
+ filter,
+ self.table.schema().fields(),
+ &bucket_keys,
+ );
+ (pred, fields)
+ } else {
+ (None, Vec::new())
+ }
+ }
+ } else {
+ (None, Vec::new())
+ };
+
+ let entries = read_all_manifest_entries(
+ file_io,
+ table_path,
+ &snapshot,
+ deletion_vectors_enabled,
+ has_primary_keys,
+ partition_predicate.as_ref(),
+ &partition_fields,
+ pushdown_data_predicates,
+ self.table.schema().id(),
+ self.table.schema().fields(),
+ bucket_predicate.as_ref(),
+ &bucket_key_fields,
+ )
+ .await?;
+ let entries = merge_manifest_entries(entries);
if entries.is_empty() {
return Ok(Plan::new(Vec::new()));
}
- // Data-evolution tables can spread one logical row across multiple files with
- // different column sets. Pruning files independently would split merge groups,
- // so keep the current fail-open behavior until we support group-aware pruning.
+ // For non-data-evolution tables, cross-schema files were kept (fail-open)
+ // by the pushdown. Apply the full schema-aware filter for those files.
let entries = if data_predicates.is_empty() || data_evolution_enabled {
entries
} else {
- let mut kept = Vec::with_capacity(entries.len());
- let mut schema_cache: HashMap<i64, Option<Arc<ResolvedStatsSchema>>> = HashMap::new();
- for entry in entries {
- if data_file_matches_predicates_for_table(
- self.table,
- entry.file(),
- &data_predicates,
- &mut schema_cache,
- )
- .await
- {
- kept.push(entry);
+ let current_schema_id = self.table.schema().id();
+ let has_cross_schema = entries
+ .iter()
+ .any(|e| e.file().schema_id != current_schema_id);
+ if !has_cross_schema {
+ entries
+ } else {
+ let mut kept = Vec::with_capacity(entries.len());
+ let mut schema_cache: HashMap<i64, Option<Arc<ResolvedStatsSchema>>> =
+ HashMap::new();
+ for entry in entries {
+ if entry.file().schema_id == current_schema_id
+ || data_file_matches_predicates_for_table(
+ self.table,
+ entry.file(),
+ &data_predicates,
+ &mut schema_cache,
+ )
+ .await
+ {
+ kept.push(entry);
+ }
}
+ kept
}
- kept
};
if entries.is_empty() {
return Ok(Plan::new(Vec::new()));
@@ -892,8 +620,26 @@ impl<'a> TableScan<'a> {
// Data-evolution tables merge overlapping row-id groups column-wise during read.
// Keep that split boundary intact and only bin-pack single-file groups.
+ // Apply group-level predicate filtering after grouping by row_id range.
let file_groups_with_raw: Vec<(Vec<DataFileMeta>, bool)> = if data_evolution_enabled {
let row_id_groups = group_by_overlapping_row_id(data_files);
+
+ // Filter groups by merged stats before splitting.
+ let row_id_groups: Vec<Vec<DataFileMeta>> = if data_predicates.is_empty() {
+ row_id_groups
+ } else {
+ row_id_groups
+ .into_iter()
+ .filter(|group| {
+ data_evolution_group_matches_predicates(
+ group,
+ &data_predicates,
+ self.table.schema().fields(),
+ )
+ })
+ .collect()
+ };
+
let (singles, multis): (Vec<_>, Vec<_>) = row_id_groups
.into_iter()
.partition(|group| group.len() == 1);
@@ -939,8 +685,14 @@ impl<'a> TableScan<'a> {
}
}
- // Apply limit pushdown to reduce the number of splits if possible
- let splits = self.apply_limit_pushdown(splits);
+ // Apply limit pushdown only when there are no data predicates.
+ // With data predicates, merged_row_count() reflects pre-filter row counts,
+ // so stopping early could return fewer rows than the limit after filtering.
+ let splits = if data_predicates.is_empty() {
+ self.apply_limit_pushdown(splits)
+ } else {
+ splits
+ };
Ok(Plan::new(splits))
}
@@ -948,15 +700,15 @@ impl<'a> TableScan<'a> {
#[cfg(test)]
mod tests {
- use super::{
- data_file_matches_predicates, group_by_overlapping_row_id, partition_matches_predicate,
- };
+ use super::partition_matches_predicate;
use crate::spec::{
- stats::BinaryTableStats, ArrayType, DataField, DataFileMeta, DataType, Datum,
- DeletionVectorMeta, FileKind, IndexFileMeta, IndexManifestEntry, IntType, Predicate,
+ stats::BinaryTableStats, ArrayType, BinaryRowBuilder, DataField, DataFileMeta, DataType,
+ Datum, DeletionVectorMeta, FileKind, IndexFileMeta, IndexManifestEntry, IntType, Predicate,
PredicateBuilder, PredicateOperator, VarCharType,
};
+ use crate::table::bucket_filter::{compute_target_buckets, extract_predicate_for_keys};
use crate::table::source::DeletionFile;
+ use crate::table::stats_filter::{data_file_matches_predicates, group_by_overlapping_row_id};
use crate::Error;
use chrono::{DateTime, Utc};
@@ -981,12 +733,14 @@ mod tests {
schema_id: 0,
level: 0,
extra_files: Vec::new(),
- creation_time: DateTime::<Utc>::from_timestamp(0, 0).unwrap(),
+ creation_time: DateTime::<Utc>::from_timestamp(0, 0),
delete_row_count: None,
embedded_index: None,
first_row_id,
write_cols: None,
external_path: None,
+ file_source: None,
+ value_stats_cols: None,
}
}
@@ -997,86 +751,13 @@ mod tests {
.collect()
}
- struct SerializedBinaryRowBuilder {
- arity: i32,
- null_bits_size: usize,
- data: Vec<u8>,
- }
-
- impl SerializedBinaryRowBuilder {
- fn new(arity: i32) -> Self {
- let null_bits_size = crate::spec::BinaryRow::cal_bit_set_width_in_bytes(arity) as usize;
- let fixed_part_size = null_bits_size + (arity as usize) * 8;
- Self {
- arity,
- null_bits_size,
- data: vec![0u8; fixed_part_size],
- }
- }
-
- fn field_offset(&self, pos: usize) -> usize {
- self.null_bits_size + pos * 8
- }
-
- fn write_string(&mut self, pos: usize, value: &str) {
- let var_offset = self.data.len();
- self.data.extend_from_slice(value.as_bytes());
- let encoded = ((var_offset as u64) << 32) | (value.len() as u64);
- let offset = self.field_offset(pos);
- self.data[offset..offset + 8].copy_from_slice(&encoded.to_le_bytes());
- }
-
- fn build_serialized(self) -> Vec<u8> {
- let mut serialized = Vec::with_capacity(4 + self.data.len());
- serialized.extend_from_slice(&self.arity.to_be_bytes());
- serialized.extend_from_slice(&self.data);
- serialized
- }
- }
-
- struct RawBinaryRowBuilder {
- arity: i32,
- null_bits_size: usize,
- data: Vec<u8>,
- }
-
- impl RawBinaryRowBuilder {
- fn new(arity: i32) -> Self {
- let null_bits_size = crate::spec::BinaryRow::cal_bit_set_width_in_bytes(arity) as usize;
- let fixed_part_size = null_bits_size + (arity as usize) * 8;
- Self {
- arity,
- null_bits_size,
- data: vec![0u8; fixed_part_size],
- }
- }
-
- fn field_offset(&self, pos: usize) -> usize {
- self.null_bits_size + pos * 8
- }
-
- fn set_null_at(&mut self, pos: usize) {
- let bit_index = pos + crate::spec::BinaryRow::HEADER_SIZE_IN_BYTES as usize;
- let byte_index = bit_index / 8;
- let bit_offset = bit_index % 8;
- self.data[byte_index] |= 1 << bit_offset;
-
- let offset = self.field_offset(pos);
- self.data[offset..offset + 8].fill(0);
- }
-
- fn write_int(&mut self, pos: usize, value: i32) {
- let offset = self.field_offset(pos);
- self.data[offset..offset + 4].copy_from_slice(&value.to_le_bytes());
- }
-
- fn build(self) -> Vec<u8> {
- debug_assert_eq!(
- self.data.len(),
- self.null_bits_size + (self.arity as usize) * 8
- );
- self.data
+ fn int_stats_row(value: Option<i32>) -> Vec<u8> {
+ let mut builder = BinaryRowBuilder::new(1);
+ match value {
+ Some(value) => builder.write_int(0, value),
+ None => builder.set_null_at(0),
}
+ builder.build_serialized()
}
fn partition_string_field() -> Vec<DataField> {
@@ -1095,23 +776,10 @@ mod tests {
)]
}
- fn int_stats_row(value: Option<i32>) -> Vec<u8> {
- let mut builder = RawBinaryRowBuilder::new(1);
- match value {
- Some(value) => builder.write_int(0, value),
- None => builder.set_null_at(0),
- }
- let raw = builder.build();
- let mut serialized = Vec::with_capacity(4 + raw.len());
- serialized.extend_from_slice(&(1_i32).to_be_bytes());
- serialized.extend_from_slice(&raw);
- serialized
- }
-
fn test_data_file_meta(
min_values: Vec<u8>,
max_values: Vec<u8>,
- null_counts: Vec<i64>,
+ null_counts: Vec<Option<i64>>,
row_count: i64,
) -> DataFileMeta {
test_data_file_meta_with_schema(
@@ -1126,7 +794,7 @@ mod tests {
fn test_data_file_meta_with_schema(
min_values: Vec<u8>,
max_values: Vec<u8>,
- null_counts: Vec<i64>,
+ null_counts: Vec<Option<i64>>,
row_count: i64,
schema_id: i64,
) -> DataFileMeta {
@@ -1143,12 +811,14 @@ mod tests {
schema_id,
level: 1,
extra_files: Vec::new(),
- creation_time: Utc::now(),
+ creation_time: Some(Utc::now()),
delete_row_count: None,
embedded_index: None,
first_row_id: None,
write_cols: None,
external_path: None,
+ file_source: None,
+ value_stats_cols: None,
}
}
@@ -1163,7 +833,7 @@ mod tests {
#[test]
fn test_partition_matches_predicate_eval_error_fails_fast() {
- let mut builder = SerializedBinaryRowBuilder::new(1);
+ let mut builder = BinaryRowBuilder::new(1);
builder.write_string(0, "2024-01-01");
let serialized = builder.build_serialized();
@@ -1185,7 +855,9 @@ mod tests {
}
const TEST_SCHEMA_ID: i64 = 0;
- const TEST_NUM_FIELDS: usize = 1;
+ fn test_schema_fields() -> Vec<DataField> {
+ int_field()
+ }
#[test]
fn test_group_by_overlapping_row_id_empty() {
@@ -1266,8 +938,12 @@ mod tests {
#[test]
fn test_data_file_matches_eq_prunes_out_of_range() {
let fields = int_field();
- let file =
- test_data_file_meta(int_stats_row(Some(10)), int_stats_row(Some(20)), vec![0], 5);
+ let file = test_data_file_meta(
+ int_stats_row(Some(10)),
+ int_stats_row(Some(20)),
+ vec![Some(0)],
+ 5,
+ );
let predicate = PredicateBuilder::new(&fields)
.equal("id", Datum::Int(30))
.unwrap();
@@ -1276,44 +952,52 @@ mod tests {
&file,
&[predicate],
TEST_SCHEMA_ID,
- TEST_NUM_FIELDS,
+ &test_schema_fields(),
));
}
#[test]
fn test_data_file_matches_is_null_prunes_when_null_count_is_zero() {
let fields = int_field();
- let file =
- test_data_file_meta(int_stats_row(Some(10)), int_stats_row(Some(20)), vec![0], 5);
+ let file = test_data_file_meta(
+ int_stats_row(Some(10)),
+ int_stats_row(Some(20)),
+ vec![Some(0)],
+ 5,
+ );
let predicate = PredicateBuilder::new(&fields).is_null("id").unwrap();
assert!(!data_file_matches_predicates(
&file,
&[predicate],
TEST_SCHEMA_ID,
- TEST_NUM_FIELDS,
+ &test_schema_fields(),
));
}
#[test]
fn test_data_file_matches_is_not_null_prunes_all_null_file() {
let fields = int_field();
- let file = test_data_file_meta(int_stats_row(None), int_stats_row(None), vec![5], 5);
+ let file = test_data_file_meta(int_stats_row(None), int_stats_row(None), vec![Some(5)], 5);
let predicate = PredicateBuilder::new(&fields).is_not_null("id").unwrap();
assert!(!data_file_matches_predicates(
&file,
&[predicate],
TEST_SCHEMA_ID,
- TEST_NUM_FIELDS,
+ &test_schema_fields(),
));
}
#[test]
fn test_data_file_matches_unsupported_predicate_fails_open() {
let fields = int_field();
- let file =
- test_data_file_meta(int_stats_row(Some(10)), int_stats_row(Some(20)), vec![0], 5);
+ let file = test_data_file_meta(
+ int_stats_row(Some(10)),
+ int_stats_row(Some(20)),
+ vec![Some(0)],
+ 5,
+ );
let pb = PredicateBuilder::new(&fields);
let predicate = Predicate::or(vec![
pb.less_than("id", Datum::Int(5)).unwrap(),
@@ -1324,14 +1008,14 @@ mod tests {
&file,
&[predicate],
TEST_SCHEMA_ID,
- TEST_NUM_FIELDS,
+ &test_schema_fields(),
));
}
#[test]
fn test_data_file_matches_corrupt_stats_fails_open() {
let fields = int_field();
- let file = test_data_file_meta(Vec::new(), Vec::new(), vec![0], 5);
+ let file = test_data_file_meta(Vec::new(), Vec::new(), vec![Some(0)], 5);
let predicate = PredicateBuilder::new(&fields)
.equal("id", Datum::Int(30))
.unwrap();
@@ -1340,7 +1024,7 @@ mod tests {
&file,
&[predicate],
TEST_SCHEMA_ID,
- TEST_NUM_FIELDS,
+ &test_schema_fields(),
));
}
@@ -1350,7 +1034,7 @@ mod tests {
let file = test_data_file_meta_with_schema(
int_stats_row(Some(10)),
int_stats_row(Some(20)),
- vec![0],
+ vec![Some(0)],
5,
5,
);
@@ -1362,41 +1046,7 @@ mod tests {
&file,
&[predicate],
TEST_SCHEMA_ID,
- TEST_NUM_FIELDS,
- ));
- }
-
- #[test]
- fn test_data_file_matches_dense_stats_arity_mismatch_fails_open() {
- let mut builder = RawBinaryRowBuilder::new(3);
- builder.write_int(0, 10);
- builder.write_int(1, 100);
- builder.write_int(2, 200);
- let raw = builder.build();
- let mut min_serialized = Vec::with_capacity(4 + raw.len());
- min_serialized.extend_from_slice(&(3_i32).to_be_bytes());
- min_serialized.extend_from_slice(&raw);
-
- let mut builder = RawBinaryRowBuilder::new(3);
- builder.write_int(0, 20);
- builder.write_int(1, 200);
- builder.write_int(2, 300);
- let raw = builder.build();
- let mut max_serialized = Vec::with_capacity(4 + raw.len());
- max_serialized.extend_from_slice(&(3_i32).to_be_bytes());
- max_serialized.extend_from_slice(&raw);
-
- let fields = int_field();
- let file = test_data_file_meta(min_serialized, max_serialized, vec![0, 0, 0], 5);
- let predicate = PredicateBuilder::new(&fields)
- .equal("id", Datum::Int(30))
- .unwrap();
-
- assert!(data_file_matches_predicates(
- &file,
- &[predicate],
- TEST_SCHEMA_ID,
- TEST_NUM_FIELDS,
+ &test_schema_fields(),
));
}
@@ -1405,7 +1055,7 @@ mod tests {
let file = test_data_file_meta_with_schema(
int_stats_row(Some(10)),
int_stats_row(Some(20)),
- vec![0],
+ vec![Some(0)],
5,
99,
);
@@ -1414,7 +1064,7 @@ mod tests {
&file,
&[Predicate::AlwaysFalse],
TEST_SCHEMA_ID,
- TEST_NUM_FIELDS,
+ &test_schema_fields(),
));
}
@@ -1423,7 +1073,7 @@ mod tests {
let file = test_data_file_meta_with_schema(
int_stats_row(Some(10)),
int_stats_row(Some(20)),
- vec![0],
+ vec![Some(0)],
5,
99,
);
@@ -1432,7 +1082,7 @@ mod tests {
&file,
&[Predicate::AlwaysTrue],
TEST_SCHEMA_ID,
- TEST_NUM_FIELDS,
+ &test_schema_fields(),
));
}
@@ -1473,4 +1123,200 @@ mod tests {
&DeletionFile::new("file:/tmp/table/index/index-file".into(), 11, 22, Some(33))
);
}
+
+ // ======================== Bucket predicate filtering ========================
+
+ fn bucket_key_fields() -> Vec<DataField> {
+ vec![DataField::new(
+ 0,
+ "id".to_string(),
+ DataType::Int(IntType::new()),
+ )]
+ }
+
+ #[test]
+ fn test_extract_predicate_for_keys_eq() {
+ let fields = vec![
+ DataField::new(0, "id".to_string(), DataType::Int(IntType::new())),
+ DataField::new(
+ 1,
+ "name".to_string(),
+ DataType::VarChar(VarCharType::default()),
+ ),
+ ];
+ let pb = PredicateBuilder::new(&fields);
+ let filter = Predicate::and(vec![
+ pb.equal("id", Datum::Int(42)).unwrap(),
+ pb.equal("name", Datum::String("alice".into())).unwrap(),
+ ]);
+
+ let keys = vec!["id".to_string()];
+ let extracted = extract_predicate_for_keys(&filter, &fields, &keys);
+ assert!(extracted.is_some());
+ match extracted.unwrap() {
+ Predicate::Leaf {
+ column, index, op, ..
+ } => {
+ assert_eq!(column, "id");
+ assert_eq!(index, 0); // remapped to key index
+ assert_eq!(op, PredicateOperator::Eq);
+ }
+ other => panic!("expected Leaf, got {other:?}"),
+ }
+ }
+
+ #[test]
+ fn test_extract_predicate_for_keys_no_match() {
+ let fields = vec![
+ DataField::new(0, "id".to_string(), DataType::Int(IntType::new())),
+ DataField::new(
+ 1,
+ "name".to_string(),
+ DataType::VarChar(VarCharType::default()),
+ ),
+ ];
+ let pb = PredicateBuilder::new(&fields);
+ let filter = pb.equal("name", Datum::String("alice".into())).unwrap();
+
+ let keys = vec!["id".to_string()];
+ let extracted = extract_predicate_for_keys(&filter, &fields, &keys);
+ assert!(extracted.is_none());
+ }
+
+ #[test]
+ fn test_compute_target_buckets_single_eq() {
+ let fields = bucket_key_fields();
+ // Build a bucket predicate (already projected to bucket key space, index=0)
+ let pred = Predicate::Leaf {
+ column: "id".into(),
+ index: 0,
+ data_type: DataType::Int(IntType::new()),
+ op: PredicateOperator::Eq,
+ literals: vec![Datum::Int(42)],
+ };
+
+ let buckets = compute_target_buckets(&pred, &fields, 4);
+ assert!(buckets.is_some());
+ let buckets = buckets.unwrap();
+ assert_eq!(buckets.len(), 1);
+ // The bucket should be deterministic
+ let bucket = *buckets.iter().next().unwrap();
+ assert!((0..4).contains(&bucket));
+ }
+
+ #[test]
+ fn test_compute_target_buckets_in_predicate() {
+ let fields = bucket_key_fields();
+ let pred = Predicate::Leaf {
+ column: "id".into(),
+ index: 0,
+ data_type: DataType::Int(IntType::new()),
+ op: PredicateOperator::In,
+ literals: vec![Datum::Int(1), Datum::Int(2), Datum::Int(3)],
+ };
+
+ let buckets = compute_target_buckets(&pred, &fields, 4);
+ assert!(buckets.is_some());
+ let buckets = buckets.unwrap();
+ // Should have at most 3 buckets (could be fewer if some hash to the same bucket)
+ assert!(!buckets.is_empty());
+ assert!(buckets.len() <= 3);
+ for &b in &buckets {
+ assert!((0..4).contains(&b));
+ }
+ }
+
+ #[test]
+ fn test_compute_target_buckets_range_returns_none() {
+ let fields = bucket_key_fields();
+ let pred = Predicate::Leaf {
+ column: "id".into(),
+ index: 0,
+ data_type: DataType::Int(IntType::new()),
+ op: PredicateOperator::Gt,
+ literals: vec![Datum::Int(10)],
+ };
+
+ let buckets = compute_target_buckets(&pred, &fields, 4);
+ assert!(
+ buckets.is_none(),
+ "Range predicates cannot determine target buckets"
+ );
+ }
+
+ #[test]
+ fn test_compute_target_buckets_composite_key() {
+ let fields = vec![
+ DataField::new(0, "a".to_string(), DataType::Int(IntType::new())),
+ DataField::new(1, "b".to_string(), DataType::Int(IntType::new())),
+ ];
+ let pred = Predicate::And(vec![
+ Predicate::Leaf {
+ column: "a".into(),
+ index: 0,
+ data_type: DataType::Int(IntType::new()),
+ op: PredicateOperator::Eq,
+ literals: vec![Datum::Int(1)],
+ },
+ Predicate::Leaf {
+ column: "b".into(),
+ index: 1,
+ data_type: DataType::Int(IntType::new()),
+ op: PredicateOperator::Eq,
+ literals: vec![Datum::Int(2)],
+ },
+ ]);
+
+ let buckets = compute_target_buckets(&pred, &fields, 8);
+ assert!(buckets.is_some());
+ let buckets = buckets.unwrap();
+ assert_eq!(buckets.len(), 1);
+ let bucket = *buckets.iter().next().unwrap();
+ assert!((0..8).contains(&bucket));
+ }
+
+ #[test]
+ fn test_compute_target_buckets_partial_key_returns_none() {
+ // Only one of two bucket key fields has an eq predicate
+ let fields = vec![
+ DataField::new(0, "a".to_string(), DataType::Int(IntType::new())),
+ DataField::new(1, "b".to_string(), DataType::Int(IntType::new())),
+ ];
+ let pred = Predicate::Leaf {
+ column: "a".into(),
+ index: 0,
+ data_type: DataType::Int(IntType::new()),
+ op: PredicateOperator::Eq,
+ literals: vec![Datum::Int(1)],
+ };
+
+ let buckets = compute_target_buckets(&pred, &fields, 8);
+ assert!(
+ buckets.is_none(),
+ "Partial bucket key should not determine target buckets"
+ );
+ }
+
+ #[test]
+ fn test_compute_target_buckets_string_key() {
+ let fields = vec![DataField::new(
+ 0,
+ "name".to_string(),
+ DataType::VarChar(VarCharType::default()),
+ )];
+ let pred = Predicate::Leaf {
+ column: "name".into(),
+ index: 0,
+ data_type: DataType::VarChar(VarCharType::default()),
+ op: PredicateOperator::Eq,
+ literals: vec![Datum::String("alice".into())],
+ };
+
+ let buckets = compute_target_buckets(&pred, &fields, 4);
+ assert!(buckets.is_some());
+ let buckets = buckets.unwrap();
+ assert_eq!(buckets.len(), 1);
+ let bucket = *buckets.iter().next().unwrap();
+ assert!((0..4).contains(&bucket));
+ }
}
diff --git a/dev/spark/README.md b/dev/spark/README.md
new file mode 100644
index 0000000..f125c1f
--- /dev/null
+++ b/dev/spark/README.md
@@ -0,0 +1,41 @@
+# Spark Provisioning for Integration Tests
+
+This directory contains the Spark + Paimon setup that provisions test tables into `/tmp/paimon-warehouse`.
+
+## Prerequisites
+
+- Docker (via colima or Docker Desktop)
+- `docker compose`
+
+## Provision Test Data
+
+```bash
+# Build and run (from repo root):
+make docker-up
+
+# Or manually:
+docker compose -f dev/docker-compose.yaml build --build-arg PAIMON_VERSION=1.3.1
+docker compose -f dev/docker-compose.yaml run --rm spark-paimon
+```
+
+`provision.py` automatically clears the warehouse directory before creating tables, so re-running is always safe.
+
+### colima Users
+
+With colima, Docker volumes mount inside the colima VM, **not** on the macOS host filesystem. After provisioning, copy the data to the host:
+
+```bash
+colima ssh -- sudo tar cf - -C /tmp paimon-warehouse | tar xf - -C /tmp
+```
+
+## Run Integration Tests
+
+```bash
+cargo test -p paimon-integration-tests
+```
+
+## Files
+
+- `Dockerfile` — Spark 3.5 + Paimon connector image
+- `spark-defaults.conf` — Spark config with Paimon catalog
+- `provision.py` — Creates all test tables and inserts data
diff --git a/dev/spark/provision.py b/dev/spark/provision.py
index c898e58..d782d46 100644
--- a/dev/spark/provision.py
+++ b/dev/spark/provision.py
@@ -573,6 +573,143 @@ def main():
"""
)
+ # ===== Non-PK table with deletion vectors enabled =====
+ # Append-only table with DV: level-0 files should NOT be filtered out
+ # because there is no primary key merge.
+ spark.sql(
+ """
+ CREATE TABLE IF NOT EXISTS simple_dv_log_table (
+ id INT,
+ name STRING
+ ) USING paimon
+ TBLPROPERTIES (
+ 'deletion-vectors.enabled' = 'true'
+ )
+ """
+ )
+ spark.sql(
+ """
+ INSERT INTO simple_dv_log_table VALUES
+ (1, 'alice'),
+ (2, 'bob'),
+ (3, 'carol')
+ """
+ )
+
+ # ===== Postpone bucket PK table (bucket = -2) =====
+ # New data lands in bucket-postpone and is NOT visible to readers until compacted.
+ # Without running compaction, the table should appear empty to batch readers.
+ spark.sql(
+ """
+ CREATE TABLE IF NOT EXISTS postpone_bucket_pk_table (
+ id INT,
+ name STRING
+ ) USING paimon
+ TBLPROPERTIES (
+ 'primary-key' = 'id',
+ 'bucket' = '-2',
+ 'deletion-vectors.enabled' = 'true'
+ )
+ """
+ )
+ spark.sql(
+ """
+ INSERT INTO postpone_bucket_pk_table VALUES
+ (1, 'alice'),
+ (2, 'bob'),
+ (3, 'carol')
+ """
+ )
+
+
+ # ===== Multi-bucket PK table for bucket predicate filtering tests =====
+ # PK table with bucket=4 so data distributes across multiple buckets.
+ # Bucket key defaults to primary key (id). Used to test that bucket predicate
+ # filtering correctly computes target buckets from equality predicates.
+ spark.sql(
+ """
+ CREATE TABLE IF NOT EXISTS multi_bucket_pk_table (
+ id INT,
+ name STRING
+ ) USING paimon
+ TBLPROPERTIES (
+ 'primary-key' = 'id',
+ 'bucket' = '4',
+ 'deletion-vectors.enabled' = 'true'
+ )
+ """
+ )
+ spark.sql(
+ """
+ INSERT INTO multi_bucket_pk_table VALUES
+ (1, 'alice'),
+ (2, 'bob'),
+ (3, 'carol'),
+ (4, 'dave'),
+ (5, 'eve'),
+ (6, 'frank'),
+ (7, 'grace'),
+ (8, 'heidi')
+ """
+ )
+
+ # ===== String bucket key tables for variable-length hash tests =====
+ # Short string keys (<=7 bytes) use inline encoding in BinaryRow.
+ spark.sql(
+ """
+ CREATE TABLE IF NOT EXISTS string_bucket_short_key (
+ code STRING,
+ value INT
+ ) USING paimon
+ TBLPROPERTIES (
+ 'primary-key' = 'code',
+ 'bucket' = '4',
+ 'deletion-vectors.enabled' = 'true'
+ )
+ """
+ )
+ spark.sql(
+ """
+ INSERT INTO string_bucket_short_key VALUES
+ ('aaa', 1),
+ ('bbb', 2),
+ ('ccc', 3),
+ ('ddd', 4),
+ ('eee', 5),
+ ('fff', 6),
+ ('ggg', 7),
+ ('hhh', 8)
+ """
+ )
+
+ # Long string keys (>7 bytes) use variable-length encoding in BinaryRow.
+ spark.sql(
+ """
+ CREATE TABLE IF NOT EXISTS string_bucket_long_key (
+ code STRING,
+ value INT
+ ) USING paimon
+ TBLPROPERTIES (
+ 'primary-key' = 'code',
+ 'bucket' = '4',
+ 'deletion-vectors.enabled' = 'true'
+ )
+ """
+ )
+ spark.sql(
+ """
+ INSERT INTO string_bucket_long_key VALUES
+ ('alpha-long-key', 1),
+ ('bravo-long-key', 2),
+ ('charlie-long-key', 3),
+ ('delta-long-key', 4),
+ ('echo-long-key', 5),
+ ('foxtrot-long-key', 6),
+ ('golf-long-key', 7),
+ ('hotel-long-key', 8)
+ """
+ )
+
if __name__ == "__main__":
main()