You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by mb...@apache.org on 2026/09/09 19:17:27 UTC
(asterixdb) branch master updated: [ASTERIXDB-3702][STO] Miscellaneous improvements for Column Sampler
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 78e55e657d [ASTERIXDB-3702][STO] Miscellaneous improvements for Column Sampler
78e55e657d is described below
commit 78e55e657d048c7da491f4df9bda90fabf61942d
Author: Ritik Raj <ra...@gmail.com>
AuthorDate: Wed Aug 19 15:55:45 2026 +0530
[ASTERIXDB-3702][STO] Miscellaneous improvements for Column Sampler
- user model changes: no
- storage format changes: no
- interface changes: yes
Details:
- ColumnBtreeSampleCursor: fix a spatially biased sample caused by
truncating a sorted draw batch at the target, draw intra-page slots
without replacement, and share one primary key pass across all draws
that land on the same page.
- Make the newer-component liveness probe existence-only: a zero-column
projection through a new ColumnBTreeExistencePointSearchCursor and
LSMColumnSampleLivenessSearchCursor, wired only where the sample
collector builds its cursor so the query path is untouched. The cloud
read context pins only what that projection needs.
- Clamp columnar samples-per-page to 1: drawing several tuples from one
page clusters the sample and moves chi-square well past its critical
value.
Ext-ref: MB-73172
Co-Authored-By: Claude Opus 4.8 <no...@anthropic.com>
Change-Id: I531e3a4be1b5d91379319130463846b9ec8ee1bf
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21579
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Ritik Raj <ri...@couchbase.com>
Reviewed-by: Murtadha Hubail <mh...@apache.org>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
---
.../btree-index-range/btree-index-range.06.plan | 48 +-
.../tpch/query-plans/query-plans.08.plan | 128 ++--
.../tpch/query-plans/query-plans.09.plan | 194 +++---
.../tpch/query-plans/query-plans.10.plan | 100 +--
.../tpch/query-plans/query-plans.11.plan | 142 ++---
asterixdb/asterix-column/pom.xml | 12 +
.../lsm/merge/MergeColumnReadMetadata.java | 44 +-
.../operation/query/QueryColumnMetadata.java | 8 +
.../CloudExistenceProjectionPinningTest.java | 280 +++++++++
.../test/sample/ColumnSampleBenchHarness.java | 697 +++++++++++++++++++++
.../ColumnSampleBenchHarnessResidencyTest.java | 123 ++++
.../test/sample/ColumnSampleBenchHarnessTest.java | 116 ++++
.../test/sample/ColumnSampleCostBaselineTest.java | 103 +++
.../sample/ColumnSampleLivenessWiringTest.java | 176 ++++++
.../test/sample/ColumnSampleUniformityTest.java | 158 +++++
.../asterix/common/config/StorageProperties.java | 35 +-
.../apache/asterix/metadata/utils/DatasetUtil.java | 2 +-
.../hyracks/storage/am/btree/impls/DiskBTree.java | 2 +-
.../am/btree/impls/DiskBTreeSampleCursor.java | 292 +++++----
.../column/api/projection/ColumnProjectorType.java | 28 +-
.../api/projection/IColumnProjectionInfo.java | 23 +
.../buffercache/read/CloudColumnReadContext.java | 12 +-
.../buffercache/read/CloudMegaPageReadContext.java | 4 +
.../lsm/btree/column/impls/btree/ColumnBTree.java | 13 +-
.../ColumnBTreeExistencePointSearchCursor.java | 105 ++++
.../impls/btree/ColumnBtreeSampleCursor.java | 514 +++++++++------
.../column/impls/btree/SampleCursorStats.java | 67 ++
.../lsm/btree/column/impls/lsm/LSMColumnBTree.java | 6 +-
.../lsm/LSMColumnSampleLivenessSearchCursor.java | 63 ++
.../BTreeSampleCollectorOperatorDescriptor.java | 7 +-
...pleCollectorOperatorDescriptorNodePushable.java | 5 +-
.../storage/am/lsm/btree/impls/LSMBTree.java | 6 +-
.../am/lsm/btree/impls/LSMIndexSampleCursor.java | 5 +-
.../hyracks-storage-am-lsm-btree-test/pom.xml | 6 +
.../lsm/btree/DiskBTreeSampleUniformityTest.java | 354 +++++++++++
.../LSMSampleFragmentationContaminationTest.java | 6 +-
.../am/lsm/btree/SampleCursorStatsTest.java | 69 ++
37 files changed, 3281 insertions(+), 672 deletions(-)
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results_cbo/index-selection/btree-index-range/btree-index-range.06.plan b/asterixdb/asterix-app/src/test/resources/runtimets/results_cbo/index-selection/btree-index-range/btree-index-range.06.plan
index a4aff1ff4a..08b534b336 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results_cbo/index-selection/btree-index-range/btree-index-range.06.plan
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results_cbo/index-selection/btree-index-range/btree-index-range.06.plan
@@ -1,26 +1,34 @@
-distribute result [$$51] [cardinality: 1569.97, doc-size: 0.0, op-cost: 0.0, total-cost: 11836.0]
+distribute result [$$51] [cardinality: 746.01, doc-size: 0.0, op-cost: 0.0, total-cost: 37265.42]
-- DISTRIBUTE_RESULT |UNPARTITIONED|
- exchange [cardinality: 1569.97, doc-size: 0.0, op-cost: 0.0, total-cost: 11836.0]
+ exchange [cardinality: 746.01, doc-size: 0.0, op-cost: 0.0, total-cost: 37265.42]
-- ONE_TO_ONE_EXCHANGE |UNPARTITIONED|
- assign [$$51] <- [{"$1": $$54}] project: [$$51] [cardinality: 1569.97, doc-size: 0.0, op-cost: 0.0, total-cost: 11836.0]
+ assign [$$51] <- [{"$1": $$54}] project: [$$51] [cardinality: 746.01, doc-size: 0.0, op-cost: 0.0, total-cost: 37265.42]
-- ASSIGN |UNPARTITIONED|
- aggregate [$$54] <- [agg-sql-sum($$58)] [cardinality: 1569.97, doc-size: 0.0, op-cost: 0.0, total-cost: 11836.0]
+ aggregate [$$54] <- [agg-sql-sum($$58)] [cardinality: 746.01, doc-size: 0.0, op-cost: 0.0, total-cost: 37265.42]
-- AGGREGATE |UNPARTITIONED|
- exchange [cardinality: 1569.97, doc-size: 0.0, op-cost: 0.0, total-cost: 11836.0]
+ exchange [cardinality: 746.01, doc-size: 0.0, op-cost: 0.0, total-cost: 37265.42]
-- RANDOM_MERGE_EXCHANGE |PARTITIONED|
- aggregate [$$58] <- [agg-sql-count(1)] [cardinality: 1569.97, doc-size: 0.0, op-cost: 0.0, total-cost: 11836.0]
+ aggregate [$$58] <- [agg-sql-count(1)] [cardinality: 746.01, doc-size: 0.0, op-cost: 0.0, total-cost: 37265.42]
-- AGGREGATE |PARTITIONED|
- select (and(gt($$A.getField("c"), 2), le($$A.getField("d"), 6000), eq($$A.getField("a"), 0), le($$52, 3), ge($$52, 1))) [cardinality: 1569.97, doc-size: 0.0, op-cost: 0.0, total-cost: 11836.0]
- -- STREAM_SELECT |PARTITIONED|
- assign [$$52] <- [$$A.getField("b")] [cardinality: 11836.0, doc-size: 0.0, op-cost: 0.0, total-cost: 11836.0]
- -- ASSIGN |PARTITIONED|
- project ([$$A]) [cardinality: 11836.0, doc-size: 0.0, op-cost: 0.0, total-cost: 11836.0]
- -- STREAM_PROJECT |PARTITIONED|
- exchange [cardinality: 11836.0, doc-size: 0.0, op-cost: 0.0, total-cost: 11836.0]
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- data-scan []<-[$$53, $$A] <- test.A [cardinality: 11836.0, doc-size: 0.0, op-cost: 0.0, total-cost: 11836.0]
- -- DATASOURCE_SCAN |PARTITIONED|
- exchange [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- empty-tuple-source [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ exchange [cardinality: 746.01, doc-size: 0.0, op-cost: 0.0, total-cost: 37265.42]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ distinct ([$$68]) [cardinality: 746.01, doc-size: 0.0, op-cost: 37265.42, total-cost: 37265.42]
+ -- PRE_SORTED_DISTINCT_BY |PARTITIONED|
+ exchange [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ order (ASC, $$68) [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STABLE_SORT [$$68(ASC)] |PARTITIONED|
+ exchange [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ select (and(gt($$57, 2), le($$56, 6000), eq($$55, 0), le($$52, 3), ge($$52, 1))) project: [$$68] [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STREAM_SELECT |PARTITIONED|
+ exchange [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ unnest-map [$$55, $$52, $$57, $$56, $$68] <- index-search("idx_abcd", 0, "Default", "test", "A", false, false, 3, $$59, $$60, $$61, 2, $$62, $$63, true, true, true) [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- BTREE_SEARCH |PARTITIONED|
+ exchange [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ assign [$$59, $$60, $$61, $$62, $$63] <- [0, 1, 2, 0, 3] [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ASSIGN |PARTITIONED|
+ empty-tuple-source [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results_cbo/tpch/query-plans/query-plans.08.plan b/asterixdb/asterix-app/src/test/resources/runtimets/results_cbo/tpch/query-plans/query-plans.08.plan
index 3ad00c394b..5886909462 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results_cbo/tpch/query-plans/query-plans.08.plan
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results_cbo/tpch/query-plans/query-plans.08.plan
@@ -1,100 +1,100 @@
-distribute result [$$209] [cardinality: 2.45, doc-size: 68.16, op-cost: 0.0, total-cost: 399672.09]
+distribute result [$$209] [cardinality: 2.42, doc-size: 68.16, op-cost: 0.0, total-cost: 403489.89]
-- DISTRIBUTE_RESULT |PARTITIONED|
- exchange [cardinality: 2.45, doc-size: 68.16, op-cost: 0.0, total-cost: 399672.09]
+ exchange [cardinality: 2.42, doc-size: 68.16, op-cost: 0.0, total-cost: 403489.89]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- assign [$$209] <- [{"$1": 7, "supp_nation": $$supp_nation, "cust_nation": $$cust_nation, "l_year": $$l_year, "revenue": $$225}] project: [$$209] [cardinality: 2.45, doc-size: 68.16, op-cost: 0.0, total-cost: 399672.09]
+ assign [$$209] <- [{"$1": 7, "supp_nation": $$supp_nation, "cust_nation": $$cust_nation, "l_year": $$l_year, "revenue": $$225}] project: [$$209] [cardinality: 2.42, doc-size: 68.16, op-cost: 0.0, total-cost: 403489.89]
-- ASSIGN |PARTITIONED|
- exchange [cardinality: 2.45, doc-size: 68.16, op-cost: 0.0, total-cost: 399672.09]
+ exchange [cardinality: 2.42, doc-size: 68.16, op-cost: 0.0, total-cost: 403489.89]
-- SORT_MERGE_EXCHANGE [$$supp_nation(ASC), $$cust_nation(ASC), $$l_year(ASC) ] |PARTITIONED|
- order (ASC, $$supp_nation) (ASC, $$cust_nation) (ASC, $$l_year) [cardinality: 2.45, doc-size: 68.16, op-cost: 1.23, total-cost: 399672.09]
+ order (ASC, $$supp_nation) (ASC, $$cust_nation) (ASC, $$l_year) [cardinality: 2.42, doc-size: 68.16, op-cost: 1.21, total-cost: 403489.89]
-- STABLE_SORT [$$supp_nation(ASC), $$cust_nation(ASC), $$l_year(ASC)] |PARTITIONED|
- exchange [cardinality: 2.45, doc-size: 68.16, op-cost: 0.0, total-cost: 399670.86]
+ exchange [cardinality: 2.42, doc-size: 68.16, op-cost: 0.0, total-cost: 403488.68]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
group by ([$$supp_nation := $$240; $$cust_nation := $$241; $$l_year := $$242]) decor ([]) {
aggregate [$$225] <- [global-sql-sum-serial($$239)] [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
-- AGGREGATE |LOCAL|
nested tuple source [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
-- NESTED_TUPLE_SOURCE |LOCAL|
- } [cardinality: 2.45, doc-size: 68.16, op-cost: 12.25, total-cost: 399670.86]
+ } [cardinality: 2.42, doc-size: 68.16, op-cost: 12.1, total-cost: 403488.68]
-- EXTERNAL_GROUP_BY[$$240, $$241, $$242] |PARTITIONED|
- exchange [cardinality: 2.45, doc-size: 68.16, op-cost: 0.0, total-cost: 399658.61]
+ exchange [cardinality: 2.42, doc-size: 68.16, op-cost: 0.0, total-cost: 403476.58]
-- HASH_PARTITION_EXCHANGE [$$240, $$241, $$242] |PARTITIONED|
group by ([$$240 := $$214; $$241 := $$215; $$242 := $$237]) decor ([]) {
aggregate [$$239] <- [local-sql-sum-serial($$207)] [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
-- AGGREGATE |LOCAL|
nested tuple source [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
-- NESTED_TUPLE_SOURCE |LOCAL|
- } [cardinality: 2.45, doc-size: 68.16, op-cost: 102.16, total-cost: 399658.61]
+ } [cardinality: 2.42, doc-size: 68.16, op-cost: 143.67, total-cost: 403476.58]
-- EXTERNAL_GROUP_BY[$$214, $$215, $$237] |PARTITIONED|
- exchange [cardinality: 398.82, doc-size: 68.16, op-cost: 0.0, total-cost: 399556.45]
+ exchange [cardinality: 565.0, doc-size: 68.16, op-cost: 0.0, total-cost: 403332.91]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- assign [$$207, $$237] <- [numeric-multiply($$234, numeric-subtract(1, $$235)), get-year(date($$213))] project: [$$207, $$214, $$215, $$237] [cardinality: 398.82, doc-size: 68.16, op-cost: 0.0, total-cost: 399556.45]
+ assign [$$207, $$237] <- [numeric-multiply($$234, numeric-subtract(1, $$235)), get-year(date($$213))] project: [$$207, $$214, $$215, $$237] [cardinality: 565.0, doc-size: 68.16, op-cost: 0.0, total-cost: 403332.91]
-- ASSIGN |PARTITIONED|
- project ([$$214, $$215, $$234, $$235, $$213]) [cardinality: 398.82, doc-size: 68.16, op-cost: 0.0, total-cost: 399556.45]
+ project ([$$214, $$215, $$234, $$235, $$213]) [cardinality: 565.0, doc-size: 68.16, op-cost: 0.0, total-cost: 403332.91]
-- STREAM_PROJECT |PARTITIONED|
- exchange [cardinality: 398.82, doc-size: 68.16, op-cost: 0.0, total-cost: 399556.45]
+ exchange [cardinality: 565.0, doc-size: 68.16, op-cost: 0.0, total-cost: 403332.91]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- join (and(eq($$223, $$222), eq($$220, $$227))) [cardinality: 398.82, doc-size: 68.16, op-cost: 6294.24, total-cost: 399556.45]
- -- HYBRID_HASH_JOIN [$$222, $$227][$$223, $$220] |PARTITIONED|
- exchange [cardinality: 9961.23, doc-size: 58.16, op-cost: 0.0, total-cost: 379638.52]
- -- HASH_PARTITION_EXCHANGE [$$227] |PARTITIONED|
- project ([$$214, $$215, $$234, $$235, $$213, $$222, $$227]) [cardinality: 9961.23, doc-size: 58.16, op-cost: 0.0, total-cost: 379638.52]
+ join (and(eq($$228, $$221), eq($$216, $$233))) [cardinality: 565.0, doc-size: 68.16, op-cost: 4706.0, total-cost: 403332.91]
+ -- HYBRID_HASH_JOIN [$$221, $$233][$$228, $$216] |PARTITIONED|
+ exchange [cardinality: 14310.99, doc-size: 58.16, op-cost: 0.0, total-cost: 397141.71]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ project ([$$214, $$215, $$234, $$235, $$213, $$221, $$233]) [cardinality: 14310.99, doc-size: 58.16, op-cost: 0.0, total-cost: 397141.71]
-- STREAM_PROJECT |PARTITIONED|
- exchange [cardinality: 9961.23, doc-size: 58.16, op-cost: 0.0, total-cost: 379638.52]
+ exchange [cardinality: 14310.99, doc-size: 58.16, op-cost: 0.0, total-cost: 397141.71]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- join (eq($$219, $$218)) [cardinality: 9961.23, doc-size: 58.16, op-cost: 49742.98, total-cost: 379638.52]
- -- HYBRID_HASH_JOIN [$$219][$$218] |PARTITIONED|
- exchange [cardinality: 149063.0, doc-size: 10.0, op-cost: 0.0, total-cost: 74755.34]
+ join (eq($$219, $$218)) [cardinality: 14310.99, doc-size: 58.16, op-cost: 60189.25, total-cost: 397141.71]
+ -- HYBRID_HASH_JOIN [$$218][$$219] |PARTITIONED|
+ exchange [cardinality: 178694.28, doc-size: 10.0, op-cost: 0.0, total-cost: 195965.61]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- assign [$$227] <- [$$o.getField(1)] project: [$$227, $$219] [cardinality: 149063.0, doc-size: 10.0, op-cost: 0.0, total-cost: 74755.34]
- -- ASSIGN |PARTITIONED|
- exchange [cardinality: 149063.0, doc-size: 10.0, op-cost: 0.0, total-cost: 74755.34]
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- data-scan []<-[$$219, $$o] <- tpch.orders [cardinality: 149063.0, doc-size: 10.0, op-cost: 0.0, total-cost: 74755.34]
- -- DATASOURCE_SCAN |PARTITIONED|
- exchange [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
+ select (and(le($$213, "1996-12-31"), ge($$213, "1995-01-01"))) [cardinality: 178694.28, doc-size: 10.0, op-cost: 0.0, total-cost: 195965.61]
+ -- STREAM_SELECT |PARTITIONED|
+ assign [$$235, $$234, $$213, $$233] <- [$$l.getField(6), $$l.getField(5), $$l.getField(10), $$l.getField(2)] project: [$$218, $$235, $$234, $$213, $$233] [cardinality: 601114.0, doc-size: 10.0, op-cost: 0.0, total-cost: 195965.61]
+ -- ASSIGN |PARTITIONED|
+ project ([$$218, $$l]) [cardinality: 601114.0, doc-size: 10.0, op-cost: 0.0, total-cost: 195965.61]
+ -- STREAM_PROJECT |PARTITIONED|
+ exchange [cardinality: 601114.0, doc-size: 10.0, op-cost: 0.0, total-cost: 195965.61]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- empty-tuple-source [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
- exchange [cardinality: 9951.86, doc-size: 48.16, op-cost: 0.0, total-cost: 245188.34]
+ data-scan []<-[$$217, $$218, $$l] <- tpch.lineitem [cardinality: 601114.0, doc-size: 10.0, op-cost: 0.0, total-cost: 195965.61]
+ -- DATASOURCE_SCAN |PARTITIONED|
+ exchange [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ empty-tuple-source [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ exchange [cardinality: 11937.93, doc-size: 48.16, op-cost: 0.0, total-cost: 129048.93]
-- BROADCAST_EXCHANGE |PARTITIONED|
- project ([$$214, $$215, $$234, $$235, $$213, $$222, $$218]) [cardinality: 9951.86, doc-size: 48.16, op-cost: 0.0, total-cost: 245188.34]
+ project ([$$214, $$215, $$221, $$219]) [cardinality: 11937.93, doc-size: 48.16, op-cost: 0.0, total-cost: 129048.93]
-- STREAM_PROJECT |PARTITIONED|
- exchange [cardinality: 9951.86, doc-size: 48.16, op-cost: 0.0, total-cost: 245188.34]
+ exchange [cardinality: 11937.93, doc-size: 48.16, op-cost: 0.0, total-cost: 129048.93]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- join (eq($$216, $$233)) [cardinality: 9951.86, doc-size: 48.16, op-cost: 47782.02, total-cost: 245188.34]
- -- HYBRID_HASH_JOIN [$$233][$$216] |PARTITIONED|
- exchange [cardinality: 180956.24, doc-size: 10.0, op-cost: 0.0, total-cost: 196531.1]
+ join (eq($$220, $$227)) [cardinality: 11937.93, doc-size: 48.16, op-cost: 41469.7, total-cost: 129048.93]
+ -- HYBRID_HASH_JOIN [$$227][$$220] |PARTITIONED|
+ exchange [cardinality: 149063.0, doc-size: 10.0, op-cost: 0.0, total-cost: 74754.21]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- select (and(le($$213, "1996-12-31"), ge($$213, "1995-01-01"))) [cardinality: 180956.24, doc-size: 10.0, op-cost: 0.0, total-cost: 196531.1]
- -- STREAM_SELECT |PARTITIONED|
- assign [$$235, $$234, $$213, $$233] <- [$$l.getField(6), $$l.getField(5), $$l.getField(10), $$l.getField(2)] project: [$$218, $$235, $$234, $$213, $$233] [cardinality: 601114.0, doc-size: 10.0, op-cost: 0.0, total-cost: 196531.1]
- -- ASSIGN |PARTITIONED|
- project ([$$218, $$l]) [cardinality: 601114.0, doc-size: 10.0, op-cost: 0.0, total-cost: 196531.1]
- -- STREAM_PROJECT |PARTITIONED|
- exchange [cardinality: 601114.0, doc-size: 10.0, op-cost: 0.0, total-cost: 196531.1]
+ assign [$$227] <- [$$o.getField(1)] project: [$$219, $$227] [cardinality: 149063.0, doc-size: 10.0, op-cost: 0.0, total-cost: 74754.21]
+ -- ASSIGN |PARTITIONED|
+ exchange [cardinality: 149063.0, doc-size: 10.0, op-cost: 0.0, total-cost: 74754.21]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ data-scan []<-[$$219, $$o] <- tpch.orders [cardinality: 149063.0, doc-size: 10.0, op-cost: 0.0, total-cost: 74754.21]
+ -- DATASOURCE_SCAN |PARTITIONED|
+ exchange [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- data-scan []<-[$$217, $$218, $$l] <- tpch.lineitem [cardinality: 601114.0, doc-size: 10.0, op-cost: 0.0, total-cost: 196531.1]
- -- DATASOURCE_SCAN |PARTITIONED|
- exchange [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- empty-tuple-source [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
- exchange [cardinality: 55.0, doc-size: 38.16, op-cost: 0.0, total-cost: 820.22]
+ empty-tuple-source [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ exchange [cardinality: 1184.41, doc-size: 38.16, op-cost: 0.0, total-cost: 11640.62]
-- BROADCAST_EXCHANGE |PARTITIONED|
- project ([$$214, $$215, $$222, $$216]) [cardinality: 55.0, doc-size: 38.16, op-cost: 0.0, total-cost: 820.22]
+ project ([$$214, $$215, $$221, $$220]) [cardinality: 1184.41, doc-size: 38.16, op-cost: 0.0, total-cost: 11640.62]
-- STREAM_PROJECT |PARTITIONED|
- exchange [cardinality: 55.0, doc-size: 38.16, op-cost: 0.0, total-cost: 820.22]
+ exchange [cardinality: 1184.41, doc-size: 38.16, op-cost: 0.0, total-cost: 11640.62]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- join (eq($$228, $$221)) [cardinality: 55.0, doc-size: 38.16, op-cost: 187.85, total-cost: 820.22]
- -- HYBRID_HASH_JOIN [$$228][$$221] |PARTITIONED|
- exchange [cardinality: 1000.0, doc-size: 10.0, op-cost: 0.0, total-cost: 423.45]
+ join (eq($$223, $$222)) [cardinality: 1184.41, doc-size: 38.16, op-cost: 4002.43, total-cost: 11640.62]
+ -- HYBRID_HASH_JOIN [$$223][$$222] |PARTITIONED|
+ exchange [cardinality: 14803.0, doc-size: 10.0, op-cost: 0.0, total-cost: 7429.26]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- assign [$$228] <- [$$s.getField(3)] project: [$$216, $$228] [cardinality: 1000.0, doc-size: 10.0, op-cost: 0.0, total-cost: 423.45]
+ assign [$$223] <- [$$c.getField(3)] project: [$$220, $$223] [cardinality: 14803.0, doc-size: 10.0, op-cost: 0.0, total-cost: 7429.26]
-- ASSIGN |PARTITIONED|
- exchange [cardinality: 1000.0, doc-size: 10.0, op-cost: 0.0, total-cost: 423.45]
+ exchange [cardinality: 14803.0, doc-size: 10.0, op-cost: 0.0, total-cost: 7429.26]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- data-scan []<-[$$216, $$s] <- tpch.supplier [cardinality: 1000.0, doc-size: 10.0, op-cost: 0.0, total-cost: 423.45]
+ data-scan []<-[$$220, $$c] <- tpch.customer [cardinality: 14803.0, doc-size: 10.0, op-cost: 0.0, total-cost: 7429.26]
-- DATASOURCE_SCAN |PARTITIONED|
exchange [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
@@ -140,13 +140,13 @@ distribute result [$$209] [cardinality: 2.45, doc-size: 68.16, op-cost: 0.0, tot
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
empty-tuple-source [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
-- EMPTY_TUPLE_SOURCE |PARTITIONED|
- exchange [cardinality: 14803.0, doc-size: 10.0, op-cost: 0.0, total-cost: 7429.15]
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- assign [$$223] <- [$$c.getField(3)] project: [$$223, $$220] [cardinality: 14803.0, doc-size: 10.0, op-cost: 0.0, total-cost: 7429.15]
+ exchange [cardinality: 1000.0, doc-size: 10.0, op-cost: 0.0, total-cost: 498.2]
+ -- BROADCAST_EXCHANGE |PARTITIONED|
+ assign [$$228] <- [$$s.getField(3)] project: [$$228, $$216] [cardinality: 1000.0, doc-size: 10.0, op-cost: 0.0, total-cost: 498.2]
-- ASSIGN |PARTITIONED|
- exchange [cardinality: 14803.0, doc-size: 10.0, op-cost: 0.0, total-cost: 7429.15]
+ exchange [cardinality: 1000.0, doc-size: 10.0, op-cost: 0.0, total-cost: 498.2]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- data-scan []<-[$$220, $$c] <- tpch.customer [cardinality: 14803.0, doc-size: 10.0, op-cost: 0.0, total-cost: 7429.15]
+ data-scan []<-[$$216, $$s] <- tpch.supplier [cardinality: 1000.0, doc-size: 10.0, op-cost: 0.0, total-cost: 498.2]
-- DATASOURCE_SCAN |PARTITIONED|
exchange [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results_cbo/tpch/query-plans/query-plans.09.plan b/asterixdb/asterix-app/src/test/resources/runtimets/results_cbo/tpch/query-plans/query-plans.09.plan
index 35ff9e4e93..ec40413ebb 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results_cbo/tpch/query-plans/query-plans.09.plan
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results_cbo/tpch/query-plans/query-plans.09.plan
@@ -1,106 +1,106 @@
-distribute result [$$223] [cardinality: 1.0, doc-size: 70.0, op-cost: 0.0, total-cost: 538842.5]
+distribute result [$$223] [cardinality: 1.0, doc-size: 70.0, op-cost: 0.0, total-cost: 544765.52]
-- DISTRIBUTE_RESULT |PARTITIONED|
- exchange [cardinality: 1.0, doc-size: 70.0, op-cost: 0.0, total-cost: 538842.5]
+ exchange [cardinality: 1.0, doc-size: 70.0, op-cost: 0.0, total-cost: 544765.52]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- assign [$$223] <- [{"$1": 8, "o_year": $$o_year, "mkt_share": numeric-divide($$238, $$239)}] project: [$$223] [cardinality: 1.0, doc-size: 70.0, op-cost: 0.0, total-cost: 538842.5]
+ assign [$$223] <- [{"$1": 8, "o_year": $$o_year, "mkt_share": numeric-divide($$238, $$239)}] project: [$$223] [cardinality: 1.0, doc-size: 70.0, op-cost: 0.0, total-cost: 544765.52]
-- ASSIGN |PARTITIONED|
- exchange [cardinality: 1.0, doc-size: 70.0, op-cost: 0.0, total-cost: 538842.5]
+ exchange [cardinality: 1.0, doc-size: 70.0, op-cost: 0.0, total-cost: 544765.52]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
group by ([$$o_year := $$262]) decor ([]) {
aggregate [$$238, $$239] <- [global-sql-sum-serial($$260), global-sql-sum-serial($$261)] [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
-- AGGREGATE |LOCAL|
nested tuple source [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
-- NESTED_TUPLE_SOURCE |LOCAL|
- } [cardinality: 1.0, doc-size: 70.0, op-cost: 5.0, total-cost: 538842.5]
+ } [cardinality: 1.0, doc-size: 70.0, op-cost: 5.0, total-cost: 544765.52]
-- EXTERNAL_GROUP_BY[$$262] |PARTITIONED|
- exchange [cardinality: 1.0, doc-size: 70.0, op-cost: 0.0, total-cost: 538837.5]
+ exchange [cardinality: 1.0, doc-size: 70.0, op-cost: 0.0, total-cost: 544760.52]
-- HASH_PARTITION_EXCHANGE [$$262] |PARTITIONED|
group by ([$$262 := $$224]) decor ([]) {
aggregate [$$260, $$261] <- [local-sql-sum-serial(switch-case(true, eq($$256, "PERU"), numeric-multiply($$254, numeric-subtract(1, $$255)), 0)), local-sql-sum-serial(numeric-multiply($$254, numeric-subtract(1, $$255)))] [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
-- AGGREGATE |LOCAL|
nested tuple source [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
-- NESTED_TUPLE_SOURCE |LOCAL|
- } [cardinality: 1.0, doc-size: 70.0, op-cost: 35.54, total-cost: 538837.5]
+ } [cardinality: 1.0, doc-size: 70.0, op-cost: 65.44, total-cost: 544760.52]
-- EXTERNAL_GROUP_BY[$$224] |PARTITIONED|
- exchange [cardinality: 138.17, doc-size: 70.0, op-cost: 0.0, total-cost: 538801.96]
+ exchange [cardinality: 257.76, doc-size: 70.0, op-cost: 0.0, total-cost: 544695.08]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- assign [$$224] <- [get-year(date($$225))] project: [$$256, $$254, $$255, $$224] [cardinality: 138.17, doc-size: 70.0, op-cost: 0.0, total-cost: 538801.96]
+ assign [$$224] <- [get-year(date($$225))] project: [$$256, $$254, $$255, $$224] [cardinality: 257.76, doc-size: 70.0, op-cost: 0.0, total-cost: 544695.08]
-- ASSIGN |PARTITIONED|
- project ([$$254, $$255, $$225, $$256]) [cardinality: 138.17, doc-size: 70.0, op-cost: 0.0, total-cost: 538801.96]
+ project ([$$254, $$255, $$225, $$256]) [cardinality: 257.76, doc-size: 70.0, op-cost: 0.0, total-cost: 544695.08]
-- STREAM_PROJECT |PARTITIONED|
- exchange [cardinality: 138.17, doc-size: 70.0, op-cost: 0.0, total-cost: 538801.96]
+ exchange [cardinality: 257.76, doc-size: 70.0, op-cost: 0.0, total-cost: 544695.08]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- join (eq($$242, $$233)) [cardinality: 138.17, doc-size: 70.0, op-cost: 75.33, total-cost: 538801.96]
+ join (eq($$242, $$233)) [cardinality: 257.76, doc-size: 70.0, op-cost: 153.88, total-cost: 544695.08]
-- HYBRID_HASH_JOIN [$$242][$$233] |PARTITIONED|
- exchange [cardinality: 138.17, doc-size: 65.0, op-cost: 0.0, total-cost: 538673.31]
- -- HASH_PARTITION_EXCHANGE [$$242] |PARTITIONED|
- project ([$$254, $$255, $$225, $$242]) [cardinality: 138.17, doc-size: 65.0, op-cost: 0.0, total-cost: 538673.31]
+ exchange [cardinality: 257.76, doc-size: 65.0, op-cost: 0.0, total-cost: 544503.68]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ project ([$$254, $$255, $$225, $$242]) [cardinality: 257.76, doc-size: 65.0, op-cost: 0.0, total-cost: 544503.68]
-- STREAM_PROJECT |PARTITIONED|
- exchange [cardinality: 138.17, doc-size: 65.0, op-cost: 0.0, total-cost: 538673.31]
+ exchange [cardinality: 257.76, doc-size: 65.0, op-cost: 0.0, total-cost: 544503.68]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- join (and(eq($$236, $$234), eq($$240, $$232))) [cardinality: 138.17, doc-size: 65.0, op-cost: 232.25, total-cost: 538673.31]
- -- HYBRID_HASH_JOIN [$$234, $$240][$$236, $$232] |PARTITIONED|
- exchange [cardinality: 690.84, doc-size: 55.0, op-cost: 0.0, total-cost: 538403.54]
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- project ([$$254, $$255, $$225, $$242, $$234, $$240]) [cardinality: 690.84, doc-size: 55.0, op-cost: 0.0, total-cost: 538403.54]
+ join (eq($$227, $$249)) [cardinality: 257.76, doc-size: 65.0, op-cost: 376.48, total-cost: 544503.68]
+ -- HYBRID_HASH_JOIN [$$249][$$227] |PARTITIONED|
+ exchange [cardinality: 261.16, doc-size: 55.0, op-cost: 0.0, total-cost: 543316.96]
+ -- HASH_PARTITION_EXCHANGE [$$249] |PARTITIONED|
+ project ([$$254, $$255, $$225, $$249]) [cardinality: 261.16, doc-size: 55.0, op-cost: 0.0, total-cost: 543316.96]
-- STREAM_PROJECT |PARTITIONED|
- exchange [cardinality: 690.84, doc-size: 55.0, op-cost: 0.0, total-cost: 538403.54]
+ exchange [cardinality: 261.16, doc-size: 55.0, op-cost: 0.0, total-cost: 543316.96]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- join (eq($$244, $$231)) [cardinality: 690.84, doc-size: 55.0, op-cost: 4567.13, total-cost: 538403.54]
- -- HYBRID_HASH_JOIN [$$231][$$244] |PARTITIONED|
- exchange [cardinality: 14803.0, doc-size: 10.0, op-cost: 0.0, total-cost: 7429.15]
+ join (and(eq($$236, $$234), eq($$240, $$232))) [cardinality: 261.16, doc-size: 55.0, op-cost: 416.74, total-cost: 543316.96]
+ -- HYBRID_HASH_JOIN [$$234, $$240][$$236, $$232] |PARTITIONED|
+ exchange [cardinality: 1305.79, doc-size: 45.0, op-cost: 0.0, total-cost: 542862.7]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- assign [$$240] <- [$$c.getField(3)] project: [$$240, $$231] [cardinality: 14803.0, doc-size: 10.0, op-cost: 0.0, total-cost: 7429.15]
- -- ASSIGN |PARTITIONED|
- exchange [cardinality: 14803.0, doc-size: 10.0, op-cost: 0.0, total-cost: 7429.15]
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- data-scan []<-[$$231, $$c] <- tpch.customer [cardinality: 14803.0, doc-size: 10.0, op-cost: 0.0, total-cost: 7429.15]
- -- DATASOURCE_SCAN |PARTITIONED|
- exchange [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- empty-tuple-source [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
- exchange [cardinality: 690.19, doc-size: 45.0, op-cost: 0.0, total-cost: 525717.07]
- -- BROADCAST_EXCHANGE |PARTITIONED|
- project ([$$254, $$255, $$225, $$242, $$234, $$244]) [cardinality: 690.19, doc-size: 45.0, op-cost: 0.0, total-cost: 525717.07]
+ project ([$$254, $$255, $$225, $$249, $$234, $$240]) [cardinality: 1305.79, doc-size: 45.0, op-cost: 0.0, total-cost: 542862.7]
-- STREAM_PROJECT |PARTITIONED|
- exchange [cardinality: 690.19, doc-size: 45.0, op-cost: 0.0, total-cost: 525717.07]
+ exchange [cardinality: 1305.79, doc-size: 45.0, op-cost: 0.0, total-cost: 542862.7]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- join (eq($$229, $$230)) [cardinality: 690.19, doc-size: 45.0, op-cost: 13517.02, total-cost: 525717.07]
- -- HYBRID_HASH_JOIN [$$230][$$229] |PARTITIONED|
- exchange [cardinality: 44031.78, doc-size: 10.0, op-cost: 0.0, total-cost: 48462.48]
+ join (eq($$244, $$231)) [cardinality: 1305.79, doc-size: 45.0, op-cost: 5335.24, total-cost: 542862.7]
+ -- HYBRID_HASH_JOIN [$$231][$$244] |PARTITIONED|
+ exchange [cardinality: 14803.0, doc-size: 10.0, op-cost: 0.0, total-cost: 7429.26]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- select (and(le($$225, "1996-12-31"), ge($$225, "1995-01-01"))) [cardinality: 44031.78, doc-size: 10.0, op-cost: 0.0, total-cost: 48462.48]
- -- STREAM_SELECT |PARTITIONED|
- assign [$$225, $$244] <- [$$o.getField(4), $$o.getField(1)] project: [$$230, $$225, $$244] [cardinality: 149063.0, doc-size: 10.0, op-cost: 0.0, total-cost: 48462.48]
- -- ASSIGN |PARTITIONED|
- exchange [cardinality: 149063.0, doc-size: 10.0, op-cost: 0.0, total-cost: 48462.48]
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- data-scan []<-[$$230, $$o] <- tpch.orders [cardinality: 149063.0, doc-size: 10.0, op-cost: 0.0, total-cost: 48462.48]
- -- DATASOURCE_SCAN |PARTITIONED|
- exchange [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- empty-tuple-source [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
- exchange [cardinality: 2336.53, doc-size: 35.0, op-cost: 0.0, total-cost: 461401.03]
+ assign [$$240] <- [$$c.getField(3)] project: [$$240, $$231] [cardinality: 14803.0, doc-size: 10.0, op-cost: 0.0, total-cost: 7429.26]
+ -- ASSIGN |PARTITIONED|
+ exchange [cardinality: 14803.0, doc-size: 10.0, op-cost: 0.0, total-cost: 7429.26]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ data-scan []<-[$$231, $$c] <- tpch.customer [cardinality: 14803.0, doc-size: 10.0, op-cost: 0.0, total-cost: 7429.26]
+ -- DATASOURCE_SCAN |PARTITIONED|
+ exchange [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ empty-tuple-source [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ exchange [cardinality: 1304.56, doc-size: 35.0, op-cost: 0.0, total-cost: 528793.63]
-- BROADCAST_EXCHANGE |PARTITIONED|
- project ([$$254, $$255, $$242, $$234, $$229]) [cardinality: 2336.53, doc-size: 35.0, op-cost: 0.0, total-cost: 461401.03]
+ project ([$$254, $$255, $$225, $$249, $$234, $$244]) [cardinality: 1304.56, doc-size: 35.0, op-cost: 0.0, total-cost: 528793.63]
-- STREAM_PROJECT |PARTITIONED|
- exchange [cardinality: 2336.53, doc-size: 35.0, op-cost: 0.0, total-cost: 461401.03]
+ exchange [cardinality: 1304.56, doc-size: 35.0, op-cost: 0.0, total-cost: 528793.63]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- join (eq($$227, $$249)) [cardinality: 2336.53, doc-size: 35.0, op-cost: 1605.16, total-cost: 461401.03]
- -- HYBRID_HASH_JOIN [$$249][$$227] |PARTITIONED|
- exchange [cardinality: 3396.12, doc-size: 25.0, op-cost: 0.0, total-cost: 458351.39]
- -- HASH_PARTITION_EXCHANGE [$$249] |PARTITIONED|
- project ([$$254, $$255, $$234, $$229, $$249]) [cardinality: 3396.12, doc-size: 25.0, op-cost: 0.0, total-cost: 458351.39]
+ join (eq($$229, $$230)) [cardinality: 1304.56, doc-size: 35.0, op-cost: 16558.28, total-cost: 528793.63]
+ -- HYBRID_HASH_JOIN [$$230][$$229] |PARTITIONED|
+ exchange [cardinality: 49080.01, doc-size: 10.0, op-cost: 0.0, total-cost: 49723.4]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ select (and(le($$225, "1996-12-31"), ge($$225, "1995-01-01"))) [cardinality: 49080.01, doc-size: 10.0, op-cost: 0.0, total-cost: 49723.4]
+ -- STREAM_SELECT |PARTITIONED|
+ assign [$$225, $$244] <- [$$o.getField(4), $$o.getField(1)] project: [$$230, $$225, $$244] [cardinality: 149063.0, doc-size: 10.0, op-cost: 0.0, total-cost: 49723.4]
+ -- ASSIGN |PARTITIONED|
+ exchange [cardinality: 149063.0, doc-size: 10.0, op-cost: 0.0, total-cost: 49723.4]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ data-scan []<-[$$230, $$o] <- tpch.orders [cardinality: 149063.0, doc-size: 10.0, op-cost: 0.0, total-cost: 49723.4]
+ -- DATASOURCE_SCAN |PARTITIONED|
+ exchange [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ empty-tuple-source [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ exchange [cardinality: 3962.14, doc-size: 25.0, op-cost: 0.0, total-cost: 458549.8]
+ -- BROADCAST_EXCHANGE |PARTITIONED|
+ project ([$$254, $$255, $$249, $$234, $$229]) [cardinality: 3962.14, doc-size: 25.0, op-cost: 0.0, total-cost: 458549.8]
-- STREAM_PROJECT |PARTITIONED|
- exchange [cardinality: 3396.12, doc-size: 25.0, op-cost: 0.0, total-cost: 458351.39]
+ exchange [cardinality: 3962.14, doc-size: 25.0, op-cost: 0.0, total-cost: 458549.8]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- join (eq($$226, $$247)) [cardinality: 3396.12, doc-size: 25.0, op-cost: 151381.78, total-cost: 458351.39]
+ join (eq($$226, $$247)) [cardinality: 3962.14, doc-size: 25.0, op-cost: 151542.1, total-cost: 458549.8]
-- HYBRID_HASH_JOIN [$$247][$$226] |PARTITIONED|
exchange [cardinality: 601114.0, doc-size: 15.0, op-cost: 0.0, total-cost: 301711.91]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- assign [$$255, $$254, $$249, $$247] <- [$$l.getField(6), $$l.getField(5), $$l.getField(2), $$l.getField(1)] project: [$$254, $$255, $$229, $$249, $$247] [cardinality: 601114.0, doc-size: 15.0, op-cost: 0.0, total-cost: 301711.91]
+ assign [$$255, $$254, $$249, $$247] <- [$$l.getField(6), $$l.getField(5), $$l.getField(2), $$l.getField(1)] project: [$$254, $$255, $$249, $$229, $$247] [cardinality: 601114.0, doc-size: 15.0, op-cost: 0.0, total-cost: 301711.91]
-- ASSIGN |PARTITIONED|
project ([$$229, $$l]) [cardinality: 601114.0, doc-size: 15.0, op-cost: 0.0, total-cost: 301711.91]
-- STREAM_PROJECT |PARTITIONED|
@@ -112,17 +112,17 @@ distribute result [$$223] [cardinality: 1.0, doc-size: 70.0, op-cost: 0.0, total
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
empty-tuple-source [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
-- EMPTY_TUPLE_SOURCE |PARTITIONED|
- exchange [cardinality: 112.88, doc-size: 10.0, op-cost: 0.0, total-cost: 5144.81]
+ exchange [cardinality: 131.7, doc-size: 10.0, op-cost: 0.0, total-cost: 5164.1]
-- BROADCAST_EXCHANGE |PARTITIONED|
- join (true) [cardinality: 112.88, doc-size: 10.0, op-cost: 87.48, total-cost: 5144.81]
+ join (true) [cardinality: 131.7, doc-size: 10.0, op-cost: 102.06, total-cost: 5164.1]
-- NESTED_LOOP |PARTITIONED|
- exchange [cardinality: 112.88, doc-size: 5.0, op-cost: 0.0, total-cost: 5054.82]
+ exchange [cardinality: 131.7, doc-size: 5.0, op-cost: 0.0, total-cost: 5059.53]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- select (eq($$p.getField(4), "ECONOMY BURNISHED NICKEL")) project: [$$226] [cardinality: 112.88, doc-size: 5.0, op-cost: 0.0, total-cost: 5054.82]
+ select (eq($$p.getField(4), "ECONOMY BURNISHED NICKEL")) project: [$$226] [cardinality: 131.7, doc-size: 5.0, op-cost: 0.0, total-cost: 5059.53]
-- STREAM_SELECT |PARTITIONED|
- exchange [cardinality: 19999.0, doc-size: 5.0, op-cost: 0.0, total-cost: 5054.82]
+ exchange [cardinality: 19999.0, doc-size: 5.0, op-cost: 0.0, total-cost: 5059.53]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- data-scan []<-[$$226, $$p] <- tpch.part [cardinality: 19999.0, doc-size: 5.0, op-cost: 0.0, total-cost: 5054.82]
+ data-scan []<-[$$226, $$p] <- tpch.part [cardinality: 19999.0, doc-size: 5.0, op-cost: 0.0, total-cost: 5059.53]
-- DATASOURCE_SCAN |PARTITIONED|
exchange [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
@@ -140,36 +140,36 @@ distribute result [$$223] [cardinality: 1.0, doc-size: 70.0, op-cost: 0.0, total
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
empty-tuple-source [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
-- EMPTY_TUPLE_SOURCE |PARTITIONED|
- exchange [cardinality: 1000.0, doc-size: 10.0, op-cost: 0.0, total-cost: 423.45]
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- assign [$$242] <- [$$s.getField(3)] project: [$$242, $$227] [cardinality: 1000.0, doc-size: 10.0, op-cost: 0.0, total-cost: 423.45]
- -- ASSIGN |PARTITIONED|
- exchange [cardinality: 1000.0, doc-size: 10.0, op-cost: 0.0, total-cost: 423.45]
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- data-scan []<-[$$227, $$s] <- tpch.supplier [cardinality: 1000.0, doc-size: 10.0, op-cost: 0.0, total-cost: 423.45]
- -- DATASOURCE_SCAN |PARTITIONED|
- exchange [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- empty-tuple-source [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
- exchange [cardinality: 25.0, doc-size: 10.0, op-cost: 0.0, total-cost: 12.52]
- -- BROADCAST_EXCHANGE |PARTITIONED|
- assign [$$236] <- [$$n1.getField(2)] project: [$$236, $$232] [cardinality: 25.0, doc-size: 10.0, op-cost: 0.0, total-cost: 12.52]
+ exchange [cardinality: 25.0, doc-size: 10.0, op-cost: 0.0, total-cost: 12.52]
+ -- BROADCAST_EXCHANGE |PARTITIONED|
+ assign [$$236] <- [$$n1.getField(2)] project: [$$236, $$232] [cardinality: 25.0, doc-size: 10.0, op-cost: 0.0, total-cost: 12.52]
+ -- ASSIGN |PARTITIONED|
+ exchange [cardinality: 25.0, doc-size: 10.0, op-cost: 0.0, total-cost: 12.52]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ replicate [cardinality: 25.0, doc-size: 10.0, op-cost: 0.0, total-cost: 12.52]
+ -- REPLICATE |PARTITIONED|
+ exchange [cardinality: 25.0, doc-size: 10.0, op-cost: 0.0, total-cost: 12.52]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ data-scan []<-[$$232, $$n1] <- tpch.nation [cardinality: 25.0, doc-size: 10.0, op-cost: 0.0, total-cost: 12.52]
+ -- DATASOURCE_SCAN |PARTITIONED|
+ exchange [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ empty-tuple-source [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ exchange [cardinality: 1000.0, doc-size: 10.0, op-cost: 0.0, total-cost: 498.2]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ assign [$$242] <- [$$s.getField(3)] project: [$$242, $$227] [cardinality: 1000.0, doc-size: 10.0, op-cost: 0.0, total-cost: 498.2]
-- ASSIGN |PARTITIONED|
- exchange [cardinality: 25.0, doc-size: 10.0, op-cost: 0.0, total-cost: 12.52]
+ exchange [cardinality: 1000.0, doc-size: 10.0, op-cost: 0.0, total-cost: 498.2]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- replicate [cardinality: 25.0, doc-size: 10.0, op-cost: 0.0, total-cost: 12.52]
- -- REPLICATE |PARTITIONED|
- exchange [cardinality: 25.0, doc-size: 10.0, op-cost: 0.0, total-cost: 12.52]
+ data-scan []<-[$$227, $$s] <- tpch.supplier [cardinality: 1000.0, doc-size: 10.0, op-cost: 0.0, total-cost: 498.2]
+ -- DATASOURCE_SCAN |PARTITIONED|
+ exchange [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- data-scan []<-[$$232, $$n1] <- tpch.nation [cardinality: 25.0, doc-size: 10.0, op-cost: 0.0, total-cost: 12.52]
- -- DATASOURCE_SCAN |PARTITIONED|
- exchange [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- empty-tuple-source [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ empty-tuple-source [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
exchange [cardinality: 25.0, doc-size: 10.0, op-cost: 0.0, total-cost: 12.52]
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BROADCAST_EXCHANGE |PARTITIONED|
assign [$$256] <- [$$n2.getField(1)] project: [$$256, $$233] [cardinality: 25.0, doc-size: 10.0, op-cost: 0.0, total-cost: 12.52]
-- ASSIGN |PARTITIONED|
assign [$$233, $$n2] <- [$$232, $$n1] project: [$$233, $$n2] [cardinality: 25.0, doc-size: 10.0, op-cost: 0.0, total-cost: 12.52]
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results_cbo/tpch/query-plans/query-plans.10.plan b/asterixdb/asterix-app/src/test/resources/runtimets/results_cbo/tpch/query-plans/query-plans.10.plan
index ab3937ab1a..bcaa05847f 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results_cbo/tpch/query-plans/query-plans.10.plan
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results_cbo/tpch/query-plans/query-plans.10.plan
@@ -1,38 +1,26 @@
-distribute result [$$137] [cardinality: 24065.62, doc-size: 50.0, op-cost: 0.0, total-cost: 702976.68]
+distribute result [$$137] [cardinality: 27595.61, doc-size: 50.0, op-cost: 0.0, total-cost: 714791.53]
-- DISTRIBUTE_RESULT |UNPARTITIONED|
- exchange [cardinality: 24065.62, doc-size: 50.0, op-cost: 0.0, total-cost: 702976.68]
+ exchange [cardinality: 27595.61, doc-size: 50.0, op-cost: 0.0, total-cost: 714791.53]
-- ONE_TO_ONE_EXCHANGE |UNPARTITIONED|
- assign [$$137] <- [{"$1": 9, "$2": $$155}] project: [$$137] [cardinality: 24065.62, doc-size: 50.0, op-cost: 0.0, total-cost: 702976.68]
+ assign [$$137] <- [{"$1": 9, "$2": $$155}] project: [$$137] [cardinality: 27595.61, doc-size: 50.0, op-cost: 0.0, total-cost: 714791.53]
-- ASSIGN |UNPARTITIONED|
- aggregate [$$155] <- [agg-sql-sum($$159)] [cardinality: 24065.62, doc-size: 50.0, op-cost: 0.0, total-cost: 702976.68]
+ aggregate [$$155] <- [agg-sql-sum($$159)] [cardinality: 27595.61, doc-size: 50.0, op-cost: 0.0, total-cost: 714791.53]
-- AGGREGATE |UNPARTITIONED|
- exchange [cardinality: 24065.62, doc-size: 50.0, op-cost: 0.0, total-cost: 702976.68]
+ exchange [cardinality: 27595.61, doc-size: 50.0, op-cost: 0.0, total-cost: 714791.53]
-- RANDOM_MERGE_EXCHANGE |PARTITIONED|
- aggregate [$$159] <- [agg-sql-count(1)] [cardinality: 24065.62, doc-size: 50.0, op-cost: 0.0, total-cost: 702976.68]
+ aggregate [$$159] <- [agg-sql-count(1)] [cardinality: 27595.61, doc-size: 50.0, op-cost: 0.0, total-cost: 714791.53]
-- AGGREGATE |PARTITIONED|
- exchange [cardinality: 24065.62, doc-size: 50.0, op-cost: 0.0, total-cost: 702976.68]
+ exchange [cardinality: 27595.61, doc-size: 50.0, op-cost: 0.0, total-cost: 714791.53]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- join (eq($$151, $$147)) [cardinality: 24065.62, doc-size: 50.0, op-cost: 67360.22, total-cost: 702976.68]
- -- HYBRID_HASH_JOIN [$$151][$$147] |PARTITIONED|
- exchange [cardinality: 149063.0, doc-size: 5.0, op-cost: 0.0, total-cost: 74755.34]
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- project ([$$151]) [cardinality: 149063.0, doc-size: 5.0, op-cost: 0.0, total-cost: 74755.34]
- -- STREAM_PROJECT |PARTITIONED|
- exchange [cardinality: 149063.0, doc-size: 5.0, op-cost: 0.0, total-cost: 74755.34]
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- data-scan []<-[$$151, $$o] <- tpch.orders [cardinality: 149063.0, doc-size: 5.0, op-cost: 0.0, total-cost: 74755.34]
- -- DATASOURCE_SCAN |PARTITIONED|
- exchange [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- empty-tuple-source [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
- exchange [cardinality: 24043.0, doc-size: 45.0, op-cost: 0.0, total-cost: 536818.12]
- -- BROADCAST_EXCHANGE |PARTITIONED|
- project ([$$147]) [cardinality: 24043.0, doc-size: 45.0, op-cost: 0.0, total-cost: 536818.12]
+ join (eq($$151, $$147)) [cardinality: 27595.61, doc-size: 50.0, op-cost: 51092.13, total-cost: 714791.53]
+ -- HYBRID_HASH_JOIN [$$147][$$151] |PARTITIONED|
+ exchange [cardinality: 27569.67, doc-size: 45.0, op-cost: 0.0, total-cost: 544751.97]
+ -- HASH_PARTITION_EXCHANGE [$$147] |PARTITIONED|
+ project ([$$147]) [cardinality: 27569.67, doc-size: 45.0, op-cost: 0.0, total-cost: 544751.97]
-- STREAM_PROJECT |PARTITIONED|
- exchange [cardinality: 24043.0, doc-size: 45.0, op-cost: 0.0, total-cost: 536818.12]
+ exchange [cardinality: 27569.67, doc-size: 45.0, op-cost: 0.0, total-cost: 544751.97]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- join (and(eq($$150, $$143), eq($$149, $$140))) [cardinality: 24043.0, doc-size: 45.0, op-cost: 159086.7, total-cost: 536818.12]
+ join (and(eq($$150, $$143), eq($$149, $$140))) [cardinality: 27569.67, doc-size: 45.0, op-cost: 162019.25, total-cost: 544751.97]
-- HYBRID_HASH_JOIN [$$143, $$140][$$150, $$149] |PARTITIONED|
exchange [cardinality: 601114.0, doc-size: 15.0, op-cost: 0.0, total-cost: 301711.91]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
@@ -48,61 +36,61 @@ distribute result [$$137] [cardinality: 24065.62, doc-size: 50.0, op-cost: 0.0,
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
empty-tuple-source [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
-- EMPTY_TUPLE_SOURCE |PARTITIONED|
- exchange [cardinality: 2656.08, doc-size: 30.0, op-cost: 0.0, total-cost: 73363.43]
+ exchange [cardinality: 4706.96, doc-size: 30.0, op-cost: 0.0, total-cost: 76313.86]
-- BROADCAST_EXCHANGE |PARTITIONED|
- project ([$$149, $$150]) [cardinality: 2656.08, doc-size: 30.0, op-cost: 0.0, total-cost: 73363.43]
+ project ([$$149, $$150]) [cardinality: 4706.96, doc-size: 30.0, op-cost: 0.0, total-cost: 76313.86]
-- STREAM_PROJECT |PARTITIONED|
- exchange [cardinality: 2656.08, doc-size: 30.0, op-cost: 0.0, total-cost: 73363.43]
+ exchange [cardinality: 4706.96, doc-size: 30.0, op-cost: 0.0, total-cost: 76313.86]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- join (eq($$153, $$152)) [cardinality: 2656.08, doc-size: 30.0, op-cost: 1353.04, total-cost: 73363.43]
+ join (eq($$153, $$152)) [cardinality: 4706.96, doc-size: 30.0, op-cost: 2378.48, total-cost: 76313.86]
-- HYBRID_HASH_JOIN [$$153][$$152] |PARTITIONED|
- exchange [cardinality: 2656.08, doc-size: 25.0, op-cost: 0.0, total-cost: 71972.86]
+ exchange [cardinality: 4706.96, doc-size: 25.0, op-cost: 0.0, total-cost: 73897.86]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- project ([$$149, $$150, $$153]) [cardinality: 2656.08, doc-size: 25.0, op-cost: 0.0, total-cost: 71972.86]
+ project ([$$149, $$150, $$153]) [cardinality: 4706.96, doc-size: 25.0, op-cost: 0.0, total-cost: 73897.86]
-- STREAM_PROJECT |PARTITIONED|
- exchange [cardinality: 2656.08, doc-size: 25.0, op-cost: 0.0, total-cost: 71972.86]
+ exchange [cardinality: 4706.96, doc-size: 25.0, op-cost: 0.0, total-cost: 73897.86]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- join (eq($$145, $$150)) [cardinality: 2656.08, doc-size: 25.0, op-cost: 1801.17, total-cost: 71972.86]
+ join (eq($$145, $$150)) [cardinality: 4706.96, doc-size: 25.0, op-cost: 2615.73, total-cost: 73897.86]
-- HYBRID_HASH_JOIN [$$150][$$145] |PARTITIONED|
- exchange [cardinality: 3860.58, doc-size: 15.0, op-cost: 0.0, total-cost: 68611.1]
+ exchange [cardinality: 4768.95, doc-size: 15.0, op-cost: 0.0, total-cost: 69344.94]
-- HASH_PARTITION_EXCHANGE [$$150] |PARTITIONED|
- project ([$$149, $$150]) [cardinality: 3860.58, doc-size: 15.0, op-cost: 0.0, total-cost: 68611.1]
+ project ([$$149, $$150]) [cardinality: 4768.95, doc-size: 15.0, op-cost: 0.0, total-cost: 69344.94]
-- STREAM_PROJECT |PARTITIONED|
- exchange [cardinality: 3860.58, doc-size: 15.0, op-cost: 0.0, total-cost: 68611.1]
+ exchange [cardinality: 4768.95, doc-size: 15.0, op-cost: 0.0, total-cost: 69344.94]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- join (eq($$144, $$149)) [cardinality: 3860.58, doc-size: 15.0, op-cost: 22041.3, total-cost: 68611.1]
+ join (eq($$144, $$149)) [cardinality: 4768.95, doc-size: 15.0, op-cost: 22494.16, total-cost: 69344.94]
-- HYBRID_HASH_JOIN [$$149][$$144] |PARTITIONED|
- exchange [cardinality: 80391.0, doc-size: 10.0, op-cost: 0.0, total-cost: 40343.82]
+ exchange [cardinality: 80391.0, doc-size: 10.0, op-cost: 0.0, total-cost: 40342.59]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- project ([$$149, $$150]) [cardinality: 80391.0, doc-size: 10.0, op-cost: 0.0, total-cost: 40343.82]
+ project ([$$149, $$150]) [cardinality: 80391.0, doc-size: 10.0, op-cost: 0.0, total-cost: 40342.59]
-- STREAM_PROJECT |PARTITIONED|
- exchange [cardinality: 80391.0, doc-size: 10.0, op-cost: 0.0, total-cost: 40343.82]
+ exchange [cardinality: 80391.0, doc-size: 10.0, op-cost: 0.0, total-cost: 40342.59]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- data-scan []<-[$$148, $$149, $$150, $$ps] <- tpch.partsupp [cardinality: 80391.0, doc-size: 10.0, op-cost: 0.0, total-cost: 40343.82]
+ data-scan []<-[$$148, $$149, $$150, $$ps] <- tpch.partsupp [cardinality: 80391.0, doc-size: 10.0, op-cost: 0.0, total-cost: 40342.59]
-- DATASOURCE_SCAN |PARTITIONED|
exchange [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
empty-tuple-source [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
-- EMPTY_TUPLE_SOURCE |PARTITIONED|
- exchange [cardinality: 959.5, doc-size: 5.0, op-cost: 0.0, total-cost: 5266.48]
+ exchange [cardinality: 1185.27, doc-size: 5.0, op-cost: 0.0, total-cost: 5322.92]
-- BROADCAST_EXCHANGE |PARTITIONED|
- select (like($$p.getField(1), "%green%")) project: [$$144] [cardinality: 959.5, doc-size: 5.0, op-cost: 0.0, total-cost: 5266.48]
+ select (like($$p.getField(1), "%green%")) project: [$$144] [cardinality: 1185.27, doc-size: 5.0, op-cost: 0.0, total-cost: 5322.92]
-- STREAM_SELECT |PARTITIONED|
- exchange [cardinality: 19999.0, doc-size: 5.0, op-cost: 0.0, total-cost: 5266.48]
+ exchange [cardinality: 19999.0, doc-size: 5.0, op-cost: 0.0, total-cost: 5322.92]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- data-scan []<-[$$144, $$p] <- tpch.part [cardinality: 19999.0, doc-size: 5.0, op-cost: 0.0, total-cost: 5266.48]
+ data-scan []<-[$$144, $$p] <- tpch.part [cardinality: 19999.0, doc-size: 5.0, op-cost: 0.0, total-cost: 5322.92]
-- DATASOURCE_SCAN |PARTITIONED|
exchange [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
empty-tuple-source [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
-- EMPTY_TUPLE_SOURCE |PARTITIONED|
- exchange [cardinality: 1000.0, doc-size: 10.0, op-cost: 0.0, total-cost: 423.45]
+ exchange [cardinality: 1000.0, doc-size: 10.0, op-cost: 0.0, total-cost: 498.2]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- assign [$$153] <- [$$s.getField(3)] project: [$$145, $$153] [cardinality: 1000.0, doc-size: 10.0, op-cost: 0.0, total-cost: 423.45]
+ assign [$$153] <- [$$s.getField(3)] project: [$$145, $$153] [cardinality: 1000.0, doc-size: 10.0, op-cost: 0.0, total-cost: 498.2]
-- ASSIGN |PARTITIONED|
- exchange [cardinality: 1000.0, doc-size: 10.0, op-cost: 0.0, total-cost: 423.45]
+ exchange [cardinality: 1000.0, doc-size: 10.0, op-cost: 0.0, total-cost: 498.2]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- data-scan []<-[$$145, $$s] <- tpch.supplier [cardinality: 1000.0, doc-size: 10.0, op-cost: 0.0, total-cost: 423.45]
+ data-scan []<-[$$145, $$s] <- tpch.supplier [cardinality: 1000.0, doc-size: 10.0, op-cost: 0.0, total-cost: 498.2]
-- DATASOURCE_SCAN |PARTITIONED|
exchange [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
@@ -120,3 +108,15 @@ distribute result [$$137] [cardinality: 24065.62, doc-size: 50.0, op-cost: 0.0,
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
empty-tuple-source [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
-- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ exchange [cardinality: 149063.0, doc-size: 5.0, op-cost: 0.0, total-cost: 74754.21]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ project ([$$151]) [cardinality: 149063.0, doc-size: 5.0, op-cost: 0.0, total-cost: 74754.21]
+ -- STREAM_PROJECT |PARTITIONED|
+ exchange [cardinality: 149063.0, doc-size: 5.0, op-cost: 0.0, total-cost: 74754.21]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ data-scan []<-[$$151, $$o] <- tpch.orders [cardinality: 149063.0, doc-size: 5.0, op-cost: 0.0, total-cost: 74754.21]
+ -- DATASOURCE_SCAN |PARTITIONED|
+ exchange [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ empty-tuple-source [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results_cbo/tpch/query-plans/query-plans.11.plan b/asterixdb/asterix-app/src/test/resources/runtimets/results_cbo/tpch/query-plans/query-plans.11.plan
index da418e53d9..9c0b5b9c55 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results_cbo/tpch/query-plans/query-plans.11.plan
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results_cbo/tpch/query-plans/query-plans.11.plan
@@ -1,112 +1,112 @@
-distribute result [$$212] [cardinality: 4634.66, doc-size: 30.0, op-cost: 0.0, total-cost: 326395.62]
+distribute result [$$212] [cardinality: 6331.65, doc-size: 30.0, op-cost: 0.0, total-cost: 339074.95]
-- DISTRIBUTE_RESULT |UNPARTITIONED|
- exchange [cardinality: 4634.66, doc-size: 30.0, op-cost: 0.0, total-cost: 326395.62]
+ exchange [cardinality: 6331.65, doc-size: 30.0, op-cost: 0.0, total-cost: 339074.95]
-- ONE_TO_ONE_EXCHANGE |UNPARTITIONED|
- limit 20 [cardinality: 4634.66, doc-size: 30.0, op-cost: 0.0, total-cost: 326395.62]
+ limit 20 [cardinality: 6331.65, doc-size: 30.0, op-cost: 0.0, total-cost: 339074.95]
-- STREAM_LIMIT |UNPARTITIONED|
- assign [$$212] <- [{"$1": 10, "c_custkey": $$c_custkey, "c_name": $$c_name, "revenue": $$230, "c_acctbal": $$c_acctbal, "n_name": $$n_name, "c_address": $$c_address, "c_phone": $$c_phone, "c_comment": $$c_comment}] project: [$$212] [cardinality: 4634.66, doc-size: 30.0, op-cost: 0.0, total-cost: 326395.62]
+ assign [$$212] <- [{"$1": 10, "c_custkey": $$c_custkey, "c_name": $$c_name, "revenue": $$230, "c_acctbal": $$c_acctbal, "n_name": $$n_name, "c_address": $$c_address, "c_phone": $$c_phone, "c_comment": $$c_comment}] project: [$$212] [cardinality: 6331.65, doc-size: 30.0, op-cost: 0.0, total-cost: 339074.95]
-- ASSIGN |PARTITIONED|
- exchange [cardinality: 4634.66, doc-size: 30.0, op-cost: 0.0, total-cost: 326395.62]
+ exchange [cardinality: 6331.65, doc-size: 30.0, op-cost: 0.0, total-cost: 339074.95]
-- SORT_MERGE_EXCHANGE [$$230(DESC) ] |PARTITIONED|
- limit 20 [cardinality: 4634.66, doc-size: 30.0, op-cost: 0.0, total-cost: 326395.62]
+ limit 20 [cardinality: 6331.65, doc-size: 30.0, op-cost: 0.0, total-cost: 339074.95]
-- STREAM_LIMIT |PARTITIONED|
- exchange [cardinality: 4634.66, doc-size: 30.0, op-cost: 0.0, total-cost: 326395.62]
+ exchange [cardinality: 6331.65, doc-size: 30.0, op-cost: 0.0, total-cost: 339074.95]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- order (topK: 20) (DESC, $$230) [cardinality: 4634.66, doc-size: 30.0, op-cost: 0.0, total-cost: 326395.62]
+ order (topK: 20) (DESC, $$230) [cardinality: 6331.65, doc-size: 30.0, op-cost: 0.0, total-cost: 339074.95]
-- STABLE_SORT [topK: 20] [$$230(DESC)] |PARTITIONED|
- exchange [cardinality: 4634.66, doc-size: 30.0, op-cost: 0.0, total-cost: 326395.62]
+ exchange [cardinality: 6331.65, doc-size: 30.0, op-cost: 0.0, total-cost: 339074.95]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
group by ([$$c_custkey := $$238; $$n_name := $$242]) decor ([$$c_name := $$239; $$c_acctbal := $$240; $$c_phone := $$241; $$c_address := $$243; $$c_comment := $$244]) {
aggregate [$$230] <- [global-sql-sum-serial($$237)] [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
-- AGGREGATE |LOCAL|
nested tuple source [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
-- NESTED_TUPLE_SOURCE |LOCAL|
- } [cardinality: 4634.66, doc-size: 30.0, op-cost: 23173.3, total-cost: 326395.62]
+ } [cardinality: 6331.65, doc-size: 30.0, op-cost: 31658.25, total-cost: 339074.95]
-- EXTERNAL_GROUP_BY[$$238, $$242] |PARTITIONED|
- exchange [cardinality: 4634.66, doc-size: 30.0, op-cost: 0.0, total-cost: 303222.32]
+ exchange [cardinality: 6331.65, doc-size: 30.0, op-cost: 0.0, total-cost: 307416.7]
-- HASH_PARTITION_EXCHANGE [$$238, $$242] |PARTITIONED|
group by ([$$238 := $$222; $$242 := $$217]) decor ([$$239 := $$214; $$240 := $$215; $$241 := $$216; $$243 := $$218; $$244 := $$219]) {
aggregate [$$237] <- [local-sql-sum-serial(numeric-multiply($$235, numeric-subtract(1, $$236)))] [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
-- AGGREGATE |LOCAL|
nested tuple source [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
-- NESTED_TUPLE_SOURCE |LOCAL|
- } [cardinality: 4634.66, doc-size: 30.0, op-cost: 5793.33, total-cost: 303222.32]
+ } [cardinality: 6331.65, doc-size: 30.0, op-cost: 7914.56, total-cost: 307416.7]
-- EXTERNAL_GROUP_BY[$$222, $$217] |PARTITIONED|
- exchange [cardinality: 4634.66, doc-size: 30.0, op-cost: 0.0, total-cost: 297428.99]
+ exchange [cardinality: 6331.65, doc-size: 30.0, op-cost: 0.0, total-cost: 299502.14]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- project ([$$235, $$236, $$222, $$214, $$215, $$216, $$217, $$218, $$219]) [cardinality: 4634.66, doc-size: 30.0, op-cost: 0.0, total-cost: 297428.99]
+ project ([$$235, $$236, $$222, $$214, $$215, $$216, $$217, $$218, $$219]) [cardinality: 6331.65, doc-size: 30.0, op-cost: 0.0, total-cost: 299502.14]
-- STREAM_PROJECT |PARTITIONED|
- exchange [cardinality: 4634.66, doc-size: 30.0, op-cost: 0.0, total-cost: 297428.99]
+ exchange [cardinality: 6331.65, doc-size: 30.0, op-cost: 0.0, total-cost: 299502.14]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- join (eq($$225, $$223)) [cardinality: 4634.66, doc-size: 30.0, op-cost: 44103.4, total-cost: 297428.99]
- -- HYBRID_HASH_JOIN [$$225][$$223] |PARTITIONED|
- exchange [cardinality: 153812.8, doc-size: 5.0, op-cost: 0.0, total-cost: 189745.24]
+ join (eq($$228, $$226)) [cardinality: 6331.65, doc-size: 30.0, op-cost: 3190.83, total-cost: 299502.14]
+ -- HYBRID_HASH_JOIN [$$228][$$226] |PARTITIONED|
+ exchange [cardinality: 6331.65, doc-size: 25.0, op-cost: 0.0, total-cost: 296273.79]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- select (eq($$l.getField(8), "R")) project: [$$235, $$236, $$225] [cardinality: 153812.8, doc-size: 5.0, op-cost: 0.0, total-cost: 189745.24]
- -- STREAM_SELECT |PARTITIONED|
- assign [$$236, $$235] <- [$$l.getField(6), $$l.getField(5)] [cardinality: 601114.0, doc-size: 5.0, op-cost: 0.0, total-cost: 189745.24]
- -- ASSIGN |PARTITIONED|
- project ([$$225, $$l]) [cardinality: 601114.0, doc-size: 5.0, op-cost: 0.0, total-cost: 189745.24]
- -- STREAM_PROJECT |PARTITIONED|
- exchange [cardinality: 601114.0, doc-size: 5.0, op-cost: 0.0, total-cost: 189745.24]
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- data-scan []<-[$$224, $$225, $$l] <- tpch.lineitem [cardinality: 601114.0, doc-size: 5.0, op-cost: 0.0, total-cost: 189745.24]
- -- DATASOURCE_SCAN |PARTITIONED|
- exchange [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- empty-tuple-source [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
- exchange [cardinality: 4491.54, doc-size: 25.0, op-cost: 0.0, total-cost: 59088.81]
- -- BROADCAST_EXCHANGE |PARTITIONED|
- project ([$$222, $$214, $$215, $$216, $$217, $$218, $$219, $$223]) [cardinality: 4491.54, doc-size: 25.0, op-cost: 0.0, total-cost: 59088.81]
+ project ([$$235, $$236, $$222, $$214, $$215, $$216, $$218, $$219, $$228]) [cardinality: 6331.65, doc-size: 25.0, op-cost: 0.0, total-cost: 296273.79]
-- STREAM_PROJECT |PARTITIONED|
- exchange [cardinality: 4491.54, doc-size: 25.0, op-cost: 0.0, total-cost: 59088.81]
+ exchange [cardinality: 6331.65, doc-size: 25.0, op-cost: 0.0, total-cost: 296273.79]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- join (eq($$228, $$226)) [cardinality: 4491.54, doc-size: 25.0, op-cost: 2270.77, total-cost: 59088.81]
- -- HYBRID_HASH_JOIN [$$228][$$226] |PARTITIONED|
- exchange [cardinality: 4491.54, doc-size: 20.0, op-cost: 0.0, total-cost: 56780.52]
+ join (eq($$222, $$233)) [cardinality: 6331.65, doc-size: 25.0, op-cost: 6868.57, total-cost: 296273.79]
+ -- HYBRID_HASH_JOIN [$$222][$$233] |PARTITIONED|
+ exchange [cardinality: 14803.0, doc-size: 10.0, op-cost: 0.0, total-cost: 7429.26]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- project ([$$222, $$214, $$215, $$216, $$218, $$219, $$223, $$228]) [cardinality: 4491.54, doc-size: 20.0, op-cost: 0.0, total-cost: 56780.52]
+ assign [$$219, $$218, $$216, $$215, $$214, $$228] <- [$$c.getField(7), $$c.getField(2), $$c.getField(4), $$c.getField(5), $$c.getField(1), $$c.getField(3)] project: [$$222, $$214, $$215, $$216, $$218, $$219, $$228] [cardinality: 14803.0, doc-size: 10.0, op-cost: 0.0, total-cost: 7429.26]
+ -- ASSIGN |PARTITIONED|
+ exchange [cardinality: 14803.0, doc-size: 10.0, op-cost: 0.0, total-cost: 7429.26]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ data-scan []<-[$$222, $$c] <- tpch.customer [cardinality: 14803.0, doc-size: 10.0, op-cost: 0.0, total-cost: 7429.26]
+ -- DATASOURCE_SCAN |PARTITIONED|
+ exchange [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ empty-tuple-source [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ exchange [cardinality: 6325.7, doc-size: 15.0, op-cost: 0.0, total-cost: 276690.3]
+ -- HASH_PARTITION_EXCHANGE [$$233] |PARTITIONED|
+ project ([$$235, $$236, $$233]) [cardinality: 6325.7, doc-size: 15.0, op-cost: 0.0, total-cost: 276690.3]
-- STREAM_PROJECT |PARTITIONED|
- exchange [cardinality: 4491.54, doc-size: 20.0, op-cost: 0.0, total-cost: 56780.52]
+ exchange [cardinality: 6325.7, doc-size: 15.0, op-cost: 0.0, total-cost: 276690.3]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- join (eq($$222, $$233)) [cardinality: 4491.54, doc-size: 20.0, op-cost: 5948.94, total-cost: 56780.52]
- -- HYBRID_HASH_JOIN [$$222][$$233] |PARTITIONED|
- exchange [cardinality: 14803.0, doc-size: 10.0, op-cost: 0.0, total-cost: 7429.15]
+ join (eq($$225, $$223)) [cardinality: 6325.7, doc-size: 15.0, op-cost: 43939.3, total-cost: 276690.3]
+ -- HYBRID_HASH_JOIN [$$225][$$223] |PARTITIONED|
+ exchange [cardinality: 143068.52, doc-size: 5.0, op-cost: 0.0, total-cost: 187059.17]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- assign [$$219, $$218, $$216, $$215, $$214, $$228] <- [$$c.getField(7), $$c.getField(2), $$c.getField(4), $$c.getField(5), $$c.getField(1), $$c.getField(3)] project: [$$222, $$214, $$215, $$216, $$218, $$219, $$228] [cardinality: 14803.0, doc-size: 10.0, op-cost: 0.0, total-cost: 7429.15]
- -- ASSIGN |PARTITIONED|
- exchange [cardinality: 14803.0, doc-size: 10.0, op-cost: 0.0, total-cost: 7429.15]
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- data-scan []<-[$$222, $$c] <- tpch.customer [cardinality: 14803.0, doc-size: 10.0, op-cost: 0.0, total-cost: 7429.15]
- -- DATASOURCE_SCAN |PARTITIONED|
- exchange [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
+ select (eq($$l.getField(8), "R")) project: [$$235, $$236, $$225] [cardinality: 143068.52, doc-size: 5.0, op-cost: 0.0, total-cost: 187059.17]
+ -- STREAM_SELECT |PARTITIONED|
+ assign [$$236, $$235] <- [$$l.getField(6), $$l.getField(5)] [cardinality: 601114.0, doc-size: 5.0, op-cost: 0.0, total-cost: 187059.17]
+ -- ASSIGN |PARTITIONED|
+ project ([$$225, $$l]) [cardinality: 601114.0, doc-size: 5.0, op-cost: 0.0, total-cost: 187059.17]
+ -- STREAM_PROJECT |PARTITIONED|
+ exchange [cardinality: 601114.0, doc-size: 5.0, op-cost: 0.0, total-cost: 187059.17]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- empty-tuple-source [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
- exchange [cardinality: 4487.32, doc-size: 10.0, op-cost: 0.0, total-cost: 38576.36]
- -- HASH_PARTITION_EXCHANGE [$$233] |PARTITIONED|
- select (and(ge($$220, "1993-07-01"), lt($$220, "1993-10-01"))) project: [$$223, $$233] [cardinality: 4487.32, doc-size: 10.0, op-cost: 0.0, total-cost: 38576.36]
+ data-scan []<-[$$224, $$225, $$l] <- tpch.lineitem [cardinality: 601114.0, doc-size: 5.0, op-cost: 0.0, total-cost: 187059.17]
+ -- DATASOURCE_SCAN |PARTITIONED|
+ exchange [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ empty-tuple-source [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ exchange [cardinality: 6590.74, doc-size: 10.0, op-cost: 0.0, total-cost: 39101.08]
+ -- BROADCAST_EXCHANGE |PARTITIONED|
+ select (and(ge($$220, "1993-07-01"), lt($$220, "1993-10-01"))) project: [$$233, $$223] [cardinality: 6590.74, doc-size: 10.0, op-cost: 0.0, total-cost: 39101.08]
-- STREAM_SELECT |PARTITIONED|
- assign [$$233, $$220] <- [$$o.getField(1), $$o.getField(4)] project: [$$223, $$233, $$220] [cardinality: 149063.0, doc-size: 10.0, op-cost: 0.0, total-cost: 38576.36]
+ assign [$$233, $$220] <- [$$o.getField(1), $$o.getField(4)] project: [$$223, $$233, $$220] [cardinality: 149063.0, doc-size: 10.0, op-cost: 0.0, total-cost: 39101.08]
-- ASSIGN |PARTITIONED|
- exchange [cardinality: 149063.0, doc-size: 10.0, op-cost: 0.0, total-cost: 38576.36]
+ exchange [cardinality: 149063.0, doc-size: 10.0, op-cost: 0.0, total-cost: 39101.08]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- data-scan []<-[$$223, $$o] <- tpch.orders [cardinality: 149063.0, doc-size: 10.0, op-cost: 0.0, total-cost: 38576.36]
+ data-scan []<-[$$223, $$o] <- tpch.orders [cardinality: 149063.0, doc-size: 10.0, op-cost: 0.0, total-cost: 39101.08]
-- DATASOURCE_SCAN |PARTITIONED|
exchange [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
empty-tuple-source [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
-- EMPTY_TUPLE_SOURCE |PARTITIONED|
- exchange [cardinality: 25.0, doc-size: 5.0, op-cost: 0.0, total-cost: 12.52]
- -- BROADCAST_EXCHANGE |PARTITIONED|
- assign [$$217] <- [$$n.getField(1)] project: [$$217, $$226] [cardinality: 25.0, doc-size: 5.0, op-cost: 0.0, total-cost: 12.52]
- -- ASSIGN |PARTITIONED|
- exchange [cardinality: 25.0, doc-size: 5.0, op-cost: 0.0, total-cost: 12.52]
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- data-scan []<-[$$226, $$n] <- tpch.nation [cardinality: 25.0, doc-size: 5.0, op-cost: 0.0, total-cost: 12.52]
- -- DATASOURCE_SCAN |PARTITIONED|
- exchange [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- empty-tuple-source [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
- -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ exchange [cardinality: 25.0, doc-size: 5.0, op-cost: 0.0, total-cost: 12.52]
+ -- BROADCAST_EXCHANGE |PARTITIONED|
+ assign [$$217] <- [$$n.getField(1)] project: [$$217, $$226] [cardinality: 25.0, doc-size: 5.0, op-cost: 0.0, total-cost: 12.52]
+ -- ASSIGN |PARTITIONED|
+ exchange [cardinality: 25.0, doc-size: 5.0, op-cost: 0.0, total-cost: 12.52]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ data-scan []<-[$$226, $$n] <- tpch.nation [cardinality: 25.0, doc-size: 5.0, op-cost: 0.0, total-cost: 12.52]
+ -- DATASOURCE_SCAN |PARTITIONED|
+ exchange [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ empty-tuple-source [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-column/pom.xml b/asterixdb/asterix-column/pom.xml
index a427e2c126..61f6e8be1e 100644
--- a/asterixdb/asterix-column/pom.xml
+++ b/asterixdb/asterix-column/pom.xml
@@ -147,5 +147,17 @@
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-test-support</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-storage-am-lsm-btree-test</artifactId>
+ <version>${hyracks.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
</dependencies>
</project>
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/merge/MergeColumnReadMetadata.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/merge/MergeColumnReadMetadata.java
index 0152f5a908..cf26c281ef 100644
--- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/merge/MergeColumnReadMetadata.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/merge/MergeColumnReadMetadata.java
@@ -18,6 +18,7 @@
*/
package org.apache.asterix.column.operation.lsm.merge;
+import static org.apache.hyracks.storage.am.lsm.btree.column.api.projection.ColumnProjectorType.EXISTENCE;
import static org.apache.hyracks.storage.am.lsm.btree.column.api.projection.ColumnProjectorType.MERGE;
import java.io.ByteArrayInputStream;
@@ -33,7 +34,9 @@ import org.apache.asterix.om.types.ARecordType;
import org.apache.hyracks.data.std.api.IValueReference;
import org.apache.hyracks.data.std.primitive.IntegerPointable;
import org.apache.hyracks.storage.am.lsm.btree.column.api.AbstractColumnTupleReader;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.projection.ColumnProjectorType;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
+import org.apache.hyracks.util.annotations.AiProvenance;
/**
* Merge column read metadata belongs to read an {@link ILSMDiskComponent}
@@ -42,11 +45,27 @@ import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
*/
public final class MergeColumnReadMetadata extends AbstractColumnImmutableReadMetadata {
private final IColumnValuesReader[] columnReaders;
+ /**
+ * How many of {@link #columnReaders} are actually projected: {@code columnReaders.length} normally, and
+ * {@code 0} for the existence-only view produced by {@link #createExistenceOnlyProjectionInfo()}.
+ */
+ private final int numberOfProjectedColumns;
private MergeColumnReadMetadata(ARecordType datasetType, ARecordType metaType, int numberOfPrimaryKeys,
IColumnValuesReader[] columnReaders, IValueReference serializedMetadata) {
- super(datasetType, metaType, numberOfPrimaryKeys, serializedMetadata, columnReaders.length, MERGE);
+ this(datasetType, metaType, numberOfPrimaryKeys, columnReaders, serializedMetadata, columnReaders.length,
+ MERGE);
+ }
+
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_4_8, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.GENERATED, notes = "Constructor taking an explicit projected-column count so an existence-only (key-only) view "
+ + "can share the parsed column readers")
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_4_8, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.REFACTORED, notes = "Projector type is now a parameter so the existence-only view can report EXISTENCE instead of MERGE")
+ private MergeColumnReadMetadata(ARecordType datasetType, ARecordType metaType, int numberOfPrimaryKeys,
+ IColumnValuesReader[] columnReaders, IValueReference serializedMetadata, int numberOfProjectedColumns,
+ ColumnProjectorType projectorType) {
+ super(datasetType, metaType, numberOfPrimaryKeys, serializedMetadata, columnReaders.length, projectorType);
this.columnReaders = columnReaders;
+ this.numberOfProjectedColumns = numberOfProjectedColumns;
}
/**
@@ -87,7 +106,28 @@ public final class MergeColumnReadMetadata extends AbstractColumnImmutableReadMe
@Override
public int getNumberOfProjectedColumns() {
- return columnReaders.length;
+ return numberOfProjectedColumns;
+ }
+
+ @Override
+ public MergeColumnReadMetadata createExistenceOnlyProjectionInfo() {
+ throw new UnsupportedOperationException(
+ getClass().getSimpleName() + " is never the projection an existence probe is built from");
+ }
+
+ /**
+ * A key-only projection over {@code primaryKeyReaders}, for the sample cursor's newer-component liveness
+ * probe. Shaped as a merge projection because {@code MergeColumnTupleReference} is already the tuple
+ * reference that reads nothing but the primary keys
+ *
+ * @param primaryKeyReaders the component's primary-key readers, in key order (column {@code i} at index
+ * {@code i}), as {@code MergeColumnTupleReference#getPrimaryKeyReaders} reads them
+ * @see org.apache.asterix.column.operation.query.QueryColumnMetadata#createExistenceOnlyProjectionInfo()
+ */
+ public static MergeColumnReadMetadata createExistenceOnly(ARecordType datasetType, ARecordType metaType,
+ IValueReference serializedMetadata, IColumnValuesReader[] primaryKeyReaders) {
+ return new MergeColumnReadMetadata(datasetType, metaType, primaryKeyReaders.length, primaryKeyReaders,
+ serializedMetadata, 0, EXISTENCE);
}
@Override
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/QueryColumnMetadata.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/QueryColumnMetadata.java
index 15a6277e49..9327431f46 100644
--- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/QueryColumnMetadata.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/query/QueryColumnMetadata.java
@@ -44,6 +44,7 @@ import org.apache.asterix.column.metadata.AbstractColumnImmutableReadMetadata;
import org.apache.asterix.column.metadata.schema.AbstractSchemaNode;
import org.apache.asterix.column.metadata.schema.ObjectSchemaNode;
import org.apache.asterix.column.metadata.schema.visitor.SchemaClipperVisitor;
+import org.apache.asterix.column.operation.lsm.merge.MergeColumnReadMetadata;
import org.apache.asterix.column.util.SchemaJSONBuilderVisitor;
import org.apache.asterix.column.values.IColumnValuesReader;
import org.apache.asterix.column.values.IColumnValuesReaderFactory;
@@ -59,6 +60,7 @@ import org.apache.hyracks.data.std.api.IValueReference;
import org.apache.hyracks.data.std.primitive.IntegerPointable;
import org.apache.hyracks.storage.am.lsm.btree.column.api.AbstractColumnTupleReader;
import org.apache.hyracks.storage.am.lsm.btree.column.api.projection.ColumnProjectorType;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.projection.IColumnProjectionInfo;
import org.apache.hyracks.util.LogRedactionUtil;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -157,6 +159,12 @@ public class QueryColumnMetadata extends AbstractColumnImmutableReadMetadata {
return assembler.getNumberOfColumns();
}
+ @Override
+ public IColumnProjectionInfo createExistenceOnlyProjectionInfo() {
+ return MergeColumnReadMetadata.createExistenceOnly(getDatasetType(), getMetaType(), serializeColumnsMetadata(),
+ primaryKeyReaders);
+ }
+
@Override
public AbstractColumnTupleReader createTupleReader() {
return new QueryColumnTupleReader(this);
diff --git a/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/sample/CloudExistenceProjectionPinningTest.java b/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/sample/CloudExistenceProjectionPinningTest.java
new file mode 100644
index 0000000000..f3288fd050
--- /dev/null
+++ b/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/sample/CloudExistenceProjectionPinningTest.java
@@ -0,0 +1,280 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.test.sample;
+
+import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.lang.reflect.Proxy;
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.List;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.projection.ColumnProjectorType;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.projection.IColumnProjectionInfo;
+import org.apache.hyracks.storage.am.lsm.btree.column.cloud.buffercache.read.CloudColumnReadContext;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.ColumnBTree;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.ColumnBTreeLeafFrameFactory;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.ColumnBTreeReadLeafFrame;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.lsm.LSMColumnBTreeOpContext;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
+import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.buffercache.ICachedPage;
+import org.apache.hyracks.storage.common.buffercache.context.IBufferCacheReadContext;
+import org.apache.hyracks.storage.common.disk.IPhysicalDrive;
+import org.apache.hyracks.storage.common.file.BufferedFileHandle;
+import org.apache.hyracks.util.annotations.AiProvenance;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Evidence that the existence-only projection changes what the <b>cloud</b> read context pins — the one thing
+ * reading the code cannot establish.
+ * <p>
+ * Zero projected columns is enough on premise ({@code DefaultColumnReadContext#prepareColumns} is a no-op), but
+ * {@link CloudColumnReadContext} branches on the projector <em>type</em> before it looks at the projected set at
+ * all. A key-only view still reporting its parent's type would be silently ignored on cloud storage — the failure
+ * this test catches.
+ * <p>
+ * <b>Runs on premise, still proves a cloud property.</b> Which pages {@code CloudColumnReadContext} asks to pin
+ * follows only from its projection info and leaf frame; a cloud IO manager is needed to <em>service</em> those
+ * pins, not to decide them. So this builds a real component, positions a real {@link ColumnBTreeReadLeafFrame}
+ * on a real mega-leaf, and records what {@code prepareColumns} asks for.
+ * <p>
+ * The recorder does not forward the read context to
+ * {@code IBufferCache#pin(long, IBufferCacheReadContext)}: the cloud context's callbacks cast pages to
+ * {@code CloudCachedPage}, which an on-premise cache never produces. It records the page id and pins through the
+ * contextless overload, so pages are real and unpinned symmetrically.
+ */
+@AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_4_8, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.TEST_GENERATED, notes = "Proves CloudColumnReadContext pins the projected column pages for the full projection and nothing beyond page zero "
+ + "for the existence-only projection")
+public class CloudExistenceProjectionPinningTest {
+
+ private static final Logger LOGGER = LogManager.getLogger();
+
+ /** Small: this test inspects per-leaf pinning decisions, not sampling behaviour. */
+ private static final int NUM_KEYS = 20000;
+ private static final int SHADOW_PCT = 55;
+ private static final int NUM_SHADOW_COMPONENTS = 3;
+
+ /**
+ * The most permissive setting, so nothing observed here can be explained away as the drive declining to
+ * cache. {@code prepareColumns} never consults it anyway — only page persistence does, which the recorder
+ * bypasses.
+ */
+ private static final IPhysicalDrive UNPRESSURED_DRIVE = new IPhysicalDrive() {
+ @Override
+ public boolean computeAndCheckIsPressured() {
+ return false;
+ }
+
+ @Override
+ public boolean isUnpressured() {
+ return true;
+ }
+
+ @Override
+ public boolean hasSpace() {
+ return true;
+ }
+ };
+
+ private final ColumnSampleBenchHarness harness =
+ new ColumnSampleBenchHarness(NUM_KEYS, SHADOW_PCT, NUM_SHADOW_COMPONENTS);
+
+ @Before
+ public void setUp() throws Exception {
+ harness.setUp();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ harness.tearDown();
+ }
+
+ @Test
+ public void existenceOnlyProjectionPinsNothingBeyondPageZeroWhereTheFullProjectionPinsItsColumns()
+ throws Exception {
+ LSMColumnBTreeOpContext opCtx = harness.createSearchOpContext();
+ List<ILSMComponent> components = opCtx.getComponentHolder();
+ ColumnBTree columnBTree = (ColumnBTree) components.get(components.size() - 1).getIndex();
+ IColumnProjectionInfo full = opCtx.createProjectionInfo();
+ Assert.assertNotNull("expected a disk component to supply the sample projection info", full);
+ // What production's sample collector installs: an all-fields query projector reporting MODIFY.
+ Assert.assertEquals(ColumnProjectorType.MODIFY, full.getProjectorType());
+
+ IColumnProjectionInfo existenceOnly = full.createExistenceOnlyProjectionInfo();
+
+ int[] leafPageIds = harness.getSampledComponentLeafPageIds();
+ Assert.assertTrue("expected the sampled component to have at least one mega-leaf page", leafPageIds.length > 0);
+
+ // Aggregated over every mega-leaf of the sampled component, not just one, so the numbers reflect what a
+ // whole sampling pass over this component would pin.
+ int pageZeroOnlyLeaves = 0;
+ int actualFullPins = 0;
+ int actualExistencePins = 0;
+ for (int leafPageId : leafPageIds) {
+ PinTrace fullTrace = prepareColumnsOn(columnBTree, full, leafPageId);
+ PinTrace existenceTrace = prepareColumnsOn(columnBTree, existenceOnly, leafPageId);
+ // The two runs must have seen the same page shape, or the comparison means nothing.
+ Assert.assertEquals(fullTrace.megaLeafPages, existenceTrace.megaLeafPages);
+ Assert.assertEquals(fullTrace.pageZeroSegments, existenceTrace.pageZeroSegments);
+ if (fullTrace.megaLeafPages - fullTrace.pageZeroSegments == 0) {
+ pageZeroOnlyLeaves++;
+ }
+ actualFullPins += fullTrace.pinnedPageIds.size();
+ actualExistencePins += existenceTrace.pinnedPageIds.size();
+ }
+
+ LOGGER.info(
+ "prepareColumns over {} mega-leaf page(s) of the sampled component: the full projection pinned "
+ + "{} page(s), the existence-only projection pinned {}",
+ leafPageIds.length, actualFullPins, actualExistencePins);
+
+ // All-single-page mega-leaves would make the two branches indistinguishable and this pass vacuously.
+ Assert.assertTrue("this test needs at least one multi-page mega-leaf to distinguish the two branches",
+ pageZeroOnlyLeaves < leafPageIds.length);
+ // Not an exact count -- MODIFY pins *coalesced* ranges, so the arithmetic follows this component's
+ // column layout. That it pins anything at all is what makes the zero below meaningful.
+ Assert.assertTrue("the full projection must pin the column pages it projects; pinned " + actualFullPins,
+ actualFullPins > 0);
+ Assert.assertEquals("the existence-only projection must pin nothing beyond page zero", 0, actualExistencePins);
+
+ // Last, deliberately: the projector type is only the mechanism, so if it is ever replaced the pin
+ // counts above are what must keep holding and what should fail first.
+ Assert.assertEquals("the existence-only projection is expected to report EXISTENCE",
+ ColumnProjectorType.EXISTENCE, existenceOnly.getProjectorType());
+ }
+
+ /**
+ * Records only what {@code prepareColumns} pins for one projection on one mega-leaf. The page-zero segment
+ * pins are recorded and discarded first: both branches pin those, and this is about what lies beyond them.
+ */
+ private PinTrace prepareColumnsOn(ColumnBTree columnBTree, IColumnProjectionInfo projectionInfo, int leafPageId)
+ throws HyracksDataException {
+ IBufferCache realBufferCache = harness.getDiskBufferCache();
+ RecordingBufferCache recorder = new RecordingBufferCache(realBufferCache);
+ IBufferCache recordingBufferCache = recorder.asBufferCache();
+ CloudColumnReadContext readContext =
+ new CloudColumnReadContext(projectionInfo, UNPRESSURED_DRIVE, new BitSet());
+ ColumnBTreeLeafFrameFactory leafFrameFactory = (ColumnBTreeLeafFrameFactory) columnBTree.getLeafFrameFactory();
+ ColumnBTreeReadLeafFrame leafFrame = leafFrameFactory.createReadFrame(projectionInfo);
+ int fileId = columnBTree.getFileId();
+ ICachedPage pageZero = realBufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, leafPageId));
+ try {
+ leafFrame.setPage(pageZero);
+ readContext.preparePageZeroSegments(leafFrame, recordingBufferCache, fileId);
+ int pageZeroSegments = leafFrame.getNumberOfPageZeroSegments();
+ int megaLeafPages = leafFrame.getMegaLeafNodeNumberOfPages();
+ recorder.clear();
+ readContext.prepareColumns(leafFrame, recordingBufferCache, fileId);
+ return new PinTrace(megaLeafPages, pageZeroSegments, recorder.pinnedPageIds());
+ } finally {
+ Throwable failure = null;
+ try {
+ readContext.close(recordingBufferCache);
+ } catch (Throwable t) {
+ failure = t;
+ }
+ try {
+ realBufferCache.unpin(pageZero);
+ } catch (Throwable t) {
+ failure = failure == null ? t : failure;
+ }
+ if (failure != null) {
+ throw HyracksDataException.create(failure);
+ }
+ }
+ }
+
+ /** What one {@code prepareColumns} call did, and the page it did it for. */
+ private static final class PinTrace {
+ private final int megaLeafPages;
+ private final int pageZeroSegments;
+ private final List<Integer> pinnedPageIds;
+
+ private PinTrace(int megaLeafPages, int pageZeroSegments, List<Integer> pinnedPageIds) {
+ this.megaLeafPages = megaLeafPages;
+ this.pageZeroSegments = pageZeroSegments;
+ this.pinnedPageIds = pinnedPageIds;
+ }
+ }
+
+ /**
+ * An {@link IBufferCache} that records every context-carrying {@code pin} and services it through the
+ * contextless overload, forwarding everything else to the real cache. Built as a dynamic proxy so it does not
+ * have to restate the ~50 methods of {@link IBufferCache}, only the two whose behaviour matters here.
+ */
+ private static final class RecordingBufferCache implements InvocationHandler {
+ private final IBufferCache delegate;
+ private final List<Integer> pinnedPageIds = new ArrayList<>();
+
+ private RecordingBufferCache(IBufferCache delegate) {
+ this.delegate = delegate;
+ }
+
+ private IBufferCache asBufferCache() {
+ return (IBufferCache) Proxy.newProxyInstance(IBufferCache.class.getClassLoader(),
+ new Class<?>[] { IBufferCache.class }, this);
+ }
+
+ private void clear() {
+ pinnedPageIds.clear();
+ }
+
+ private List<Integer> pinnedPageIds() {
+ return new ArrayList<>(pinnedPageIds);
+ }
+
+ @Override
+ public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
+ if (isPinWithContext(method)) {
+ long dpid = (Long) args[0];
+ pinnedPageIds.add(BufferedFileHandle.getPageId(dpid));
+ return delegate.pin(dpid);
+ }
+ if (isUnpinWithContext(method)) {
+ delegate.unpin((ICachedPage) args[0]);
+ return null;
+ }
+ try {
+ return method.invoke(delegate, args);
+ } catch (InvocationTargetException e) {
+ throw e.getCause();
+ }
+ }
+
+ private static boolean isPinWithContext(Method method) {
+ return "pin".equals(method.getName()) && method.getParameterCount() == 2
+ && method.getParameterTypes()[0] == long.class
+ && method.getParameterTypes()[1] == IBufferCacheReadContext.class;
+ }
+
+ private static boolean isUnpinWithContext(Method method) {
+ return "unpin".equals(method.getName()) && method.getParameterCount() == 2
+ && method.getParameterTypes()[0] == ICachedPage.class
+ && method.getParameterTypes()[1] == IBufferCacheReadContext.class;
+ }
+ }
+}
diff --git a/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/sample/ColumnSampleBenchHarness.java b/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/sample/ColumnSampleBenchHarness.java
new file mode 100644
index 0000000000..24bcc2ad6c
--- /dev/null
+++ b/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/sample/ColumnSampleBenchHarness.java
@@ -0,0 +1,697 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.test.sample;
+
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.asterix.builders.RecordBuilder;
+import org.apache.asterix.column.ColumnManagerFactory;
+import org.apache.asterix.column.filter.NoOpColumnFilterEvaluatorFactory;
+import org.apache.asterix.column.operation.query.QueryColumnTupleProjector;
+import org.apache.asterix.common.exceptions.NoOpWarningCollector;
+import org.apache.asterix.formats.nontagged.BinaryComparatorFactoryProvider;
+import org.apache.asterix.formats.nontagged.NullIntrospector;
+import org.apache.asterix.formats.nontagged.TypeTraitProvider;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.BuiltinType;
+import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.om.utils.ProjectionFiltrationTypeUtil;
+import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import org.apache.hyracks.api.dataflow.value.ITypeTraits;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.IIOManager;
+import org.apache.hyracks.api.util.CleanupUtils;
+import org.apache.hyracks.api.util.HyracksConstants;
+import org.apache.hyracks.data.std.primitive.IntegerPointable;
+import org.apache.hyracks.data.std.primitive.LongPointable;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleReference;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
+import org.apache.hyracks.storage.am.btree.impls.BTreeOpContext;
+import org.apache.hyracks.storage.am.btree.impls.RangePredicate;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexCursor;
+import org.apache.hyracks.storage.am.common.impls.IndexAccessParameters;
+import org.apache.hyracks.storage.am.common.impls.NoOpIndexAccessParameters;
+import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback;
+import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
+import org.apache.hyracks.storage.am.common.util.ResourceReleaseUtils;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnManagerFactory;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.projection.ColumnProjectorType;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.projection.IColumnProjectionInfo;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.projection.IColumnTupleProjector;
+import org.apache.hyracks.storage.am.lsm.btree.column.cloud.buffercache.IColumnReadContext;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.ColumnBTree;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.ColumnBtreeSampleCursor;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.SampleCursorStats;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.lsm.LSMColumnBTreeOpContext;
+import org.apache.hyracks.storage.am.lsm.btree.column.utils.LSMColumnBTreeUtil;
+import org.apache.hyracks.storage.am.lsm.btree.impls.AntimatterAwareTupleAcceptor;
+import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTree;
+import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeBatchPointSearchCursor;
+import org.apache.hyracks.storage.am.lsm.btree.util.LSMBTreeTestHarness;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
+import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
+import org.apache.hyracks.storage.am.lsm.common.impls.DiskComponentMetadata;
+import org.apache.hyracks.storage.am.lsm.common.impls.VirtualBufferCache;
+import org.apache.hyracks.storage.common.buffercache.BufferCache;
+import org.apache.hyracks.storage.common.buffercache.ClockPageReplacementStrategy;
+import org.apache.hyracks.storage.common.buffercache.ColumnBufferPool;
+import org.apache.hyracks.storage.common.buffercache.DefaultDiskCachedPageAllocator;
+import org.apache.hyracks.storage.common.buffercache.DelayPageCleanerPolicy;
+import org.apache.hyracks.storage.common.buffercache.HeapBufferAllocator;
+import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.buffercache.IColumnBufferPool;
+import org.apache.hyracks.storage.common.buffercache.IPageReplacementStrategy;
+import org.apache.hyracks.storage.common.buffercache.context.read.DefaultBufferCacheReadContextProvider;
+import org.apache.hyracks.storage.common.compression.NoOpCompressorDecompressorFactory;
+import org.apache.hyracks.storage.common.disk.NoOpDiskCacheMonitoringService;
+import org.apache.hyracks.storage.common.file.FileMapManager;
+import org.apache.hyracks.storage.common.file.IFileMapManager;
+import org.apache.hyracks.util.annotations.AiProvenance;
+import org.apache.hyracks.util.trace.ITracer;
+
+/**
+ * Stands up a real column LSM BTree whose oldest disk component is partially shadowed by newer ones, and drives
+ * the real {@code ColumnBtreeSampleCursor} over it. Setup and cursor driving only — assertions belong in the
+ * tests that consume this.
+ * <p>
+ * <b>Why not just use {@link LSMBTreeTestHarness}?</b> It is configured for 256-byte pages and a ~256 KB total
+ * budget ({@code AccessMethodTestsConfig.LSM_BTREE_*}); a column mega-leaf does not fit in 256 bytes and the
+ * budget cannot hold the records this bench needs. So the page-size-dependent pieces — disk and virtual buffer
+ * caches — are built locally with column-sized pages, and everything else is taken from the shared harness,
+ * which is never modified.
+ * <p>
+ * <b>Fidelity contract — no real disk I/O.</b> The disk buffer cache is sized from the tuples the index actually
+ * <em>stores</em>, {@code numKeys + }{@link #getShadowBoundary()}: every shadowed key is stored twice, so at
+ * {@code shadowPercent = 100} the index holds {@code 2 × numKeys}. The whole index therefore stays resident and
+ * every page the sampler touches is a cache hit — which is what makes per-phase attribution stable enough to
+ * compare runs, but also means wall times here are a lower bound on production and nothing of the form "phase X
+ * is I/O bound" can be concluded. {@code ColumnSampleBenchHarnessResidencyTest} enforces it against the real
+ * allocation counters rather than trusting the formula.
+ * <p>
+ * <b>Scaling:</b> each insert round must fit one in-memory component, so the virtual buffer caches are sized
+ * from {@code numKeys} ({@link #memoryPagesFor(int)}); an overflow would auto-flush mid-round and change the
+ * shadow structure's shape. Memory runs ~{@code 2 * numKeys * 256 B}, so 600 000 keys holds ~300 MB against the
+ * 4 GB test heap.
+ */
+@AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_4_8, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.TEST_GENERATED, notes = "Column LSM sample bench harness: builds a shadowed column LSM and drives the column sample cursor")
+public class ColumnSampleBenchHarness {
+
+ /** Column pages are buffer-cache pages; 32 KB matches the product default. */
+ public static final int PAGE_SIZE = 32 * 1024;
+ /** Floors for the buffer budgets; both scale up with {@code numKeys} (see {@link #memoryPagesFor(int)}). */
+ static final int MIN_DISK_NUM_PAGES = 1024;
+ static final int MIN_MEM_NUM_PAGES = 256;
+ /**
+ * Deliberately generous: measured well under 128 B (tagged PK + ~50-byte closed record + slot overhead),
+ * doubled for margin. A round that does not fit one memory component auto-flushes mid-round and changes the
+ * shadow structure's shape, so this is sized to guarantee fit, not to be tight.
+ */
+ private static final int MEMORY_BYTES_PER_TUPLE = 256;
+ /**
+ * On-disk per-<b>stored</b>-tuple budget, keeping the index buffer-cache resident. Derived from the footprint
+ * {@link #getIndexPageCount()} actually measures, not estimated: 20.5-23.0 B observed across
+ * {@code shadowPercent} 55/90/100 × {@code numShadowComponents} 3/6/8 at 20k/165k/650k keys (the column
+ * encoding compresses the ascending int64 PK hard), rounded up to 32 B for a denser record shape, then a 3×
+ * safety factor. An unmeasured configuration would have to be 4× heavier per tuple to threaten residency.
+ * <p>
+ * Applied to {@code numKeys + shadowBoundary}, not {@code numKeys}: shadowed keys are stored twice. A sizing
+ * input only — the contract is enforced against the real allocator, not this estimate.
+ */
+ private static final int DISK_BYTES_PER_STORED_TUPLE = 96;
+ private static final int NUM_MUTABLE_COMPONENTS = 2;
+ private static final int MAX_OPEN_FILES = Integer.MAX_VALUE;
+ private static final int IO_QUEUE_LEN = 10;
+ private static final double BLOOM_FILTER_FALSE_POSITIVE_RATE = 0.01;
+
+ // Column manager knobs (mirrors DatasetFormatInfo defaults closely enough for a bench).
+ private static final int MAX_TUPLE_COUNT = 15000;
+ private static final double TOLERANCE = 0.15;
+ private static final int MAX_LEAF_NODE_SIZE = 8 * PAGE_SIZE;
+
+ // Granule matched to PAGE_SIZE so one credit is one page. Pool size and memory doubled over the brief's
+ // guess because ALL_COLUMNS loads whole column mega-pages per sampled page, genuinely exercising the pool.
+ // Timeout at 2 minutes so credit pressure on a slow CI machine cannot become a spurious mid-run failure.
+ private static final int COLUMN_BUFFER_GRANULE_BYTES = PAGE_SIZE;
+ private static final int COLUMN_BUFFER_POOL_SIZE = 128;
+ private static final long COLUMN_BUFFER_MAX_MEMORY = 128L * 1024 * 1024;
+ private static final long COLUMN_BUFFER_RESERVE_TIMEOUT_MILLIS = TimeUnit.MINUTES.toMillis(2);
+
+ // Sample cursor knobs, defaulted to the values production ships in StorageProperties so the baseline this
+ // harness measures is the one the product runs. The cursor raises the attempt ceiling itself to
+ // max(configured, 32 * max(leafPages, target)).
+ // Matches StorageProperties.STORAGE_MAX_SAMPLE_LEAF_ATTEMPTS's default.
+ private static final int DEFAULT_MAX_LEAF_ATTEMPTS = 500;
+ // Matches StorageProperties.STORAGE_SAMPLE_LEAF_DRAW_BATCH_SIZE's default.
+ private static final int DEFAULT_LEAF_DRAW_BATCH_SIZE = 32768;
+
+ private static final String PK_FIELD = "id";
+ private static final String PAYLOAD_FIELD = "payload";
+ private static final String PAYLOAD_PREFIX = "payload-value-";
+
+ private final int numKeys;
+ private final int shadowPercent;
+ private final int numShadowComponents;
+
+ private final LSMBTreeTestHarness harness = new LSMBTreeTestHarness();
+
+ // Tuple construction scratch state (single-threaded by construction).
+ private final RecordBuilder recordBuilder = new RecordBuilder();
+ private final UTF8StringSerializerDeserializer stringSerde = new UTF8StringSerializerDeserializer();
+ private final ArrayBackedValueStorage fieldValue = new ArrayBackedValueStorage();
+ private final ArrayTupleBuilder insertTupleBuilder = new ArrayTupleBuilder(2);
+ private final ArrayTupleReference insertTuple = new ArrayTupleReference();
+ private final ArrayTupleBuilder deleteTupleBuilder = new ArrayTupleBuilder(1);
+ private final ArrayTupleReference deleteTuple = new ArrayTupleReference();
+
+ private final Projection projection;
+
+ private int maxLeafAttempts = DEFAULT_MAX_LEAF_ATTEMPTS;
+ private int leafDrawBatchSize = DEFAULT_LEAF_DRAW_BATCH_SIZE;
+ /** {@code < 0} means "read the real value from the sampled component's metadata", as production does. */
+ private int maxLeafTupleCountOverride = -1;
+
+ private ARecordType recordType;
+ private IBufferCache diskBufferCache;
+ private IPageReplacementStrategy pageReplacementStrategy;
+ private IColumnBufferPool columnBufferPool;
+ private List<IVirtualBufferCache> virtualBufferCaches;
+ private LSMBTree lsmBtree;
+
+ /**
+ * What the sample scan projects out of each sampled row, i.e. how much column data phase 2 must materialize.
+ */
+ public enum Projection {
+ /**
+ * Every column, via the same projector production's sample collector installs: a
+ * {@code QueryColumnTupleProjector} over {@link ProjectionFiltrationTypeUtil#ALL_FIELDS_TYPE} reporting
+ * {@link ColumnProjectorType#MODIFY}. That is what
+ * {@code BTreeSampleCollectorOperatorDescriptorNodePushable} ends up with:
+ * {@code BTreeSearchOperatorNodePushable#addAdditionalIndexAccessorParams} puts the descriptor's tuple
+ * projector under {@code HyracksConstants.TUPLE_PROJECTOR}, and for a columnar dataset
+ * {@code IndexUtil#createPrimaryIndexScanTupleProjectorFactory} makes that a
+ * {@code PrimaryScanColumnTupleProjectorFactory}, so {@code ColumnUtil#getTupleProjector} finds a
+ * projector and never falls back to {@code columnManager.getMergeColumnProjector()}. The sample index
+ * stores whole records, hence all fields. This is the faithful baseline.
+ */
+ ALL_COLUMNS,
+ /**
+ * Primary key only. Phase 2 then loads no column mega-pages, which isolates phase 1 (leaf draws + liveness
+ * + PK seeks) but makes {@code livenessSharePct()} an upper bound rather than a real share.
+ */
+ PK_ONLY
+ }
+
+ /** Equivalent to {@code ColumnSampleBenchHarness(numKeys, shadowPercent, numShadowComponents, ALL_COLUMNS)}. */
+ public ColumnSampleBenchHarness(int numKeys, int shadowPercent, int numShadowComponents) {
+ this(numKeys, shadowPercent, numShadowComponents, Projection.ALL_COLUMNS);
+ }
+
+ public ColumnSampleBenchHarness(int numKeys, int shadowPercent, int numShadowComponents, Projection projection) {
+ this.numKeys = numKeys;
+ this.shadowPercent = shadowPercent;
+ this.numShadowComponents = numShadowComponents;
+ this.projection = projection;
+ }
+
+ public void setUp() throws Exception {
+ // Note: the shared harness's own 256-byte-page disk buffer cache is created here and left unused (it is
+ // closed again by harness.tearDown()); only its page-size-independent pieces are consumed below.
+ harness.setUp();
+ IIOManager ioManager = harness.getIOManager();
+ diskBufferCache = createDiskBufferCache(ioManager, diskPagesFor((long) numKeys + getShadowBoundary()));
+ columnBufferPool = new ColumnBufferPool(COLUMN_BUFFER_GRANULE_BYTES, COLUMN_BUFFER_POOL_SIZE,
+ COLUMN_BUFFER_MAX_MEMORY, COLUMN_BUFFER_RESERVE_TIMEOUT_MILLIS);
+ virtualBufferCaches = new ArrayList<>();
+ int memPages = memoryPagesFor(numKeys);
+ for (int i = 0; i < NUM_MUTABLE_COMPONENTS; i++) {
+ virtualBufferCaches.add(new VirtualBufferCache(new HeapBufferAllocator(), PAGE_SIZE, memPages));
+ }
+
+ recordType = new ARecordType("bench", new String[] { PK_FIELD, PAYLOAD_FIELD },
+ new IAType[] { BuiltinType.AINT64, BuiltinType.ASTRING }, false);
+ ITypeTraits[] typeTraits = new ITypeTraits[] { TypeTraitProvider.INSTANCE.getTypeTrait(BuiltinType.AINT64),
+ TypeTraitProvider.INSTANCE.getTypeTrait(recordType) };
+ IBinaryComparatorFactory[] cmpFactories = new IBinaryComparatorFactory[] {
+ BinaryComparatorFactoryProvider.INSTANCE.getBinaryComparatorFactory(BuiltinType.AINT64, true) };
+ int[] bloomFilterKeyFields = new int[] { 0 };
+
+ IColumnManagerFactory columnManagerFactory = new ColumnManagerFactory(recordType, null,
+ Collections.singletonList(Collections.singletonList(PK_FIELD)), Collections.singletonList(0), PAGE_SIZE,
+ MAX_TUPLE_COUNT, TOLERANCE, MAX_LEAF_NODE_SIZE);
+
+ lsmBtree = LSMColumnBTreeUtil.createLSMTree(harness.getNCConfig(), ioManager, virtualBufferCaches,
+ harness.getFileReference(), diskBufferCache, columnBufferPool, typeTraits, cmpFactories,
+ bloomFilterKeyFields, BLOOM_FILTER_FALSE_POSITIVE_RATE, harness.getMergePolicy(),
+ harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallbackFactory(),
+ harness.getPageWriteCallbackFactory(), null /* btreeFields */, harness.getMetadataPageManagerFactory(),
+ false /* updateAware */, ITracer.NONE, NoOpCompressorDecompressorFactory.INSTANCE,
+ TypeTraitProvider.INSTANCE.getTypeTrait(BuiltinType.ANULL), NullIntrospector.INSTANCE,
+ columnManagerFactory, false /* atomic */, NoOpDiskCacheMonitoringService.INSTANCE);
+ lsmBtree.create();
+ lsmBtree.activate();
+ buildComponents();
+ }
+
+ /**
+ * Builds the base disk component (all {@code numKeys} keys) followed by {@code numShadowComponents} newer disk
+ * components that delete-and-reinsert the first {@code numKeys * shadowPercent / 100} keys, so those keys are
+ * shadowed in the newer components and must not be emitted when the oldest component is sampled.
+ */
+ private void buildComponents() throws HyracksDataException {
+ ILSMIndexAccessor accessor = lsmBtree.createAccessor(NoOpIndexAccessParameters.INSTANCE);
+ for (int key = 0; key < numKeys; key++) {
+ insert(accessor, key);
+ }
+ accessor.scheduleFlush();
+
+ int shadowBoundary = getShadowBoundary();
+ int perComponent = numShadowComponents == 0 ? 0 : (int) divideRoundingUp(shadowBoundary, numShadowComponents);
+ for (int component = 0; component < numShadowComponents; component++) {
+ int from = component * perComponent;
+ int to = Math.min(shadowBoundary, from + perComponent);
+ for (int key = from; key < to; key++) {
+ delete(accessor, key);
+ insert(accessor, key);
+ }
+ accessor.scheduleFlush();
+ }
+ }
+
+ /** Keys in {@code [0, shadowBoundary)} are shadowed by a newer component; keys at or above it stay live. */
+ public int getShadowBoundary() {
+ return numKeys * shadowPercent / 100;
+ }
+
+ private static long divideRoundingUp(long dividend, long divisor) {
+ return (dividend + divisor - 1) / divisor;
+ }
+
+ private void insert(ILSMIndexAccessor accessor, int key) throws HyracksDataException {
+ insertTupleBuilder.reset();
+ writeTaggedInt64(insertTupleBuilder.getDataOutput(), key);
+ insertTupleBuilder.addFieldEndOffset();
+ writeRecord(insertTupleBuilder.getDataOutput(), key);
+ insertTupleBuilder.addFieldEndOffset();
+ insertTuple.reset(insertTupleBuilder.getFieldEndOffsets(), insertTupleBuilder.getByteArray());
+ accessor.insert(insertTuple);
+ }
+
+ private void delete(ILSMIndexAccessor accessor, int key) throws HyracksDataException {
+ deleteTupleBuilder.reset();
+ writeTaggedInt64(deleteTupleBuilder.getDataOutput(), key);
+ deleteTupleBuilder.addFieldEndOffset();
+ deleteTuple.reset(deleteTupleBuilder.getFieldEndOffsets(), deleteTupleBuilder.getByteArray());
+ accessor.delete(deleteTuple);
+ }
+
+ private void writeRecord(DataOutput out, int key) throws HyracksDataException {
+ recordBuilder.reset(recordType);
+ recordBuilder.init();
+ fieldValue.reset();
+ writeTaggedInt64(fieldValue.getDataOutput(), key);
+ recordBuilder.addField(0, fieldValue);
+ fieldValue.reset();
+ writeTaggedString(fieldValue.getDataOutput(), PAYLOAD_PREFIX + key);
+ recordBuilder.addField(1, fieldValue);
+ recordBuilder.write(out, true);
+ }
+
+ private static void writeTaggedInt64(DataOutput out, long value) throws HyracksDataException {
+ try {
+ out.writeByte(ATypeTag.SERIALIZED_INT64_TYPE_TAG);
+ out.writeLong(value);
+ } catch (IOException e) {
+ throw HyracksDataException.create(e);
+ }
+ }
+
+ private void writeTaggedString(DataOutput out, String value) throws HyracksDataException {
+ try {
+ out.writeByte(ATypeTag.SERIALIZED_STRING_TYPE_TAG);
+ stringSerde.serialize(value, out);
+ } catch (IOException e) {
+ throw HyracksDataException.create(e);
+ }
+ }
+
+ public void tearDown() throws Exception {
+ try {
+ if (lsmBtree != null) {
+ lsmBtree.deactivate();
+ lsmBtree.destroy();
+ }
+ } finally {
+ try {
+ if (columnBufferPool != null) {
+ columnBufferPool.close();
+ }
+ if (diskBufferCache != null) {
+ diskBufferCache.close();
+ }
+ } finally {
+ harness.tearDown();
+ }
+ }
+ }
+
+ /** Result of one sample drive over the oldest disk component. */
+ public static final class SampleRun {
+ public final int emitted;
+ public final int[] sampledKeys;
+ public final long wallNanos;
+
+ SampleRun(int emitted, int[] sampledKeys, long wallNanos) {
+ this.emitted = emitted;
+ this.sampledKeys = sampledKeys;
+ this.wallNanos = wallNanos;
+ }
+ }
+
+ /**
+ * Drives the real column sample cursor over the <b>oldest</b> disk component with the cross-component liveness
+ * search pointed at <b>all</b> newer components — the post-mutation shape whose cost is under study. Mirrors
+ * {@link org.apache.hyracks.storage.am.lsm.btree.impls.LSMIndexSampleCursor}'s wiring for the last component.
+ *
+ * @param targetCardinality number of live tuples the sampler should try to collect
+ * @param seed sample seed
+ * @param stats cost-breakdown holder attached to the cursor; pass a fresh instance per run
+ */
+ public SampleRun runSample(int targetCardinality, long seed, SampleCursorStats stats) throws HyracksDataException {
+ LSMColumnBTreeOpContext opCtx = createSearchOpContext();
+ List<ILSMComponent> components = opCtx.getComponentHolder(); // newest first
+ int oldestIndex = components.size() - 1;
+ ILSMComponent oldest = components.get(oldestIndex);
+ List<ILSMComponent> newer = new ArrayList<>(components.subList(0, oldestIndex));
+ RangePredicate searchPred = new RangePredicate(null, null, true, true, null, null);
+
+ // The cursor cannot emit more than the target, so the result array is exactly sized.
+ int[] sampledKeys = new int[Math.max(1, targetCardinality)];
+ int emitted = 0;
+ long startNanos = 0;
+ long wallNanos = 0;
+ // Everything acquired below is released in the finally; acquisition happens INSIDE the try so a failure
+ // part-way through cannot leak a cursor holding pinned pages (which would in turn make the buffer-cache
+ // close in tearDown() complain about unpinned pages and bury the original failure).
+ LSMBTreeBatchPointSearchCursor searchCursor = null;
+ ColumnBTree.ColumnBTreeAccessor sampleAccessor = null;
+ ITreeIndexCursor sampleCursor = null;
+ try {
+ opCtx.getSearchInitialState().reset(searchPred, newer);
+ // The same factory LSMIndexSampleCursor uses, so the harness measures the liveness cursor
+ // production actually gets (existence-only, key-only projection) rather than the query cursor.
+ searchCursor = lsmBtree.createSampleLivenessSearchCursor(opCtx);
+ searchCursor.open(opCtx.getSearchInitialState(), searchPred);
+
+ ColumnBTree columnBTree = (ColumnBTree) oldest.getIndex();
+ sampleAccessor = (ColumnBTree.ColumnBTreeAccessor) columnBTree
+ .createAccessor(NoOpIndexAccessParameters.INSTANCE, opCtx, oldestIndex);
+ sampleCursor = sampleAccessor.createSampleCursor(targetCardinality, seed, searchCursor, maxLeafAttempts,
+ leafDrawBatchSize, resolveMaxLeafTupleCount(oldest), AntimatterAwareTupleAcceptor.INSTANCE);
+ ((ColumnBtreeSampleCursor) sampleCursor).setStats(stats);
+
+ startNanos = System.nanoTime();
+ sampleAccessor.diskSampleScan(sampleCursor);
+ while (sampleCursor.hasNext()) {
+ sampleCursor.next();
+ ITupleReference tuple = sampleCursor.getTuple();
+ // Field 0 is the tagged primary key: one type-tag byte followed by the 8-byte value.
+ long key = LongPointable.getLong(tuple.getFieldData(0), tuple.getFieldStart(0) + 1);
+ sampledKeys[emitted++] = (int) key;
+ }
+ // Recorded before teardown so the reported wall time covers the scan only.
+ wallNanos = System.nanoTime() - startNanos;
+ } finally {
+ Throwable failure = null;
+ failure = ResourceReleaseUtils.close(sampleCursor, failure);
+ failure = CleanupUtils.destroy(failure, sampleCursor, sampleAccessor);
+ failure = ResourceReleaseUtils.close(searchCursor, failure);
+ failure = CleanupUtils.destroy(failure, searchCursor);
+ if (failure != null) {
+ throw HyracksDataException.create(failure);
+ }
+ }
+ return new SampleRun(emitted, Arrays.copyOf(sampledKeys, emitted), wallNanos);
+ }
+
+ /**
+ * Number of mega-leaf pages in the oldest (sampled) disk component. The sampler draws uniformly over these, so
+ * this is the population size a leaf-page uniformity check must be computed against.
+ */
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_4_8, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.REFACTORED, notes = "Release the accessor and read context in a finally, matching the cleanup pattern runSample already uses")
+ public int getSampledComponentLeafPageCount() throws HyracksDataException {
+ return getSampledComponentLeafPageIds().length;
+ }
+
+ /**
+ * The mega-leaf page ids of the oldest (sampled) disk component, in key order — the same enumeration the
+ * sample cursor draws from. Exposed so a test can position a read leaf frame on a real mega-leaf page and
+ * observe what a given {@link IColumnReadContext} pins for it.
+ */
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_4_8, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.TEST_GENERATED, notes = "Expose the sampled component's leaf page ids; getSampledComponentLeafPageCount now delegates here")
+ public int[] getSampledComponentLeafPageIds() throws HyracksDataException {
+ LSMColumnBTreeOpContext opCtx = createSearchOpContext();
+ List<ILSMComponent> components = opCtx.getComponentHolder();
+ int oldestIndex = components.size() - 1;
+ ColumnBTree columnBTree = (ColumnBTree) components.get(oldestIndex).getIndex();
+ IColumnProjectionInfo projectionInfo = opCtx.createProjectionInfo();
+ IColumnReadContext readContext = opCtx.createPageZeroContext(projectionInfo);
+ ColumnBTree.ColumnBTreeAccessor accessor = (ColumnBTree.ColumnBTreeAccessor) columnBTree
+ .createAccessor(NoOpIndexAccessParameters.INSTANCE, oldestIndex, projectionInfo, readContext);
+ try {
+ BTreeOpContext btreeOpCtx = accessor.getOpContext();
+ btreeOpCtx.reset();
+ return columnBTree.enumerateLeafPageIds(columnBTree.getRootPageId(), btreeOpCtx, readContext);
+ } finally {
+ // Currently safe either way (leaf-page enumeration unpins symmetrically and no columns are
+ // prepared), but release explicitly so this stays safe if the method is later extended to do more
+ // than enumerate leaf page ids.
+ Throwable failure = CleanupUtils.destroy(null, accessor);
+ try {
+ readContext.close(diskBufferCache);
+ } catch (Throwable t) {
+ failure = failure == null ? t : failure;
+ }
+ if (failure != null) {
+ throw HyracksDataException.create(failure);
+ }
+ }
+ }
+
+ /**
+ * Search op context carrying the configured tuple projector, with the operational components resolved
+ * (newest first). Public so tests can inspect what the sampling wiring derives from it (the projection info
+ * and the cursors built off it) without duplicating the setup.
+ */
+ public LSMColumnBTreeOpContext createSearchOpContext() throws HyracksDataException {
+ IndexAccessParameters iap =
+ new IndexAccessParameters(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
+ // Both modes install a projector, because production's sample collector does: see Projection.ALL_COLUMNS.
+ // Installing nothing would exercise columnManager.getMergeColumnProjector(), which on this path only the
+ // row-format collector (DefaultTupleProjectorFactory, not an IColumnTupleProjector) ever reaches.
+ iap.getParameters().put(HyracksConstants.TUPLE_PROJECTOR,
+ projection == Projection.PK_ONLY ? createPrimaryKeyOnlyProjector() : createAllColumnsProjector());
+ LSMColumnBTreeOpContext opCtx = (LSMColumnBTreeOpContext) lsmBtree.createOpContext(iap);
+ opCtx.setOperation(IndexOperation.SEARCH);
+ lsmBtree.getOperationalComponents(opCtx);
+ return opCtx;
+ }
+
+ /**
+ * Mirrors {@code PrimaryScanColumnTupleProjector} — what
+ * {@code IndexUtil#createPrimaryIndexScanTupleProjectorFactory} builds for a columnar dataset — which is
+ * package-private, hence rebuilt here rather than reused.
+ */
+ private IColumnTupleProjector createAllColumnsProjector() {
+ return new QueryColumnTupleProjector(recordType, 1 /* numberOfPrimaryKeys */,
+ ProjectionFiltrationTypeUtil.ALL_FIELDS_TYPE, Collections.emptyMap(),
+ NoOpColumnFilterEvaluatorFactory.INSTANCE, NoOpColumnFilterEvaluatorFactory.INSTANCE,
+ NoOpWarningCollector.INSTANCE, null /* no task context: no filters are evaluated */,
+ ColumnProjectorType.MODIFY);
+ }
+
+ private IColumnTupleProjector createPrimaryKeyOnlyProjector() {
+ return new QueryColumnTupleProjector(recordType, 1 /* numberOfPrimaryKeys */,
+ ProjectionFiltrationTypeUtil.EMPTY_TYPE, Collections.emptyMap(),
+ NoOpColumnFilterEvaluatorFactory.INSTANCE, NoOpColumnFilterEvaluatorFactory.INSTANCE,
+ NoOpWarningCollector.INSTANCE, null /* no task context: no filters are evaluated */,
+ ColumnProjectorType.QUERY);
+ }
+
+ /**
+ * Per-component max leaf tuple count, read from the sampled component's metadata exactly as
+ * {@link org.apache.hyracks.storage.am.lsm.btree.impls.LSMIndexSampleCursor} does. The cursor uses it as the
+ * denominator of the fill-proportional (Olken) page-acceptance probability, so passing a real value is what
+ * makes {@code stats.pagesRejected} meaningful and matches the estimator production ships.
+ *
+ * @return the override set via {@link #setMaxLeafTupleCount(int)} if any, else the component's recorded value,
+ * else 0 (fill-rejection disabled) when the component carries no such metadata
+ */
+ private int resolveMaxLeafTupleCount(ILSMComponent component) throws HyracksDataException {
+ if (maxLeafTupleCountOverride >= 0) {
+ return maxLeafTupleCountOverride;
+ }
+ DiskComponentMetadata metadata = (DiskComponentMetadata) component.getMetadata();
+ ArrayBackedValueStorage maxTupleCountRef = new ArrayBackedValueStorage();
+ if (metadata.get(DiskComponentMetadata.MAX_LEAF_TUPLE_COUNT_KEY, maxTupleCountRef)
+ && maxTupleCountRef.getLength() >= Integer.BYTES) {
+ return IntegerPointable.getInteger(maxTupleCountRef.getByteArray(), maxTupleCountRef.getStartOffset());
+ }
+ return 0;
+ }
+
+ public LSMBTree getIndex() {
+ return lsmBtree;
+ }
+
+ /**
+ * The column-sized disk buffer cache the components live in. Exposed so a test can pin real mega-leaf pages
+ * directly instead of going through a cursor.
+ */
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_4_8, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.TEST_GENERATED, notes = "Expose the disk buffer cache for a direct page-pinning test")
+ public IBufferCache getDiskBufferCache() {
+ return diskBufferCache;
+ }
+
+ public int getNumKeys() {
+ return numKeys;
+ }
+
+ public int getShadowPercent() {
+ return shadowPercent;
+ }
+
+ public int getNumShadowComponents() {
+ return numShadowComponents;
+ }
+
+ public int getLeafDrawBatchSize() {
+ return leafDrawBatchSize;
+ }
+
+ /** Overrides the sampler's leaf-draw batch size for the next {@link #runSample} (batch-size sweeps). */
+ public void setLeafDrawBatchSize(int leafDrawBatchSize) {
+ this.leafDrawBatchSize = leafDrawBatchSize;
+ }
+
+ public int getMaxLeafAttempts() {
+ return maxLeafAttempts;
+ }
+
+ public void setMaxLeafAttempts(int maxLeafAttempts) {
+ this.maxLeafAttempts = maxLeafAttempts;
+ }
+
+ public Projection getProjection() {
+ return projection;
+ }
+
+ /**
+ * Forces the sampler's {@code maxLeafTupleCount}, overriding the value read from the sampled component's
+ * metadata. Pass {@code 0} to disable fill-proportional page rejection entirely, or a negative value to go
+ * back to the production behaviour of reading it from the component.
+ */
+ public void setMaxLeafTupleCount(int maxLeafTupleCount) {
+ this.maxLeafTupleCountOverride = maxLeafTupleCount;
+ }
+
+ /** The {@code maxLeafTupleCount} the next {@link #runSample} will pass to the cursor. */
+ public int getEffectiveMaxLeafTupleCount() throws HyracksDataException {
+ List<ILSMComponent> components = createSearchOpContext().getComponentHolder();
+ return resolveMaxLeafTupleCount(components.get(components.size() - 1));
+ }
+
+ /**
+ * Virtual-buffer-cache pages per mutable component, sized so one whole insert round fits in one memory
+ * component (otherwise the LSM auto-flushes mid-round: the column flush path then reaches
+ * {@code ColumnBTree.createAccessor(IIndexAccessParameters)} and fails outright).
+ */
+ static int memoryPagesFor(int numKeys) {
+ return (int) Math.max(MIN_MEM_NUM_PAGES, divideRoundingUp(numKeys * MEMORY_BYTES_PER_TUPLE, PAGE_SIZE));
+ }
+
+ /**
+ * Disk-buffer-cache pages, sized to keep every component resident (see the fidelity contract above).
+ *
+ * @param storedTuples total tuples across all components, i.e. {@code numKeys + shadowBoundary}
+ */
+ static int diskPagesFor(long storedTuples) {
+ return (int) Math.max(MIN_DISK_NUM_PAGES,
+ divideRoundingUp(storedTuples * DISK_BYTES_PER_STORED_TUPLE, PAGE_SIZE));
+ }
+
+ private IBufferCache createDiskBufferCache(IIOManager ioManager, int diskNumPages) {
+ pageReplacementStrategy = new ClockPageReplacementStrategy(new HeapBufferAllocator(),
+ DefaultDiskCachedPageAllocator.INSTANCE, PAGE_SIZE, diskNumPages);
+ IFileMapManager fileMapManager = new FileMapManager();
+ ThreadFactory threadFactory = Thread::new;
+ return new BufferCache(ioManager, pageReplacementStrategy, new DelayPageCleanerPolicy(1000), fileMapManager,
+ MAX_OPEN_FILES, IO_QUEUE_LEN, threadFactory, new HashMap<>(),
+ DefaultBufferCacheReadContextProvider.DEFAULT);
+ }
+
+ /* *********************************************************************************
+ * Buffer-cache residency instrumentation — enforces the no-real-disk-I/O contract
+ * *********************************************************************************
+ */
+
+ /**
+ * Number of buffer-cache pages the page-replacement strategy has <b>actually allocated</b> so far. This is
+ * runtime allocator state, read straight off {@link ClockPageReplacementStrategy}, not a restatement of the
+ * sizing formula: the strategy allocates lazily on demand and only ever starts evicting once this reaches
+ * {@link #getBufferCachePageBudget()} (see {@code ClockPageReplacementStrategy#findVictim}, which loops while
+ * {@code numPages + multiplier > maxAllowedNumPages}). So
+ * {@code getAllocatedBufferCachePages() < getBufferCachePageBudget()} after a workload is direct evidence that
+ * <b>no page was ever evicted</b> during it, and therefore that no measurement included a re-read from disk.
+ */
+ public int getAllocatedBufferCachePages() {
+ return pageReplacementStrategy.getNumPages();
+ }
+
+ /** The disk buffer cache's real configured page budget. */
+ public int getBufferCachePageBudget() {
+ return pageReplacementStrategy.getMaxAllowedNumPages();
+ }
+
+ /**
+ * Total page count of every disk component's file, read from the real file handles via
+ * {@link IBufferCache#getNumPagesOfFile(int)} — the index's measured on-disk footprint, independent of the
+ * {@code *_BYTES_PER_TUPLE} estimates used to size the caches.
+ */
+ public int getIndexPageCount() throws HyracksDataException {
+ int pages = 0;
+ for (ILSMDiskComponent component : lsmBtree.getDiskComponents()) {
+ pages += diskBufferCache.getNumPagesOfFile(((ColumnBTree) component.getIndex()).getFileId());
+ }
+ return pages;
+ }
+}
diff --git a/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/sample/ColumnSampleBenchHarnessResidencyTest.java b/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/sample/ColumnSampleBenchHarnessResidencyTest.java
new file mode 100644
index 0000000000..3dc0976d3b
--- /dev/null
+++ b/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/sample/ColumnSampleBenchHarnessResidencyTest.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.test.sample;
+
+import org.apache.asterix.column.test.sample.ColumnSampleBenchHarness.Projection;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.SampleCursorStats;
+import org.apache.hyracks.util.annotations.AiProvenance;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Enforces {@link ColumnSampleBenchHarness}'s no-real-disk-I/O contract: a full sample scan must evict no
+ * buffer-cache page, so every measurement through the harness is CPU and buffer-cache cost, never device
+ * latency.
+ * <p>
+ * Not parameterized over {@link Projection}, unlike {@link ColumnSampleBenchHarnessTest}: phase-2 column
+ * materialization is served by the column buffer pool, not the disk buffer cache these checks read, so the
+ * outcome cannot depend on the projection — and a second 475 000-insert build would buy an identical result.
+ */
+@AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_4_8, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.TEST_GENERATED, notes = "Split out of ColumnSampleBenchHarnessTest: the buffer-cache residency/budget checks do not depend "
+ + "on Projection, so they no longer need to run once per projection value")
+public class ColumnSampleBenchHarnessResidencyTest {
+
+ /**
+ * At ~50 leaf pages, where the disk budget is far tighter than at the 20 000-key default.
+ * <p>
+ * The signal is real rather than a restatement of the sizing formula:
+ * {@code getAllocatedBufferCachePages()} is {@code ClockPageReplacementStrategy}'s lazily-grown allocation
+ * count, and it only evicts once that count reaches the budget. Strictly below budget after a full scan
+ * therefore proves nothing was re-read from disk — which re-deriving {@code numKeys * bytesPerTuple} could
+ * not.
+ */
+ @Test
+ public void wholeIndexStaysBufferCacheResidentAtTaskScale() throws Exception {
+ ColumnSampleBenchHarness bigHarness = new ColumnSampleBenchHarness(165000, 55, 3, Projection.ALL_COLUMNS);
+ bigHarness.setUp();
+ try {
+ int budget = bigHarness.getBufferCachePageBudget();
+ // Measured footprint of the real component files, independent of the sizing estimate.
+ int indexPages = bigHarness.getIndexPageCount();
+ Assert.assertTrue("index footprint " + indexPages + " pages must fit the " + budget
+ + "-page buffer cache with room to spare", indexPages * 2 < budget);
+
+ SampleCursorStats stats = new SampleCursorStats();
+ bigHarness.runSample(1063, 12345L, stats);
+
+ int allocated = bigHarness.getAllocatedBufferCachePages();
+ Assert.assertTrue(
+ "buffer cache allocated " + allocated + " of " + budget + " pages: at budget the clock strategy "
+ + "starts evicting, so measurements would include real disk I/O",
+ allocated < budget);
+ System.out.println("harness residency: leafPages=" + bigHarness.getSampledComponentLeafPageCount()
+ + " indexPages=" + indexPages + " allocated=" + allocated + " budget=" + budget + " headroom="
+ + String.format("%.2fx", (double) budget / allocated));
+ } finally {
+ bigHarness.tearDown();
+ }
+ }
+
+ /**
+ * Same contract where the <b>sizing formula</b> supplies the budget rather than the
+ * {@code MIN_DISK_NUM_PAGES} floor, so the formula and its stored-tuple basis are both load-bearing.
+ * <p>
+ * {@code (250000, 90, 6)} clears the floor by ~36 % at 475 000 stored tuples — the cheapest configuration
+ * that does (700 000 inserts at {@code shadowPercent=100}) while still leaving live keys in the sampled
+ * component; at {@code shadowPercent=100} the oldest component has none and phase 2 goes unexercised.
+ */
+ @Test
+ public void diskBudgetIsSizedFromStoredTuplesAndKeepsIndexResident() throws Exception {
+ ColumnSampleBenchHarness bigHarness = new ColumnSampleBenchHarness(250000, 90, 6, Projection.ALL_COLUMNS);
+ bigHarness.setUp();
+ try {
+ int budget = bigHarness.getBufferCachePageBudget();
+ // The formula, not the floor, must be what sizes this cache - otherwise the rest proves nothing.
+ Assert.assertTrue(
+ "budget " + budget + " must exceed the " + ColumnSampleBenchHarness.MIN_DISK_NUM_PAGES
+ + "-page floor, else the floor is what is being tested",
+ budget > ColumnSampleBenchHarness.MIN_DISK_NUM_PAGES);
+ // Every shadowed key is stored twice, so the basis must be numKeys + shadowBoundary. Sizing from
+ // numKeys alone lands on the floor at this configuration, so this fails if that regresses.
+ Assert.assertTrue(
+ "budget " + budget + " must exceed the "
+ + ColumnSampleBenchHarness.diskPagesFor(bigHarness.getNumKeys())
+ + " pages a numKeys-only basis would give",
+ budget > ColumnSampleBenchHarness.diskPagesFor(bigHarness.getNumKeys()));
+
+ int indexPages = bigHarness.getIndexPageCount();
+ Assert.assertTrue("index footprint " + indexPages + " pages must fit the " + budget
+ + "-page buffer cache with room to spare", indexPages * 2 < budget);
+
+ bigHarness.runSample(1063, 12345L, new SampleCursorStats());
+
+ int allocated = bigHarness.getAllocatedBufferCachePages();
+ Assert.assertTrue("buffer cache allocated " + allocated + " of " + budget
+ + " pages: at budget the clock strategy starts evicting, so measurements would include real "
+ + "disk I/O", allocated < budget);
+ System.out.println("harness residency/formula: storedTuples="
+ + (bigHarness.getNumKeys() + bigHarness.getShadowBoundary()) + " leafPages="
+ + bigHarness.getSampledComponentLeafPageCount() + " indexPages=" + indexPages + " allocated="
+ + allocated + " budget=" + budget + " numKeysOnlyBudget="
+ + ColumnSampleBenchHarness.diskPagesFor(bigHarness.getNumKeys()) + " headroom="
+ + String.format("%.2fx", (double) budget / allocated));
+ } finally {
+ bigHarness.tearDown();
+ }
+ }
+}
diff --git a/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/sample/ColumnSampleBenchHarnessTest.java b/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/sample/ColumnSampleBenchHarnessTest.java
new file mode 100644
index 0000000000..10935b9e1a
--- /dev/null
+++ b/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/sample/ColumnSampleBenchHarnessTest.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.test.sample;
+
+import java.util.Arrays;
+import java.util.Collection;
+
+import org.apache.asterix.column.test.sample.ColumnSampleBenchHarness.Projection;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.SampleCursorStats;
+import org.apache.hyracks.util.annotations.AiProvenance;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+/**
+ * Smoke test for {@link ColumnSampleBenchHarness}, run once per projection mode: {@code ALL_COLUMNS} is the
+ * production-faithful baseline (phase 2 materializes every column), {@code PK_ONLY} isolates phase 1.
+ * <p>
+ * The buffer-cache residency/budget checks used to live here too, parameterized over {@code Projection} like
+ * everything else in this class. They moved out to {@link ColumnSampleBenchHarnessResidencyTest} because their
+ * assertions are about the on-disk buffer cache (component file size vs. disk-page budget), which the
+ * {@code Projection} does not affect - column materialization during phase 2 goes through the separate column
+ * buffer pool, not the disk buffer cache these checks observe. Parameterizing them bought nothing but building
+ * a 475 000-insert index twice for identical checks.
+ */
+@RunWith(Parameterized.class)
+@AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_4_8, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.TEST_GENERATED, notes = "Smoke test proving the column LSM sample bench harness stands up a shadowed column index and "
+ + "drives the real column sample cursor in both projection modes")
+public class ColumnSampleBenchHarnessTest {
+
+ @Parameterized.Parameters(name = "{0}")
+ public static Collection<Object[]> projections() {
+ return Arrays.asList(new Object[] { Projection.ALL_COLUMNS }, new Object[] { Projection.PK_ONLY });
+ }
+
+ private final Projection projection;
+ private final ColumnSampleBenchHarness harness;
+
+ public ColumnSampleBenchHarnessTest(Projection projection) {
+ this.projection = projection;
+ this.harness = new ColumnSampleBenchHarness(20000, 55, 3, projection);
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ harness.setUp();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ harness.tearDown();
+ }
+
+ @Test
+ public void harnessBuildsShadowedComponentsAndSamplesLiveKeysOnly() throws Exception {
+ // The shadow structure only exists if every flush produced its own disk component (NoMergePolicy).
+ Assert.assertEquals("expected one base component plus one per shadow round",
+ 1 + harness.getNumShadowComponents(), harness.getIndex().getDiskComponents().size());
+ // A single-mega-leaf component would make the leaf-draw phase degenerate.
+ int leafPages = harness.getSampledComponentLeafPageCount();
+ Assert.assertTrue("expected the sampled component to span several mega-leaf pages, got " + leafPages,
+ leafPages > 1);
+ // Fill-proportional page rejection is only exercised when the real per-component value is in play.
+ int maxLeafTupleCount = harness.getEffectiveMaxLeafTupleCount();
+ Assert.assertTrue("expected the component's recorded max leaf tuple count to be read, got " + maxLeafTupleCount,
+ maxLeafTupleCount > 0);
+
+ SampleCursorStats stats = new SampleCursorStats();
+ ColumnSampleBenchHarness.SampleRun run = harness.runSample(1063, 12345L, stats);
+
+ Assert.assertTrue("expected the sampler to emit tuples", run.emitted > 0);
+ Assert.assertTrue("expected measurable time", run.wallNanos > 0);
+ // Shadowed keys live in [0, shadowBoundary) and must never be emitted.
+ for (int key : run.sampledKeys) {
+ Assert.assertTrue("shadowed key " + key + " leaked into the sample", key >= harness.getShadowBoundary());
+ }
+ // The instrumentation is inert unless a stats object is attached; prove it populated.
+ Assert.assertTrue("expected phase-1 attempts to be counted", stats.attempts > 0);
+ Assert.assertTrue("expected liveness time to be counted", stats.livenessNanos > 0);
+ Assert.assertTrue("expected phase-2 page pins to be counted", stats.phase2PagePins > 0);
+ System.out.println(
+ "harness smoke [" + projection + "]: leafPages=" + leafPages + " maxLeafTupleCount=" + maxLeafTupleCount
+ + " emitted=" + run.emitted + " wallMs=" + run.wallNanos / 1_000_000 + " " + stats.format());
+ }
+
+ /**
+ * Tasks 3-7 drive several runs off one harness instance, so a run must be repeatable and must not disturb the
+ * index state for the next one.
+ */
+ @Test
+ public void repeatedRunsWithTheSameSeedAgree() throws Exception {
+ ColumnSampleBenchHarness.SampleRun first = harness.runSample(500, 99L, new SampleCursorStats());
+ ColumnSampleBenchHarness.SampleRun second = harness.runSample(500, 99L, new SampleCursorStats());
+ Assert.assertEquals(first.emitted, second.emitted);
+ Assert.assertArrayEquals(first.sampledKeys, second.sampledKeys);
+ }
+}
diff --git a/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/sample/ColumnSampleCostBaselineTest.java b/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/sample/ColumnSampleCostBaselineTest.java
new file mode 100644
index 0000000000..ba1ea2f251
--- /dev/null
+++ b/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/sample/ColumnSampleCostBaselineTest.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.test.sample;
+
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.SampleCursorStats;
+import org.apache.hyracks.util.annotations.AiProvenance;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Prints the sampling cost breakdown across mutation levels, to answer whether the newer-component liveness
+ * search dominates.
+ * <p>
+ * <b>A measurement, not a gate.</b> {@code livenessSharePct} is printed, never asserted: batching the liveness
+ * search would <em>lower</em> it, so a "share must stay high" assertion would fail on the improvement it exists
+ * to motivate. The assertions that do run — disjoint phase timings, the leaf-page floor, the page-group
+ * amortization — are invariants unrelated to that share.
+ * <p>
+ * {@code NUM_KEYS} is 165 000, not the 20 000 the plan asked for: column materialization pays per page, so a
+ * ~7-page index under-exercises phase 2 and reproduces the very spatial distortion the bias fix removed.
+ */
+@AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_4_8, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.TEST_GENERATED, notes = "Baseline cost-breakdown diagnostic: measures whether the liveness search dominates column sample cost post bias-fix")
+public class ColumnSampleCostBaselineTest {
+
+ private static final int NUM_KEYS = 165000;
+ private static final int NUM_SHADOW_COMPONENTS = 3;
+ private static final int TARGET = 1063;
+ private static final long SEED = 12345L;
+ private static final int[] SHADOW_LEVELS = { 0, 15, 55 };
+ /**
+ * Keeps phase-2 column materialization genuinely exercised. Deliberately independent of
+ * {@code ColumnSampleUniformityTest#MIN_DISTINCT_PAGES}, which guards spatial clustering instead.
+ */
+ private static final int MIN_LEAF_PAGES_FOR_PHASE2_COVERAGE = 50;
+ /** Historical reference point recorded when this baseline was first measured; see the class javadoc. */
+ private static final double HISTORICAL_LIVENESS_SHARE_PCT = 50.0;
+ /** Guards that same-page draws share one PK pass rather than each paying its own rewind. */
+ private static final int MIN_DRAWS_PER_PAGE_GROUP = 3;
+
+ @Test
+ public void printCostBreakdownAcrossMutationLevels() throws Exception {
+ System.out.println("=".repeat(120));
+ System.out.println("Column sample cost baseline (numKeys=" + NUM_KEYS + ", numShadowComponents="
+ + NUM_SHADOW_COMPONENTS + ", target=" + TARGET + ", seed=" + SEED + ")");
+ double worstLivenessShare = 0.0;
+ for (int shadowPct : SHADOW_LEVELS) {
+ ColumnSampleBenchHarness harness = new ColumnSampleBenchHarness(NUM_KEYS, shadowPct, NUM_SHADOW_COMPONENTS);
+ harness.setUp();
+ try {
+ int leafPages = harness.getSampledComponentLeafPageCount();
+ // Asserted, not just printed: a change to the harness's sizing constants that silently shrinks
+ // the index would otherwise drift this baseline back into the ~7-page distortion unnoticed.
+ Assert.assertTrue(
+ "expected at least " + MIN_LEAF_PAGES_FOR_PHASE2_COVERAGE + " sampled mega-leaf pages to "
+ + "keep Phase 2 column materialization genuinely exercised, got " + leafPages
+ + " at shadow=" + shadowPct + "%; check the harness's sizing constants",
+ leafPages >= MIN_LEAF_PAGES_FOR_PHASE2_COVERAGE);
+ // Warm up once so class loading and page-cache effects do not skew the measured run.
+ harness.runSample(TARGET, SEED, new SampleCursorStats());
+ SampleCursorStats stats = new SampleCursorStats();
+ ColumnSampleBenchHarness.SampleRun run = harness.runSample(TARGET, SEED, stats);
+ System.out.printf("shadow=%2d%% leafPages=%3d wall=%7.1fms emitted=%4d %s%n", shadowPct, leafPages,
+ run.wallNanos / 1e6, run.emitted, stats.format());
+ // Disjoint slices, or livenessSharePct and everything derived from it is silently distorted.
+ // Against a live cursor, which a pure-arithmetic unit test cannot do.
+ long phaseSum = stats.livenessNanos + stats.pkSeekNanos + stats.phase2Nanos;
+ Assert.assertTrue("phase timings exceed the measured wall time (double-counting): sum=" + phaseSum
+ + " wall=" + run.wallNanos, phaseSum <= run.wallNanos);
+ // Structural, not a timing gate: revert to one PK pass per draw and pageGroups equals attempts,
+ // failing this. The floor sits far below what is measured (~7x at shadow=55%) so it catches the
+ // regression without pinning a performance number.
+ Assert.assertTrue(
+ "expected same-page draws to share one PK pass, but pageGroups=" + stats.pageGroups
+ + " against attempts=" + stats.attempts + " at shadow=" + shadowPct + "%",
+ stats.pageGroups * MIN_DRAWS_PER_PAGE_GROUP <= stats.attempts);
+ worstLivenessShare = Math.max(worstLivenessShare, stats.livenessSharePct());
+ } finally {
+ harness.tearDown();
+ }
+ }
+ // Printed, never asserted -- see the class javadoc.
+ System.out.printf(
+ "liveness share (max across shadow levels): %.1f%% (cleared historical %.0f%% threshold: " + "%s)%n",
+ worstLivenessShare, HISTORICAL_LIVENESS_SHARE_PCT, worstLivenessShare > HISTORICAL_LIVENESS_SHARE_PCT);
+ System.out.println("=".repeat(120));
+ }
+}
diff --git a/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/sample/ColumnSampleLivenessWiringTest.java b/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/sample/ColumnSampleLivenessWiringTest.java
new file mode 100644
index 0000000000..7908f56a88
--- /dev/null
+++ b/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/sample/ColumnSampleLivenessWiringTest.java
@@ -0,0 +1,176 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.test.sample;
+
+import org.apache.asterix.column.operation.lsm.merge.MergeColumnReadMetadata;
+import org.apache.asterix.column.operation.query.QueryColumnMetadata;
+import org.apache.asterix.column.values.IColumnValuesReader;
+import org.apache.asterix.column.values.reader.PrimitiveColumnValuesReader;
+import org.apache.hyracks.api.util.CleanupUtils;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.projection.ColumnProjectorType;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.projection.IColumnProjectionInfo;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.lsm.LSMColumnBTreeOpContext;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.lsm.LSMColumnBatchPointSearchCursor;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.lsm.LSMColumnSampleLivenessSearchCursor;
+import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeBatchPointSearchCursor;
+import org.apache.hyracks.util.annotations.AiProvenance;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Guards both halves of the existence-only liveness optimization — the cheaper cursor and the key-only
+ * projection — at their wiring seams.
+ * <p>
+ * The dangerous failure is not "it stopped working" (the cost tests show that) but "it leaked onto the query
+ * path", which a query needs the projected columns and a positioned tuple for and which nothing else here would
+ * notice.
+ */
+@AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_4_8, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.TEST_GENERATED, notes = "Wiring guard: the existence-only liveness cursor and key-only projection must reach the sample "
+ + "path and only the sample path")
+public class ColumnSampleLivenessWiringTest {
+
+ /** Small: this test inspects wiring and projection shape, not sampling behaviour. */
+ private static final int NUM_KEYS = 20000;
+ private static final int SHADOW_PCT = 55;
+ private static final int NUM_SHADOW_COMPONENTS = 3;
+
+ private final ColumnSampleBenchHarness harness =
+ new ColumnSampleBenchHarness(NUM_KEYS, SHADOW_PCT, NUM_SHADOW_COMPONENTS);
+
+ @Before
+ public void setUp() throws Exception {
+ harness.setUp();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ harness.tearDown();
+ }
+
+ @Test
+ public void sampleLivenessProbeGetsTheExistenceOnlyCursorAndQueriesDoNot() throws Exception {
+ LSMColumnBTreeOpContext opCtx = harness.createSearchOpContext();
+ LSMBTreeBatchPointSearchCursor livenessCursor = null;
+ LSMBTreeBatchPointSearchCursor queryCursor = null;
+ try {
+ livenessCursor = harness.getIndex().createSampleLivenessSearchCursor(opCtx);
+ Assert.assertTrue(
+ "the sample cursor's liveness probe must get the existence-only cursor, got "
+ + livenessCursor.getClass().getName(),
+ livenessCursor instanceof LSMColumnSampleLivenessSearchCursor);
+
+ queryCursor = harness.getIndex().createBatchPointSearchCursor(opCtx);
+ // Exact class, not instanceof: the existence-only cursor is a SUBCLASS of this one, so an
+ // instanceof check would pass even if the query path had been switched over to it. Query point
+ // searches read the projected payload and would silently get nothing.
+ Assert.assertEquals("query batch point search must not get the existence-only cursor",
+ LSMColumnBatchPointSearchCursor.class, queryCursor.getClass());
+ } finally {
+ Throwable failure = CleanupUtils.destroy(null, livenessCursor, queryCursor);
+ if (failure != null) {
+ throw new AssertionError(failure);
+ }
+ }
+ }
+
+ @Test
+ public void existenceOnlyProjectionDropsEveryColumnButKeepsThePrimaryKeys() throws Exception {
+ IColumnProjectionInfo full = harness.createSearchOpContext().createProjectionInfo();
+ Assert.assertNotNull("expected a disk component to supply the sample projection info", full);
+ // The sample collector installs an all-fields projector reporting MODIFY. Asserting MERGE here would
+ // pass against a shape production never produces, and keep passing with the reduction fully broken.
+ Assert.assertEquals(ColumnProjectorType.MODIFY, full.getProjectorType());
+ Assert.assertTrue("phase 2's projection must be a query read projection; got " + full.getClass().getName(),
+ full instanceof QueryColumnMetadata);
+ Assert.assertTrue("the full sample projection is expected to project every column; got "
+ + full.getNumberOfProjectedColumns(), full.getNumberOfProjectedColumns() > 0);
+
+ IColumnProjectionInfo existenceOnly = full.createExistenceOnlyProjectionInfo();
+ // The interface default returns `this` -- correct, but not the reduction under test.
+ Assert.assertNotSame("expected a reduced view, not the interface's identity fallback", full, existenceOnly);
+ Assert.assertEquals("existence-only projection must project no columns", 0,
+ existenceOnly.getNumberOfProjectedColumns());
+ Assert.assertEquals("existence-only projection must filter no columns", 0,
+ existenceOnly.getNumberOfFilteredColumns());
+ // The primary keys settle existence, so they must survive untouched.
+ Assert.assertEquals(full.getNumberOfPrimaryKeys(), existenceOnly.getNumberOfPrimaryKeys());
+ // EXISTENCE, not the parent's MODIFY: CloudColumnReadContext branches on the projector type before it
+ // looks at the projected set, so the type is what keeps the reduction real on cloud storage.
+ Assert.assertEquals("existence-only projection must report EXISTENCE so cloud reads pin only page zero",
+ ColumnProjectorType.EXISTENCE, existenceOnly.getProjectorType());
+
+ // The reduced view is merge-shaped, and a merge projection refuses to be reduced -- nothing should ask
+ // it to, so the refusal is what keeps that assumption checked.
+ Assert.assertThrows(UnsupportedOperationException.class, existenceOnly::createExistenceOnlyProjectionInfo);
+ }
+
+ /**
+ * Phase 2 must keep <b>every</b> column: it materializes the sampled tuples into the sample index, so a
+ * reduced projection reaching it writes that index with missing column data — silent corruption, the worst
+ * outcome available here. Two ways in, both closed:
+ * <ol>
+ * <li><b>Mutation</b> — a reduction applied in place would reduce the object phase 2 holds, so the full
+ * projection is re-inspected <em>after</em> the reduced view is derived.</li>
+ * <li><b>Wiring</b> — the reduction belongs only to
+ * {@code LSMColumnSampleLivenessSearchCursor#createAccessor}; phase 2's accessor comes from the unreduced
+ * {@code createProjectionInfo()}, and this asserts what that path produces.</li>
+ * </ol>
+ * That phase 2's projection really does still pin the pages it projects is asserted in
+ * {@link CloudExistenceProjectionPinningTest}.
+ */
+ @Test
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_4_8, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.TEST_GENERATED, notes = "Guards phase 2 against the existence-only reduction, both by mutation and by wiring")
+ public void derivingTheExistenceOnlyViewLeavesPhase2sProjectionIntact() throws Exception {
+ LSMColumnBTreeOpContext opCtx = harness.createSearchOpContext();
+ // What the sample cursor's own accessor builds its projection from.
+ IColumnProjectionInfo phase2Projection = opCtx.createProjectionInfo();
+ Assert.assertNotNull("expected a disk component to supply the sample projection info", phase2Projection);
+ Assert.assertTrue("phase 2 must be given a query read projection; got " + phase2Projection.getClass().getName(),
+ phase2Projection instanceof QueryColumnMetadata);
+ QueryColumnMetadata query = (QueryColumnMetadata) phase2Projection;
+
+ int projectedBefore = query.getNumberOfProjectedColumns();
+ PrimitiveColumnValuesReader[] keyReadersBefore = query.getPrimaryKeyReaders();
+ Assert.assertEquals(ColumnProjectorType.MODIFY, query.getProjectorType());
+ Assert.assertTrue("phase 2's projection must project every column; got " + projectedBefore,
+ projectedBefore > 0);
+ Assert.assertTrue("an all-fields projection must project more than the primary keys",
+ projectedBefore > keyReadersBefore.length);
+
+ // Derive the liveness probe's reduced view, then re-check phase 2's projection.
+ IColumnProjectionInfo existenceOnly = query.createExistenceOnlyProjectionInfo();
+ Assert.assertEquals("existence-only view must be a separate object, leaving phase 2's projection alone",
+ ColumnProjectorType.MODIFY, query.getProjectorType());
+ Assert.assertEquals("deriving the existence-only view must not reduce phase 2's projected column count",
+ projectedBefore, query.getNumberOfProjectedColumns());
+ Assert.assertSame("the existence-only view must share, not replace, phase 2's primary-key readers",
+ keyReadersBefore, query.getPrimaryKeyReaders());
+ IColumnValuesReader[] reducedReaders = ((MergeColumnReadMetadata) existenceOnly).getColumnReaders();
+ Assert.assertEquals("the existence-only view must expose the primary keys and nothing else",
+ keyReadersBefore.length, reducedReaders.length);
+ for (int i = 0; i < keyReadersBefore.length; i++) {
+ Assert.assertSame("primary-key reader " + i + " must be shared, not copied", keyReadersBefore[i],
+ reducedReaders[i]);
+ Assert.assertEquals("primary key " + i + " must stay bound to column " + i, i,
+ reducedReaders[i].getColumnIndex());
+ }
+ }
+}
diff --git a/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/sample/ColumnSampleUniformityTest.java b/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/sample/ColumnSampleUniformityTest.java
new file mode 100644
index 0000000000..ebaeebe647
--- /dev/null
+++ b/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/sample/ColumnSampleUniformityTest.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.test.sample;
+
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.SampleCursorStats;
+import org.apache.hyracks.util.annotations.AiProvenance;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Uniformity guard for {@code ColumnBtreeSampleCursor}: the sample must be uniform over the component's
+ * <em>live</em> tuples, because the CBO derives NDV and selectivity from it. A sample clustered in primary-key
+ * order is silently wrong for every column correlated with the key — sequential ids, timestamps, the common case.
+ * <p>
+ * Chi-square goodness-of-fit on pooled counts over {@link #NUM_BINS} equal-width bins of the live key range, at
+ * {@code NUM_BINS - 1} degrees of freedom; many seeds pooled so the statistic is stable rather than one noisy
+ * draw.
+ * <p>
+ * <b>Sizing matters.</b> {@value #NUM_KEYS} keys give ~50 mega-leaf pages, so each bin covers roughly one page
+ * and a sampler draining only the lowest page ids piles every key into the first bins. A handful of pages could
+ * not distinguish clustering from noise at all, since one page would span several bins.
+ * <p>
+ * Bins are in <em>key</em> space, not page space, deliberately: a fix that spreads page visits without spreading
+ * tuple selection must not be able to satisfy this.
+ */
+@AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_4_8, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.TEST_GENERATED, notes = "Chi-square uniformity guard over the live key space for the column sample cursor")
+public class ColumnSampleUniformityTest {
+
+ /** ~50 mega-leaf pages in the sampled component (measured ~3265 keys per leaf page). */
+ private static final int NUM_KEYS = 165000;
+ private static final int SHADOW_PCT = 55;
+ private static final int NUM_SHADOW_COMPONENTS = 3;
+ private static final int NUM_BINS = 20;
+ private static final int NUM_SEEDS = 40;
+ private static final int TARGET = 1063;
+ /**
+ * Distinct leaf pages per run. Only ~24 of the 51 hold a live key at 55 % shadowing and a correct sampler
+ * reaches nearly all of them; 20 leaves slack for page-boundary effects while staying far above the 2-3 the
+ * bug produced.
+ */
+ private static final int MIN_DISTINCT_PAGES = 20;
+ /**
+ * Total leaf pages in the component — a different concern from {@link #MIN_DISTINCT_PAGES}: too few and
+ * clustering is indistinguishable from noise however many pages a run visits. Equal to {@link #NUM_BINS} by
+ * coincidence; do not conflate them.
+ */
+ private static final int MIN_LEAF_PAGES_FOR_CLUSTERING_DETECTION = 20;
+ // 19 df, alpha = 0.001. Deliberately loose: guards gross bias, not small-sample noise.
+ private static final double CHI_SQUARE_CRITICAL = 43.82;
+
+ private final ColumnSampleBenchHarness harness =
+ new ColumnSampleBenchHarness(NUM_KEYS, SHADOW_PCT, NUM_SHADOW_COMPONENTS);
+
+ @Before
+ public void setUp() throws Exception {
+ harness.setUp();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ harness.tearDown();
+ }
+
+ @Test
+ public void sampleIsUniformOverLiveKeys() throws Exception {
+ double chiSquare = runAndScore("uniform sample", CHI_SQUARE_CRITICAL);
+ Assert.assertTrue("sample distribution is not uniform over live keys: chiSquare=" + chiSquare
+ + " exceeds critical " + CHI_SQUARE_CRITICAL, chiSquare < CHI_SQUARE_CRITICAL);
+ }
+
+ /**
+ * Pools {@link #NUM_SEEDS} runs, prints the bin histogram and the distinct-page counts, asserts that no
+ * shadowed key leaked and that every run emitted exactly {@link #TARGET} tuples, and returns the chi-square
+ * statistic against the uniform expectation.
+ */
+ private double runAndScore(String label, double criticalValue) throws Exception {
+ int leafPages = harness.getSampledComponentLeafPageCount();
+ Assert.assertTrue("a component with " + leafPages + " leaf pages is too coarse to detect spatial clustering",
+ leafPages >= MIN_LEAF_PAGES_FOR_CLUSTERING_DETECTION);
+
+ int shadowBoundary = harness.getShadowBoundary();
+ int liveCount = NUM_KEYS - shadowBoundary;
+ long[] binCounts = new long[NUM_BINS];
+ long total = 0;
+ long totalPagePins = 0;
+ long minPagePins = Long.MAX_VALUE;
+ long maxPagePins = 0;
+
+ for (int seed = 0; seed < NUM_SEEDS; seed++) {
+ SampleCursorStats stats = new SampleCursorStats();
+ ColumnSampleBenchHarness.SampleRun run = harness.runSample(TARGET, seed, stats);
+ // Exactness matters here, not just "close to the target": under-shooting would mean the batch loop
+ // gave up early, and over-shooting would mean the cursor emitted more than it was asked for.
+ Assert.assertEquals("seed " + seed + " did not emit exactly the requested sample size", TARGET,
+ run.emitted);
+ for (int key : run.sampledKeys) {
+ Assert.assertTrue("shadowed key leaked: " + key, key >= shadowBoundary);
+ int bin = (int) ((long) (key - shadowBoundary) * NUM_BINS / liveCount);
+ binCounts[Math.min(bin, NUM_BINS - 1)]++;
+ total++;
+ }
+ totalPagePins += stats.phase2PagePins;
+ minPagePins = Math.min(minPagePins, stats.phase2PagePins);
+ maxPagePins = Math.max(maxPagePins, stats.phase2PagePins);
+ }
+
+ Assert.assertTrue("no samples collected; cannot assess uniformity", total > 0);
+ double expected = (double) total / NUM_BINS;
+ double chiSquare = 0.0;
+ for (long observed : binCounts) {
+ double delta = observed - expected;
+ chiSquare += delta * delta / expected;
+ }
+
+ System.out.println("=".repeat(96));
+ System.out.printf("column sample uniformity [%s]: keys=%d liveKeys=%d leafPages=%d target=%d seeds=%d%n", label,
+ NUM_KEYS, liveCount, leafPages, TARGET, NUM_SEEDS);
+ System.out.printf(" distinct pages materialized per run: min=%d max=%d avg=%.1f (of %d leaf pages)%n",
+ minPagePins, maxPagePins, (double) totalPagePins / NUM_SEEDS, leafPages);
+ System.out.printf(" emitted=%d expectedPerBin=%.1f chiSquare=%.2f (critical %.2f, %d df)%n", total, expected,
+ chiSquare, criticalValue, NUM_BINS - 1);
+ for (int bin = 0; bin < NUM_BINS; bin++) {
+ int from = shadowBoundary + (int) ((long) bin * liveCount / NUM_BINS);
+ int to = shadowBoundary + (int) ((long) (bin + 1) * liveCount / NUM_BINS);
+ // expected > 0 here: total > 0 was asserted above.
+ int barLength = (int) Math.round(40 * binCounts[bin] / expected);
+ System.out.printf(" bin %2d [%6d,%6d) %7d %s%n", bin, from, to, binCounts[bin],
+ "#".repeat(Math.min(barLength, 200)));
+ }
+ System.out.println("=".repeat(96));
+
+ // The most legible symptom of the sorted-batch bug was page coverage, not the statistic: the sampler
+ // materialized 3 of 51 leaf pages. Only ~24 pages hold a live key at 55 %
+ // shadowing, so at the target/page ratio here a correct sampler visits essentially all of them. Asserted
+ // directly so the headline symptom is guarded independently of the chi-square machinery.
+ Assert.assertTrue("only " + minPagePins + " of " + leafPages + " leaf pages were materialized in some run; "
+ + "the sampler is concentrating on a few pages", minPagePins >= MIN_DISTINCT_PAGES);
+ return chiSquare;
+ }
+}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
index b81ff92c9b..3421ef1aac 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
@@ -44,6 +44,7 @@ import org.apache.hyracks.api.config.IOptionType;
import org.apache.hyracks.api.config.Section;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
import org.apache.hyracks.util.StorageUtil;
+import org.apache.hyracks.util.annotations.AiProvenance;
public class StorageProperties extends AbstractProperties {
@@ -69,7 +70,6 @@ public class StorageProperties extends AbstractProperties {
STORAGE_GLOBAL_CLEANUP(BOOLEAN, true),
STORAGE_GLOBAL_CLEANUP_TIMEOUT(POSITIVE_INTEGER, (int) TimeUnit.MINUTES.toSeconds(10)),
STORAGE_COLUMN_MAX_TUPLE_COUNT(NONNEGATIVE_INTEGER, 15000),
- STORAGE_COLUMN_SAMPLES_PER_PAGE(NONNEGATIVE_INTEGER, 1),
STORAGE_COLUMN_FREE_SPACE_TOLERANCE(DOUBLE, 0.15d),
STORAGE_COLUMN_MAX_LEAF_NODE_SIZE(INTEGER_BYTE_UNIT, StorageUtil.getIntSizeInBytes(10, MEGABYTE)),
STORAGE_FORMAT(STRING, "row"),
@@ -115,7 +115,6 @@ public class StorageProperties extends AbstractProperties {
case STORAGE_PARTITIONS_COUNT:
case STORAGE_FORMAT:
case STORAGE_COLUMN_MAX_TUPLE_COUNT:
- case STORAGE_COLUMN_SAMPLES_PER_PAGE:
case STORAGE_COLUMN_FREE_SPACE_TOLERANCE:
case STORAGE_COLUMN_MAX_LEAF_NODE_SIZE:
case STORAGE_MAX_SAMPLE_LEAF_ATTEMPTS:
@@ -173,9 +172,6 @@ public class StorageProperties extends AbstractProperties {
return "The maximum time to wait for nodes to respond to global storage cleanup requests";
case STORAGE_COLUMN_MAX_TUPLE_COUNT:
return "The maximum number of tuples to be stored per a mega leaf page";
- case STORAGE_COLUMN_SAMPLES_PER_PAGE:
- return "The number of sample tuples to collect from each accepted mega-page during columnar sampling. "
- + "Values > 1 reduce Phase 2 I/O by amortizing column-stream setup cost across multiple tuples.";
case STORAGE_COLUMN_FREE_SPACE_TOLERANCE:
return "The percentage of the maximum tolerable empty space for a physical mega leaf page (e.g.,"
+ " 0.15 means a physical page with 15% or less empty space is tolerable)";
@@ -201,10 +197,16 @@ public class StorageProperties extends AbstractProperties {
return "The maximum time in milliseconds to wait for acquiring a buffer from the column buffer pool.";
case STORAGE_MAX_SAMPLE_LEAF_ATTEMPTS:
return "The maximum number of attempts to find a random leaf page during B-tree sampling. "
- + "If exceeded, the sampling operation will fail with an error.";
+ + "If exceeded, the sampling operation stops early and returns a smaller sample.";
case STORAGE_SAMPLE_LEAF_DRAW_BATCH_SIZE:
- return "The batch size of random draws to perform at once during B-tree sampling. "
- + "Higher values reduce random I/O but increase memory overhead during sampling.";
+ return "The upper bound on the number of random draws performed per refill during B-tree "
+ + "sampling. The effective batch size is the smaller of this value and the sample "
+ + "still outstanding, so it has no effect on a sample smaller than it. It is retained "
+ + "as a resource bound rather than a tuning knob: a sampling cursor allocates draw "
+ + "buffers proportional to the batch size, so this caps them at about 0.5 MB per "
+ + "cursor however large a sample is requested. Lowering it below the sample size only "
+ + "costs I/O locality - batches are always consumed in full, so it cannot affect which "
+ + "tuples the sample contains.";
default:
throw new IllegalStateException("NYI: " + this);
}
@@ -229,7 +231,6 @@ public class StorageProperties extends AbstractProperties {
public boolean hidden() {
switch (this) {
case STORAGE_GLOBAL_CLEANUP:
- case STORAGE_COLUMN_SAMPLES_PER_PAGE:
case STORAGE_MAX_SAMPLE_LEAF_ATTEMPTS:
case STORAGE_SAMPLE_LEAF_DRAW_BATCH_SIZE:
return true;
@@ -288,6 +289,18 @@ public class StorageProperties extends AbstractProperties {
return accessor.getInt(Option.STORAGE_MAX_SAMPLE_LEAF_ATTEMPTS);
}
+ /**
+ * Upper bound on one batch of random leaf draws during sampling.
+ * <p>
+ * Both sample cursors size a batch at {@code min(this, outstanding shortfall)}, so at any sane setting this value
+ * does not bind and has no observable effect on the emitted sample. It is nevertheless <b>kept, not deprecated</b>,
+ * because "inert" is not the same as "purposeless": the per-cursor draw buffers are sized to the batch, so this is
+ * the ceiling on that allocation — about 0.5 MB per sampling cursor at the default — and without it a request for a
+ * very large sample would allocate in proportion. Lowering it below the sample size shrinks each batch and costs
+ * I/O locality only: batches are always consumed in full, so the batch size cannot influence <em>which</em> tuples
+ * the sample contains. Removing the option would mean re-deriving that ceiling from something else, for no gain.
+ */
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_4_8, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.DOC_GENERATED, notes = "Record why the sample leaf-draw batch size is retained now that it no longer binds in practice")
public int getSampleLeafDrawBatchSize() {
return accessor.getInt(Option.STORAGE_SAMPLE_LEAF_DRAW_BATCH_SIZE);
}
@@ -384,10 +397,6 @@ public class StorageProperties extends AbstractProperties {
return accessor.getInt(Option.STORAGE_COLUMN_MAX_TUPLE_COUNT);
}
- public int getColumnSamplesPerPage() {
- return accessor.getInt(Option.STORAGE_COLUMN_SAMPLES_PER_PAGE);
- }
-
public double getColumnFreeSpaceTolerance() {
return accessor.getDouble(Option.STORAGE_COLUMN_FREE_SPACE_TOLERANCE);
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
index 5f3827b0b6..d65ed4a108 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
@@ -484,7 +484,7 @@ public class DatasetUtil {
indexHelperFactory, false, false, null, searchCallbackFactory, null, null, false, null, null, -1, false,
null, null, projectorFactory, null, partitioningProperties.getComputeStorageMap(),
sampleCardinalityTargetPerPartition, sampleSeed, storageProperties.getMaxSampleLeafAttempts(),
- storageProperties.getSampleLeafDrawBatchSize(), storageProperties.getColumnSamplesPerPage());
+ storageProperties.getSampleLeafDrawBatchSize());
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, sampleOp, primaryPartitionConstraint);
return sampleOp;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/DiskBTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/DiskBTree.java
index 32a15cc756..97c609c238 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/DiskBTree.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/DiskBTree.java
@@ -361,7 +361,7 @@ public class DiskBTree extends BTree {
public ITreeIndexCursor createSampleCursor(long componentSampleCardinality, long sampleSeed,
ILSMIndexBatchPointCursor searchCursor, int maxLeafAttempts, int leafDrawBatchSize,
- int maxLeafTupleCount, int samplesPerPage, ITupleAcceptor antimatterAcceptor) {
+ int maxLeafTupleCount, ITupleAcceptor antimatterAcceptor) {
IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) btree.getLeafFrameFactory().createFrame();
return new DiskBTreeSampleCursor((DiskBTree) btree, leafFrame, componentSampleCardinality, sampleSeed, ctx,
getBufferCacheOperationContext(), searchCursor, maxLeafAttempts, leafDrawBatchSize,
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/DiskBTreeSampleCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/DiskBTreeSampleCursor.java
index ab7c34478d..76adfbae2d 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/DiskBTreeSampleCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/DiskBTreeSampleCursor.java
@@ -46,12 +46,11 @@ import it.unimi.dsi.fastutil.longs.LongOpenHashSet;
import it.unimi.dsi.fastutil.longs.LongSet;
/**
- * Disk B-tree sample cursor for row storage.
- * <p>
- * Performs single-phase random sampling: picks random leaf pages and tuples,
- * checks antimatter and newer-component existence, and immediately yields
- * accepted samples. For row storage with typical low sampling density
- * (scattered samples across many pages), this approach minimizes page pins.
+ * Disk B-tree sample cursor for row storage. Single-phase: pick a random leaf page and slot, check antimatter
+ * and newer-component liveness, yield immediately. Single-phase because a row tuple is fully materialized by
+ * pinning its page — there is nothing to defer, unlike the column cursor, which splits selection from column
+ * loading. At the low sampling densities row components see, samples scatter over many pages, so pins dominate
+ * and yielding on the spot keeps that count minimal.
*/
public final class DiskBTreeSampleCursor extends EnforcedIndexCursor implements ITreeIndexCursor {
private static final Logger LOGGER = LogManager.getLogger();
@@ -66,48 +65,32 @@ public final class DiskBTreeSampleCursor extends EnforcedIndexCursor implements
private final ITupleAcceptor antimatterAcceptor;
// u64: (pageId << 32) | tupleIndex
private final LongSet seenTupleIndexes;
- // Sampling related fields
- // Multiplier applied to the component tuple-population ceiling to derive the
- // effective consecutive-miss give-up threshold (see doOpen). Empirically the
- // reservoir stalls at roughly threshold/3 when the threshold is too tight, so a
- // multiple well above 3 is needed to reliably reach near-exhaustive targets.
- @AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_4_8, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.GENERATED, notes = "Population-scaled give-up threshold to stop near-exhaustive samples truncating at ~500 attempts")
- private static final long POPULATION_ATTEMPT_MULTIPLIER = 32L;
+ // Well above 3 because the reservoir was observed to stall at ~threshold/3 when the threshold is tight.
+ private static final long SAMPLE_ATTEMPT_MULTIPLIER = 32L;
private final int maxLeafFindingAttempts;
- // Effective consecutive-miss give-up threshold, scaled to the component's tuple
- // population in doOpen(). maxLeafFindingAttempts alone is a fixed floor that
- // silently truncates near-exhaustive samples (target close to the component's
- // live-tuple count): as the reservoir fills, random draws increasingly hit
- // already-seen tuples, so the consecutive-miss streak needed to find the last
- // few tuples grows with the population. Scaling the cap to the population keeps
- // collection from stopping short while still bounding the genuinely-exhausted
- // case (target >= population) to O(population) draws.
+ // Consecutive-miss give-up threshold, scaled to the target in doOpen(): as the reservoir fills, draws
+ // increasingly hit already-seen tuples, so the streak needed for the last few grows with the target and a
+ // fixed floor truncates near-exhaustive samples.
private int effectiveMaxLeafFindingAttempts;
private final long componentSampleCardinality;
private final Random randomNumGen;
private ICachedPage page = null;
private int pageId = -1;
- // Pre-enumerated leaf page IDs for perfectly uniform random access, held as a succinct
- // bitmap+select (ASTERIXDB-3702) instead of an int[] (~15x less memory for dense row components).
- // Enumeration touches only interior pages (level >= 1), skipping all leaf-level I/O; get(i)
- // returns the i-th leaf id in ascending order (identical to the old leafPageIds[i]).
+ // Uniform random access to any leaf without an int[]: succinct bitmap+select (ASTERIXDB-3702), ~15x less
+ // memory on dense row components.
private BitmapLeafIds leafPageIds = null;
- // Batched random draws stored in flat parallel primitive arrays (SoA) to
- // keep the hot per-draw read cache-friendly and free of object headers /
- // pointer chasing. Sorting is done on a packed long[] key
- // ((pageId << 32) | drawIndex) via Arrays.sort, which avoids the boxed
- // object comparator dispatch of the previous List<LeafDraw> approach while
- // still grouping draws by pageId for pin reuse. A single reusable LeafDraw
- // view is repointed per draw to preserve the existing call shape.
+ // Draws as parallel primitive arrays: no object headers or pointer chasing on the hot per-draw read.
private final double[] drawAcceptanceSamples;
- private final int[] drawTupleStartSeeds;
- // Packed sort key: (pageId << 32) | originalDrawIndex. Sorting by the full
- // long orders by pageId first, keeping the side arrays addressable by the
- // low 32 bits without physically permuting them.
+ // Packed (pageId << 32) | drawIndex. Sorting the long orders by pageId -- grouping same-page draws for pin
+ // reuse -- while the low 32 bits keep the side arrays addressable without permuting them.
private final long[] drawSortKeys;
private final LeafDraw leafDrawView;
private int pendingLeafDrawIndex;
- private final int leafDrawBatchSize;
+ // Upper bound on one batch; the batch actually drawn is sized to the outstanding shortfall
+ // (see refillLeafDrawBatch) and is never larger than this.
+ private final int drawBatchCapacity;
+ // Number of valid draws in the current batch. Every one of them is consumed before the next refill.
+ private int currentBatchSize;
private IBufferCache bufferCache;
private int fileId = -1;
// search predicate
@@ -120,17 +103,13 @@ public final class DiskBTreeSampleCursor extends EnforcedIndexCursor implements
private int hasNextAttemptCount = 0;
private int totalAccessCount;
- // Rejection sampling static upper bound (Olken & Rotem)
- // to correct for bias from partially-filled pages
+ // Olken & Rotem rejection bound, correcting the bias from partially-filled pages.
private final int leafTupleCapacity;
- // Cached once: avoids querying the logging framework and, more importantly,
- // gates the per-draw System.nanoTime() timing calls (a syscall-class operation) and the per-batch
- // unique-page counting loop so they are skipped entirely on the hot path. Gated on TRACE (not DEBUG,
- // which is on by default) so this diagnostic instrumentation never runs in normal operation.
+ // Gates the per-draw System.nanoTime() calls and the per-batch unique-page loop off the hot path. TRACE,
+ // not DEBUG, which is on by default -- this instrumentation must never run in normal operation.
private final boolean traceTimingEnabled;
- // Debug and traceability
private long totalTimeTakenToFindRandomLeaf = 0;
private long totalTimeTakenToFindRandomTuples = 0;
private long totalLeafDrawBatches = 0;
@@ -140,6 +119,7 @@ public final class DiskBTreeSampleCursor extends EnforcedIndexCursor implements
private long totalReusedPinnedPageHits = 0;
private boolean endedPreemptively = false;
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_4_8, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.REFACTORED, notes = "Draw-batch capacity derived from the sample target")
public DiskBTreeSampleCursor(DiskBTree diskBTree, IBTreeLeafFrame leafFrame, long componentSampleCardinality,
long sampleSeed, BTreeOpContext ctx, IBufferCacheReadContext bufferCacheOpCtx,
ILSMIndexBatchPointCursor searchCursor, int maxLeafFindingAttempts, int leafDrawBatchSize,
@@ -161,72 +141,102 @@ public final class DiskBTreeSampleCursor extends EnforcedIndexCursor implements
this.maxLeafFindingAttempts = maxLeafFindingAttempts;
this.leafTupleCapacity = maxLeafTupleCount;
this.traceTimingEnabled = LOGGER.isTraceEnabled();
- this.leafDrawBatchSize = (int) Math.max(leafDrawBatchSize, componentSampleCardinality);
-
- // Pre-allocate flat SoA draw buffers once to avoid churn during refills.
- this.drawAcceptanceSamples = new double[this.leafDrawBatchSize];
- this.drawTupleStartSeeds = new int[this.leafDrawBatchSize];
- this.drawSortKeys = new long[this.leafDrawBatchSize];
- this.leafDrawView = new LeafDraw(-1, 0.0, 0);
-
- this.pendingLeafDrawIndex = this.leafDrawBatchSize;
+ // A draw yields at most one sample, so a batch beyond the outstanding shortfall is waste, and it must
+ // be consumed whole to stay unbiased (refillLeafDrawBatch). The configured size stays an upper bound --
+ // an I/O-locality knob.
+ long capacity = leafDrawBatchSize > 0 ? Math.min(leafDrawBatchSize, componentSampleCardinality)
+ : componentSampleCardinality;
+ this.drawBatchCapacity = (int) Math.max(1, Math.min(Integer.MAX_VALUE, capacity));
+
+ // Allocated once: refills reuse these rather than churning.
+ this.drawAcceptanceSamples = new double[this.drawBatchCapacity];
+ this.drawSortKeys = new long[this.drawBatchCapacity];
+ this.leafDrawView = new LeafDraw(-1, 0.0);
+
+ this.pendingLeafDrawIndex = 0;
+ this.currentBatchSize = 0;
}
@Override
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_4_8, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.ASSISTED, notes = "Reset the batch so a reused cursor cannot inherit a stale one")
protected void doOpen(ICursorInitialState initialState, ISearchPredicate searchPred) throws HyracksDataException {
if (page != null) {
releasePage();
}
int rootPageId = ((BTreeCursorInitialState) initialState).getPageId();
- // Enumerate all leaf page IDs by traversing only interior pages.
- // The optimised DFS skips leaf-level I/O (level-1 nodes record their
- // children's page IDs directly), so the cost is proportional to the
- // number of interior pages (~0.4 % of total pages for typical fan-out).
- // The resulting array gives perfectly uniform random access to any leaf.
+ // Interior pages only: level-1 nodes record their children's ids, so this skips all leaf-level I/O and
+ // costs ~0.4 % of total pages at typical fan-out -- the price of uniform random leaf access.
leafPageIds = bTree.enumerateLeafPageIdsCompact(rootPageId, bTreeOpCtx, bufferCacheOpCtx);
- // Scale the give-up threshold to a multiple of the component's tuple-population
- // ceiling (leaf pages x max tuples/leaf). The per-component sample target never
- // exceeds the population (the caller caps it at the component's live-tuple
- // estimate), so a target is always reachable; the loop must not give up before
- // reaching it. Random draws with a growing already-seen set miss increasingly
- // often as the reservoir fills, so the consecutive-miss streak needed to find
- // the last few tuples grows super-linearly in the population - a give-up
- // threshold equal to the population alone stops far short (observed collected
- // ~= threshold/3). POPULATION_ATTEMPT_MULTIPLIER buys enough headroom to reach
- // the target, while still bounding the genuinely-exhausted case (target >
- // available, e.g. a "high" sample of a tiny dataset) to O(multiplier x
- // population) draws. Computed as long, then clamped to int: for huge components
- // the product overflows int, but there the target is far below the population so
- // the loop exits on reaching it; clamping to Integer.MAX_VALUE just avoids a
- // negative (overflowed) threshold that would itself cause truncation.
- long populationCeiling = (long) leafPageIds.size() * leafTupleCapacity;
- effectiveMaxLeafFindingAttempts = (int) Math.min(Integer.MAX_VALUE,
- Math.max(maxLeafFindingAttempts, POPULATION_ATTEMPT_MULTIPLIER * populationCeiling));
+ // MULTIPLIER * max(leafPages, target). Scaled to the TARGET, not the tuple population: the caller
+ // already caps the target at the component's live-tuple estimate, so it is reachable and the loop must
+ // not quit early. Scaling by population instead (leafPages x maxLeafTupleCount ~= total tuples) makes an
+ // unreachable target grind ~32x total tuples however small it is -- tolerable here, where an attempt is
+ // a slot lookup, but minutes-to-hours on the column cursor, which pays a PK-decode per attempt. The
+ // leafPages floor lets a tiny target still reach every page; the clamp stops the product overflowing
+ // negative, which would itself truncate.
+ long attemptCeiling = SAMPLE_ATTEMPT_MULTIPLIER * Math.max(leafPageIds.size(), componentSampleCardinality);
+ effectiveMaxLeafFindingAttempts =
+ (int) Math.min(Integer.MAX_VALUE, Math.max(maxLeafFindingAttempts, attemptCeiling));
+ pendingLeafDrawIndex = 0;
+ currentBatchSize = 0;
if (LOGGER.isDebugEnabled()) {
- LOGGER.debug("DiskBTreeSampleCursor: {} leaf pages enumerated, target {} samples", leafPageIds.size(),
- componentSampleCardinality);
+ LOGGER.debug("DiskBTreeSampleCursor: {} leaf pages enumerated, target {} samples, attempts cap {}",
+ leafPageIds.size(), componentSampleCardinality, effectiveMaxLeafFindingAttempts);
}
}
+ /**
+ * Advances to the next accepted sample.
+ * <p>
+ * <b>The emitted sample is exactly uniform over the component's live tuples.</b> A draw picks a page
+ * uniformly ({@code 1/P}), accepts it with the Olken & Rotem fill correction ({@code n/C}, see
+ * {@link #pinAndAcceptLeafPage}), then picks one of the page's {@code n} slots. A tuple is thus reached with
+ * probability {@code (1/P)(n/C)(1/n) = 1/(P·C)}, free of the page's fill — cancelling {@code n} is what the
+ * correction is for. Antimatter and shadowing are deterministic predicates and repeats are rejected, so this
+ * is uniform sampling without replacement.
+ * <p>
+ * Uniformity then needs every stopping rule to be a function of <b>counts only</b>, since such rules commute
+ * with relabelling the live tuples. Truncating a pageId-sorted batch is the one rule that is not — it keeps
+ * precisely the lowest pageIds — hence full-batch consumption and stop conditions tested only at batch
+ * boundaries ({@link #refillLeafDrawBatch()}). No {@code samplesPerPage} here, unlike the column cursor, so a
+ * batch cannot overshoot and nothing needs downsampling.
+ * <p>
+ * <b>Caller contract: drain the cursor.</b> Emission is lazy, one sample per {@code hasNext()}, so
+ * "consumed in full" holds only if the consumer calls until {@code hasNext()} is {@code false}. Stopping
+ * early keeps the low-pageId prefix of the batch in flight — the same spatial filter, relocated to the
+ * caller. {@code LSMIndexSampleCursor} is the only production driver and drains fully; a consumer wanting a
+ * partial sample must ask for a smaller {@code componentSampleCardinality}, not abandon a larger one.
+ * <p>
+ * The attempts cap is also tested only at batch boundaries, so it can overshoot by one batch — count-based
+ * either way, so bounded extra work and no effect on uniformity.
+ */
@Override
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_4_8, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.REFACTORED, notes = "Full-batch consumption so the pageId sort cannot act as a spatial filter")
protected boolean doHasNext() throws HyracksDataException {
- while (sampledCount < componentSampleCardinality && hasNextAttemptCount < effectiveMaxLeafFindingAttempts) {
- LeafDraw leafDraw = nextLeafDraw();
- if (leafDraw == null) {
- break;
+ while (true) {
+ if (pendingLeafDrawIndex >= currentBatchSize) {
+ // The ONLY legal stopping point: mid-batch would keep just the sorted batch's low-pageId
+ // prefix, a spatial filter rather than a sample.
+ if (hasNextAttemptCount >= effectiveMaxLeafFindingAttempts) {
+ break;
+ }
+ refillLeafDrawBatch();
+ if (currentBatchSize == 0) {
+ // Target reached, or the component has no leaf pages.
+ break;
+ }
}
+ LeafDraw leafDraw = nextLeafDraw();
if (!pinAndAcceptLeafPage(leafDraw)) {
hasNextAttemptCount++;
continue;
}
- int foundTupleIndex = findRandomTuple(leafDraw.tupleStartSeed);
+ int foundTupleIndex = findRandomTuple();
long pageTupleKey = getPageTupleKey(pageId, foundTupleIndex);
- // Skip if no valid tuple found or already seen
if (foundTupleIndex == -1 || seenTupleIndexes.contains(pageTupleKey)) {
hasNextAttemptCount++;
continue;
}
- // Check if tuple exists in newer LSM components
searchKeys.clear();
foundIndexes.clear();
searchKeys.add(frameTuple);
@@ -234,14 +244,13 @@ public final class DiskBTreeSampleCursor extends EnforcedIndexCursor implements
searchCursor.setPredicate(batchPredicate);
searchCursor.hasNextWithPredicate(foundIndexes);
if (foundIndexes.isEmpty()) {
- // Tuple is unique and not found in newer components - valid sample
- // Keep page pinned and return immediately
+ // Live: keep the page pinned, the caller reads the tuple straight off it.
hasNextAttemptCount = 0;
seenTupleIndexes.add(pageTupleKey);
sampledCount++;
return true;
} else {
- // Tuple exists in newer component, try again
+ // Shadowed by a newer component.
hasNextAttemptCount++;
}
}
@@ -249,47 +258,61 @@ public final class DiskBTreeSampleCursor extends EnforcedIndexCursor implements
return false;
}
+ /** Decodes the next draw of the current batch. The caller guarantees the batch is not exhausted. */
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_4_8, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.REFACTORED, notes = "Pure decoding; refill moved to the caller so stopping points are visible in the loop")
private LeafDraw nextLeafDraw() {
- if (pendingLeafDrawIndex >= leafDrawBatchSize) {
- refillLeafDrawBatch();
- if (leafPageIds == null || leafPageIds.size() == 0) {
- return null;
- }
- }
- // Resolve the next draw in sorted (pageId-ascending) order. The sort key packs
- // (pageId << 32 | drawIndex): the high 32 bits ARE the pageId (no separate array), the low 32
- // bits index back into the unsorted SoA side arrays.
+ // High 32 bits of the sort key ARE the pageId; the low 32 index back into the unsorted side arrays.
long sortKey = drawSortKeys[pendingLeafDrawIndex++];
int drawIndex = (int) sortKey;
leafDrawView.pageId = (int) (sortKey >>> 32);
leafDrawView.acceptanceSample = drawAcceptanceSamples[drawIndex];
- leafDrawView.tupleStartSeed = drawTupleStartSeeds[drawIndex];
return leafDrawView;
}
+ /**
+ * Draws the next batch of leaf-page visits, pageId-sorted so the pins go in ascending order.
+ * <p>
+ * <b>The batch is sized to the outstanding shortfall so it can be consumed whole.</b> Sorting is only a
+ * permutation, and a permutation changes nothing — <em>provided every draw is consumed</em>. Truncating a
+ * pageId-sorted batch keeps precisely the lowest pageIds, turning the sort into a spatial filter; a fixed
+ * 32768-draw batch guaranteed that, since the target was met within a short prefix. At {@code remaining}
+ * draws the target can only be reached on the last one, because a draw yields at most one sample — so the
+ * caller can consume all of it. Total draws are unchanged; only the sort windows shrink.
+ * <p>
+ * The configured {@code leafDrawBatchSize} therefore no longer binds unless set below the target. It is kept
+ * as the ceiling on the draw-buffer allocation, and a smaller batch is still consumed whole, so it can only
+ * change pin locality, never which tuples are sampled.
+ * <p>
+ * Sets {@link #currentBatchSize} to 0 when nothing more is needed (target met) or possible (no leaf pages).
+ */
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_4_8, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.REFACTORED, notes = "Batch sized to the shortfall so it can be consumed in full")
private void refillLeafDrawBatch() {
pendingLeafDrawIndex = 0;
- if (leafPageIds == null || leafPageIds.size() == 0 || leafDrawBatchSize <= 0) {
+ currentBatchSize = 0;
+ if (leafPageIds == null || leafPageIds.size() == 0) {
return;
}
- for (int i = 0; i < leafDrawBatchSize; i++) {
+ long remaining = componentSampleCardinality - sampledCount;
+ if (remaining <= 0) {
+ return;
+ }
+ int batchSize = (int) Math.min(drawBatchCapacity, remaining);
+ for (int i = 0; i < batchSize; i++) {
int randomLeafIndex = randomNumGen.nextInt(leafPageIds.size());
int targetPageId = leafPageIds.get(randomLeafIndex);
drawAcceptanceSamples[i] = randomNumGen.nextDouble();
- drawTupleStartSeeds[i] = randomNumGen.nextInt();
- // Pack (pageId, drawIndex): pageIds are non-negative leaf page IDs,
- // so the high 32 bits sort correctly as a signed long.
+ // pageIds are non-negative, so the high 32 bits sort correctly as a signed long.
drawSortKeys[i] = (((long) targetPageId) << 32) | (i & 0xffffffffL);
}
- // Primitive long[] sort: branch-predictable, comparator-free, and
- // cache-friendly versus the previous boxed-object comparator sort.
- Arrays.sort(drawSortKeys);
+ // Comparator-free and cache-friendly, unlike a boxed-object sort.
+ Arrays.sort(drawSortKeys, 0, batchSize);
+ currentBatchSize = batchSize;
if (traceTimingEnabled) {
int uniquePages = 0;
long prevPageId = Long.MIN_VALUE;
- for (int i = 0; i < leafDrawBatchSize; i++) {
+ for (int i = 0; i < batchSize; i++) {
long currentPageId = drawSortKeys[i] >>> 32;
if (currentPageId != prevPageId) {
uniquePages++;
@@ -297,7 +320,7 @@ public final class DiskBTreeSampleCursor extends EnforcedIndexCursor implements
}
}
totalLeafDrawBatches++;
- totalLeafDraws += leafDrawBatchSize;
+ totalLeafDraws += batchSize;
totalLeafDrawUniquePages += uniquePages;
}
}
@@ -307,16 +330,9 @@ public final class DiskBTreeSampleCursor extends EnforcedIndexCursor implements
}
/**
- * Selects a random leaf page from the pre-enumerated leaf page ID array.
- * <p>
- * Each leaf has exactly {@code 1 / N_leaves} probability of being chosen.
- * The enumeration cost is amortized: it touches only interior pages once
- * during {@code doOpen()}, then every sample here is a single direct pin.
- * <p>
- * <b>Page-fill correction (Olken & Rotem, VLDB 1989):</b> Each selected leaf
- * is accepted with probability {@code tupleCount / leafTupleCapacity} to
- * correct for the bias where tuples on partially-filled pages have higher
- * per-tuple selection probability.
+ * Pins the draw's leaf page and applies the <b>page-fill correction</b> (Olken & Rotem, VLDB 1989):
+ * accept with probability {@code tupleCount / leafTupleCapacity}, without which tuples on partly-filled
+ * pages would have a higher per-tuple selection probability.
*/
private boolean pinAndAcceptLeafPage(LeafDraw leafDraw) throws HyracksDataException {
// Skip the nanoTime() pair entirely unless trace logging will consume it.
@@ -335,11 +351,10 @@ public final class DiskBTreeSampleCursor extends EnforcedIndexCursor implements
}
int tupleCount = leafFrame.getTupleCount();
if (tupleCount == 0) {
+ // Load bearing: findRandomTuple's nextInt(bound) needs a positive bound.
return false;
}
- // Rejection sampling: accept this page with probability proportional to
- // its tuple count against a static capacity bound.
- // Skip correction if capacity unknown (pre-upgrade component lacking metadata).
+ // Unknown capacity (pre-upgrade component, no metadata) => accept every page.
if (leafTupleCapacity > 0) {
double acceptProb = (double) tupleCount / leafTupleCapacity;
if (leafDraw.acceptanceSample >= acceptProb) {
@@ -348,7 +363,7 @@ public final class DiskBTreeSampleCursor extends EnforcedIndexCursor implements
}
return true;
} finally {
- // Ensures elapsed time is correctly tracked even if the page is rejected
+ // In a finally so a rejected page still counts its elapsed time.
if (traceTimingEnabled) {
totalTimeTakenToFindRandomLeaf += (System.nanoTime() - nanos);
}
@@ -356,19 +371,22 @@ public final class DiskBTreeSampleCursor extends EnforcedIndexCursor implements
}
/**
- * Picks a random tuple from the current leaf page.
+ * Picks a random slot on the current leaf page, or {@code -1} to reject the draw.
+ * <p>
+ * <b>Antimatter rejects the whole draw</b> rather than re-drawing on the same page: an O(1) decision that
+ * keeps the per-draw law uniform over the component, so the sample reflects the data's physical density.
* <p>
- * <b>Antimatter Rejection:</b> If the selected tuple is antimatter, it immediately
- * returns -1 (rejecting the draw completely). This is O(1) and preserves the unbiased
- * uniform distribution of live tuples, accurately reflecting the physical density of the data.
- * </p>
+ * The slot comes from {@link Random#nextInt(int)}, exactly uniform on {@code [0, tupleCount)} because it
+ * rejects the generator's non-representable tail. Folding a pre-drawn seed with {@code % tupleCount} instead
+ * over-weights the low slots unless {@code tupleCount} is a power of two. The bound must be positive, which
+ * {@link #pinAndAcceptLeafPage}'s {@code tupleCount == 0} rejection guarantees.
*/
- private int findRandomTuple(int tupleStartSeed) {
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_4_8, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.REFACTORED, notes = "Slot drawn with nextInt(bound); the % fold was non-uniform")
+ private int findRandomTuple() {
long nanos = traceTimingEnabled ? System.nanoTime() : 0L;
int numberOfTuples = leafFrame.getTupleCount();
- // Strips sign bit for fast, uniform modulo arithmetic
- int targetTupleIndex = (tupleStartSeed & 0x7FFFFFFF) % numberOfTuples;
+ int targetTupleIndex = randomNumGen.nextInt(numberOfTuples);
frameTuple.resetByTupleIndex(leafFrame, targetTupleIndex);
int result = antimatterAcceptor.accept(frameTuple) ? -1 : targetTupleIndex;
@@ -395,6 +413,7 @@ public final class DiskBTreeSampleCursor extends EnforcedIndexCursor implements
}
@Override
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_4_8, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.ASSISTED, notes = "Reset the batch by emptying it; the fixed batch length is gone")
protected void doClose() throws HyracksDataException {
if (LOGGER.isTraceEnabled()) {
double avgBatchDraws = totalLeafDrawBatches == 0 ? 0.0 : (double) totalLeafDraws / totalLeafDrawBatches;
@@ -423,7 +442,8 @@ public final class DiskBTreeSampleCursor extends EnforcedIndexCursor implements
pageId = -1;
leafPageIds = null;
- pendingLeafDrawIndex = leafDrawBatchSize;
+ pendingLeafDrawIndex = 0;
+ currentBatchSize = 0;
totalAccessCount = 0;
totalTimeTakenToFindRandomLeaf = 0;
@@ -453,24 +473,22 @@ public final class DiskBTreeSampleCursor extends EnforcedIndexCursor implements
private void releasePage() {
if (page != null) {
- // first page of the cursor.
bufferCache.unpin(page);
}
page = null;
pageId = -1;
}
- // Mutable single-instance view over the SoA draw arrays. Repointed per draw
- // in nextLeafDraw(); never escapes the cursor, so reuse is safe.
+ // One mutable view over the draw arrays, repointed per draw in nextLeafDraw(). Never escapes the cursor,
+ // so the reuse is safe.
private static final class LeafDraw {
private int pageId;
private double acceptanceSample;
- private int tupleStartSeed;
- private LeafDraw(int pageId, double acceptanceSample, int tupleStartSeed) {
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_4_8, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.REFACTORED, notes = "A draw is a page plus its Olken acceptance sample; the slot is drawn where used")
+ private LeafDraw(int pageId, double acceptanceSample) {
this.pageId = pageId;
this.acceptanceSample = acceptanceSample;
- this.tupleStartSeed = tupleStartSeed;
}
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/projection/ColumnProjectorType.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/projection/ColumnProjectorType.java
index 29258b5b70..5d24b322fe 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/projection/ColumnProjectorType.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/projection/ColumnProjectorType.java
@@ -18,8 +18,34 @@
*/
package org.apache.hyracks.storage.am.lsm.btree.column.api.projection;
+/**
+ * What an {@link IColumnProjectionInfo} is going to be read for.
+ * <p>
+ * Compared with {@code ==} only — nothing switches on this type, so a new constant cannot fall through a
+ * {@code default}. Three sites branch on it: {@code CloudColumnReadContext} (which pages to pin),
+ * {@code CloudMegaPageReadContext} (whether to persist cloud-read pages locally) and
+ * {@code CloudColumnIndexDiskCacheManager} (whether to feed the column-eviction planner). On-premise
+ * ({@code DefaultColumnReadContext}) ignores it.
+ */
public enum ColumnProjectorType {
+ /**
+ * Reads <b>every</b> column, so the cloud read context pins the whole mega-leaf in one request instead of
+ * coalescing per-column ranges.
+ */
MERGE,
QUERY,
- MODIFY
+ MODIFY,
+ /**
+ * <b>Existence-only</b>: answers "is this primary key in this component?" and materializes no column value.
+ * From {@link IColumnProjectionInfo#createExistenceOnlyProjectionInfo()}, used only by the sample cursor's
+ * liveness probe. Pins page zero and its segments — the PK binary search reads key values there — and
+ * nothing more, i.e. it pins like {@code QUERY} with an empty projected set.
+ * <p>
+ * Its own constant rather than an existing one because both alternatives misbehave: {@link #MERGE} makes
+ * {@code CloudColumnReadContext#prepareColumns} pin the whole mega-leaf, the exact I/O being avoided, and
+ * {@link #QUERY} feeds the column-eviction planner an access record
+ * ({@code CloudColumnIndexDiskCacheManager#createReadContext}) — a zero-column probe must not register as a
+ * query that touched no columns.
+ */
+ EXISTENCE
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/projection/IColumnProjectionInfo.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/projection/IColumnProjectionInfo.java
index b1bfe87546..bddf156fc6 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/projection/IColumnProjectionInfo.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/api/projection/IColumnProjectionInfo.java
@@ -53,4 +53,27 @@ public interface IColumnProjectionInfo {
* @return the type of {@link IColumnTupleProjector} that created this projection info
*/
ColumnProjectorType getProjectorType();
+
+ /**
+ * A key-only view of this projection, for accessors that only ask "does this key exist in this component?" —
+ * the sample cursor's liveness probe. Existence is settled by the PK binary search
+ * ({@code IColumnTupleIterator#findTupleIndex}, PK values only), so dropping the non-key columns cannot
+ * change an answer; it only avoids pinning mega-pages that are released unread. See
+ * {@code ColumnBTreeExistencePointSearchCursor}.
+ * <p>
+ * A reduced view <b>must</b> report {@link ColumnProjectorType#EXISTENCE}. Zero projected columns is not
+ * enough on cloud storage: {@code CloudColumnReadContext} branches on the projector type first, and for
+ * {@link ColumnProjectorType#MERGE} pins the whole mega-leaf without consulting the projected set at all.
+ * <p>
+ * Returning {@code this} — the default — means "no reduced view available": correct, just slower. An
+ * implementation that no existence probe can reach may instead throw {@link UnsupportedOperationException},
+ * so the assumption stays checked rather than silently supporting a path nothing exercises.
+ *
+ * @return a projection reporting zero projected columns, zero filtered columns and
+ * {@link ColumnProjectorType#EXISTENCE}, or {@code this}
+ * @throws UnsupportedOperationException if this projection is never the one an existence probe is built from
+ */
+ default IColumnProjectionInfo createExistenceOnlyProjectionInfo() {
+ return this;
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/buffercache/read/CloudColumnReadContext.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/buffercache/read/CloudColumnReadContext.java
index c456ab7a6e..68637c36aa 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/buffercache/read/CloudColumnReadContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/buffercache/read/CloudColumnReadContext.java
@@ -19,6 +19,7 @@
package org.apache.hyracks.storage.am.lsm.btree.column.cloud.buffercache.read;
import static org.apache.hyracks.cloud.buffercache.context.DefaultCloudReadContext.readAndPersistPage;
+import static org.apache.hyracks.storage.am.lsm.btree.column.api.projection.ColumnProjectorType.EXISTENCE;
import static org.apache.hyracks.storage.am.lsm.btree.column.api.projection.ColumnProjectorType.MERGE;
import static org.apache.hyracks.storage.am.lsm.btree.column.api.projection.ColumnProjectorType.MODIFY;
import static org.apache.hyracks.storage.am.lsm.btree.column.api.projection.ColumnProjectorType.QUERY;
@@ -68,7 +69,10 @@ public final class CloudColumnReadContext implements IColumnReadContext {
columnCtx = new CloudMegaPageReadContext(operation, columnRanges, drive);
projectedColumns = new BitSet();
mergedPageRanges = AbstractPageRangesComputer.create(MAX_RANGES_COUNT);
- if (operation == QUERY || operation == MODIFY) {
+ // MERGE reads every column, so it never consults projectedColumns (see prepareColumns' pinAll branch) and
+ // the set is deliberately left empty. EXISTENCE reports zero projected columns by construction, so the
+ // loop below is a no-op for it; it is listed anyway so the set always matches what the projection says.
+ if (operation == QUERY || operation == MODIFY || operation == EXISTENCE) {
for (int i = 0; i < projectionInfo.getNumberOfProjectedColumns(); i++) {
int columnIndex = projectionInfo.getColumnIndex(i);
if (columnIndex >= 0) {
@@ -155,6 +159,9 @@ public final class CloudColumnReadContext implements IColumnReadContext {
// Pinning all the segments of the page zero
// as the column eviction logic is based on the length of the columns which
// gets evaluated from the page zero segments.
+ // markAll is for MERGE only: it reads every column, so every segment's offsets are required. EXISTENCE
+ // needs the zeroth segment alone (primary keys are the lowest column indices, hence always in it), so it
+ // marks like a projected read and the unneeded segments are unpinned by columnRanges.reset below.
BitSet pageZeroSegmentRanges =
leafFrame.markRequiredPageZeroSegments(projectedColumns, pageZeroId, operation == MERGE);
// will unpin the non-required segments after columnRanges.reset()
@@ -179,6 +186,9 @@ public final class CloudColumnReadContext implements IColumnReadContext {
pinAll(fileId, pageZeroId + numberOfPageZeroSegments - 1,
leafFrame.getMegaLeafNodeNumberOfPages() - numberOfPageZeroSegments, bufferCache);
} else {
+ // QUERY/MODIFY pin the coalesced ranges of their projected columns. EXISTENCE lands here too, with an
+ // empty projectedColumns set: pinProjected skips primary keys (they live in page zero) so it pins
+ // nothing at all, which is the whole point of an existence-only probe on cloud storage.
pinProjected(fileId, pageZeroId, bufferCache);
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/buffercache/read/CloudMegaPageReadContext.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/buffercache/read/CloudMegaPageReadContext.java
index 7baa20384f..bb47d98a23 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/buffercache/read/CloudMegaPageReadContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/buffercache/read/CloudMegaPageReadContext.java
@@ -129,6 +129,10 @@ public final class CloudMegaPageReadContext implements IBufferCacheReadContext {
* Note: 'empty' can be false while 'cloudOnly is true'. We cannot read from disk as the page can be
* evicted at any moment. In other words, the sweeper told us that it is going to evict this page; hence
* 'cloudOnly' is true.
+ *
+ * Only MERGE is exempted from persisting: an EXISTENCE probe reads a component that stays, and the
+ * only pages it pins through this context are page-zero segments, which every later query needs
+ * anyway -- so it persists them exactly as a QUERY would.
*/
boolean persist = empty && !cloudOnly && !evictable && operation != MERGE && drive.isUnpressured();
readFromStream(ioManager, fileHandle, header, cPage, persist, threadStats);
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTree.java
index fca5a9dfdf..95e8fc0037 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTree.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTree.java
@@ -122,14 +122,14 @@ public class ColumnBTree extends DiskBTree {
@Override
public ITreeIndexCursor createSampleCursor(long componentSampleCardinality, long sampleSeed,
ILSMIndexBatchPointCursor searchCursor, int maxLeafAttempts, int leafDrawBatchSize,
- int maxLeafTupleCount, int samplesPerPage, ITupleAcceptor antimatterAcceptor) {
+ int maxLeafTupleCount, ITupleAcceptor antimatterAcceptor) {
// antimatterAcceptor is unused here: column tuples (IColumnTupleIterator extends ILSMTreeTupleReference)
// report antimatter directly, so ColumnBtreeSampleCursor checks it without the injected acceptor.
ColumnBTreeLeafFrameFactory columnLeafFrameFactory = (ColumnBTreeLeafFrameFactory) leafFrameFactory;
ColumnBTreeReadLeafFrame readLeafFrame = columnLeafFrameFactory.createReadFrame(projectionInfo);
return new ColumnBtreeSampleCursor((ColumnBTree) btree, readLeafFrame, ctx, context,
componentSampleCardinality, sampleSeed, index, searchCursor, maxLeafAttempts, leafDrawBatchSize,
- maxLeafTupleCount, samplesPerPage);
+ maxLeafTupleCount);
}
@Override
@@ -142,6 +142,15 @@ public class ColumnBTree extends DiskBTree {
index, context);
}
+ public ColumnBTreeExistencePointSearchCursor createExistencePointCursor() {
+ ColumnBTreeLeafFrameFactory columnLeafFrameFactory = (ColumnBTreeLeafFrameFactory) leafFrameFactory;
+ ColumnBTreeReadLeafFrame readLeafFrame = columnLeafFrameFactory.createReadFrame(projectionInfo);
+ return new ColumnBTreeExistencePointSearchCursor(
+ readLeafFrame, (IIndexCursorStats) iap.getParameters()
+ .getOrDefault(HyracksConstants.INDEX_CURSOR_STATS, NoOpIndexCursorStats.INSTANCE),
+ index, context);
+ }
+
@Override
protected IBufferCacheReadContext getBufferCacheOperationContext() {
return context;
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeExistencePointSearchCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeExistencePointSearchCursor.java
new file mode 100644
index 0000000000..7900841a23
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeExistencePointSearchCursor.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.btree.column.impls.btree;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.storage.am.lsm.btree.column.cloud.buffercache.IColumnReadContext;
+import org.apache.hyracks.storage.common.IIndexCursorStats;
+import org.apache.hyracks.storage.common.ISearchPredicate;
+import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.util.annotations.AiProvenance;
+
+/**
+ * Point-search cursor answering <b>existence only</b>: {@code hasNext()} says whether the key is physically on
+ * the leaf page, antimatter included, and nothing is positioned for reading. Its one consumer is the sample
+ * cursor's liveness probe ({@code LSMBTreePointSearchCursor#keyExistsIncludingAntimatter}); wired only from
+ * {@code LSMColumnBTree#createSampleLivenessSearchCursor}. Never hand it to a query path.
+ * <p>
+ * <b>Same verdict as {@link ColumnBTreePointSearchCursor}, key for key.</b> The base cursor's verdict is a pure
+ * function of {@code getLowKeyIndex()}: with no high key, {@code end} is always the last slot, so a match yields
+ * {@code start <= end} and {@code shouldYieldFirstCall()} then re-compares the key against the very index the
+ * exact match was found at — {@code 0} by construction. The {@code frameTuple.reset(...)} in between pins and
+ * resets every projected column and cannot change the answer, only its cost. So this cursor computes the index
+ * and stops. {@code findTupleIndex} random-accesses PK values ({@code setKeyAt} → {@code reader.getValue}),
+ * touching no definition levels and pinning no column page, so with a key-only projection the probe reads page
+ * zero and nothing else.
+ * <p>
+ * <b>Antimatter counts as present</b>, and must: a newer delete shadows the older live tuple, so reporting it
+ * absent would leak logically-deleted rows into the sample. PK values are written for antimatter too, so
+ * {@code findTupleIndex} matches it like any insert.
+ * <p>
+ * <b>Same-page search reuse is disabled</b> ({@link #getLastPageId()} returns
+ * {@link IBufferCache#INVALID_PAGEID}, so {@code DiskBTree#search} always descends from the root): without a
+ * positioned {@code frameTuple} there is no monotonic {@code tupleIndex} for it to build on, and the sampler's
+ * probe keys are not ascending anyway.
+ */
+@AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_4_8, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.GENERATED, notes = "Existence-only column point-search cursor for the sample cursor's newer-component liveness probe")
+public final class ColumnBTreeExistencePointSearchCursor extends ColumnBTreePointSearchCursor {
+
+ public ColumnBTreeExistencePointSearchCursor(ColumnBTreeReadLeafFrame frame, IIndexCursorStats stats, int index,
+ IColumnReadContext context) {
+ super(frame, stats, index, context);
+ }
+
+ @Override
+ protected void initCursorPosition(ISearchPredicate searchPred) throws HyracksDataException {
+ setSearchPredicate(searchPred);
+ yieldFirstCall = false;
+ // Settled here and nowhere else: no setCursorPosition(), so no frameTuple.reset() and no column
+ // materialization.
+ if (getExactTupleIndex() < 0) {
+ // Leaves the frame tuple consumed, as the base class's "start > end" branch does.
+ frameTuple.consume();
+ } else {
+ yieldFirstCall = true;
+ }
+ }
+
+ /**
+ * @return the exactly-matched index, or a negative indicator. Unlike
+ * {@link ColumnBTreePointSearchCursor#getLowKeyIndex()} the negative is returned as-is rather than
+ * folded into {@code getTupleCount()}: here "absent" is the answer.
+ */
+ private int getExactTupleIndex() throws HyracksDataException {
+ return frameTuple.findTupleIndex(pred.getLowKey(), pred.getLowKeyComparator(), lowKeyFtm, lowKeyFtp);
+ }
+
+ @Override
+ public int getLastPageId() {
+ return IBufferCache.INVALID_PAGEID;
+ }
+
+ @Override
+ public void setCursorToNextKey(ISearchPredicate searchPred) throws HyracksDataException {
+ throw new UnsupportedOperationException(
+ "same-page search reuse is disabled for " + getClass().getSimpleName() + "; getLastPageId() is "
+ + "always INVALID_PAGEID so DiskBTree#search must always descend from the root");
+ }
+
+ /**
+ * @throws UnsupportedOperationException always: the frame tuple is never positioned, so returning it would
+ * hand out unpositioned column state. Better a crash than a silently wrong tuple.
+ */
+ @Override
+ public ITupleReference doGetTuple() {
+ throw new UnsupportedOperationException(
+ getClass().getSimpleName() + " answers existence only; its tuple is never positioned");
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBtreeSampleCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBtreeSampleCursor.java
index 67aab71dff..1706d192d8 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBtreeSampleCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBtreeSampleCursor.java
@@ -49,23 +49,24 @@ import it.unimi.dsi.fastutil.longs.LongOpenHashSet;
import it.unimi.dsi.fastutil.longs.LongSet;
/**
- * Two-phase column sample cursor.
+ * Two-phase column sample cursor. Splitting selection from collection is what keeps the cost proportional to the
+ * sample rather than to the component: phase 1 rejects most candidates, and only survivors are worth a mega-page
+ * load.
* <p>
- * <b>Phase 1 — Selection (page0-only):</b> picks random leaf pages via batched
- * sorted draws to minimize random I/O, pins only page0, uses
- * {@link AbstractColumnTupleReference#startSamplingPage} +
- * {@link AbstractColumnTupleReference#seekForwardPKOnly} to position PKs without
- * loading column mega-pages. Per-page candidate indices are sorted ascending so the
- * sequential PK level decoders are scanned forward-only (no rewind). Checks antimatter
- * and newer-component existence from PK data alone. Accepted tuples are recorded as
- * packed {@code (pageId << 32) | tupleIndex} longs.
+ * <b>Phase 1 — selection, page0 only.</b> Draws random leaf pages, pins page0, and decides antimatter and
+ * newer-component liveness from PK data alone ({@link AbstractColumnTupleReference#startSamplingPage} +
+ * {@link AbstractColumnTupleReference#seekForwardPKOnly}), so no column mega-pages are touched. Survivors are
+ * recorded as packed {@code (pageId << 32) | tupleIndex}.
* <p>
- * <b>Phase 2 — Collection (sorted, full column load):</b> the collected
- * samples are sorted by pageId (then tupleIndex within a page). For each
- * distinct page we do one full column load via {@code reset()}, then
- * advance to additional tuples on the same page via the cheap forward-only
- * {@code setAt()}. This amortises mega-page I/O and gives cache-friendly
- * sequential access.
+ * <b>Phase 2 — collection, sorted, full column load.</b> Survivors sorted by pageId, so each page's mega-pages
+ * are loaded once and further tuples on it reached by the forward-only {@code setAt()}.
+ * <p>
+ * Draws are sorted by pageId for sequential page0 I/O, which also makes same-page draws contiguous — they are
+ * consumed as one <b>page group</b> sharing a single rewind and forward PK pass, instead of one scan per draw.
+ * <p>
+ * <b>Every sort here orders a set that is then consumed in full.</b> That is the load-bearing property: a
+ * pageId-sorted set consumed only partly keeps the lowest pageIds, which is a spatial filter, not a sample. See
+ * {@link #runPhase1Selection()}.
*/
public class ColumnBtreeSampleCursor extends EnforcedIndexCursor implements ITreeIndexCursor, IColumnReadMultiPageOp {
@@ -81,39 +82,30 @@ public class ColumnBtreeSampleCursor extends EnforcedIndexCursor implements ITre
// u64: (pageId << 32) | tupleIndex
private final LongSet seenTupleIndexes;
- // Multiplier applied to the component tuple-population ceiling to derive the
- // effective consecutive-miss give-up threshold (see doOpen). Mirrors
- // DiskBTreeSampleCursor: the reservoir stalls well short of the target when the
- // threshold only equals the population, so a multiple well above that is needed.
- @AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_4_8, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.GENERATED, notes = "Population-scaled give-up threshold to stop near-exhaustive samples truncating at ~500 attempts")
- private static final long POPULATION_ATTEMPT_MULTIPLIER = 32L;
+ // Give-up headroom must scale with the target, not be fixed: the coupon-collector tail near the end of
+ // collection grows with it, so a fixed threshold truncates near-exhaustive samples. Mirrors
+ // DiskBTreeSampleCursor. Scaled in doOpen().
+ private static final long SAMPLE_ATTEMPT_MULTIPLIER = 32L;
private final int maxLeafFindingAttempts;
- // Effective consecutive-miss give-up threshold, scaled to the component's tuple
- // population in doOpen(). A fixed maxLeafFindingAttempts silently truncates
- // near-exhaustive samples (target close to the component's live-tuple count) as
- // random draws increasingly hit already-collected pages/tuples; scaling to the
- // population keeps collection from stopping short while bounding the
- // genuinely-exhausted case to O(population) draws. Mirrors DiskBTreeSampleCursor.
private int effectiveMaxLeafFindingAttempts;
- private final int samplesPerPage;
- // Max number of random probes per page (oversampled vs samplesPerPage to absorb
- // antimatter / collision / newer-component rejections).
- private final int maxPageSlotRetries;
- // Reused scratch buffer of per-page candidate tuple indices, sorted ascending so the
- // PK level decoders can be scanned forward-only (no rewind) within a page.
- private final int[] pageTupleCandidates;
private final long componentSampleCardinality;
private final Random randomNumGen;
- // Phase 1 batched I/O draws stored as flat parallel primitive arrays (SoA).
- // Sorted via a packed long[] key ((pageId << 32) | drawIndex) so page0 pins
- // proceed in ascending pageId order without boxed-object comparator overhead.
+ // Draws as parallel primitive arrays, sorted via a packed long key ((pageId << 32) | drawIndex) so the
+ // page0 pins go in ascending pageId order without a boxed comparator.
private final double[] drawAcceptanceSamples;
- private final int[] drawTupleStartSeeds;
private final long[] drawSortKeys;
- private final LeafDraw leafDrawView;
private int pendingLeafDrawIndex;
- private final int leafDrawBatchSize;
+
+ // A page group's candidate slots, packed (tupleIndex << 32) | drawOrdinal: one primitive sort then orders
+ // them ascending by slot for the forward-only PK pass, while still saying which draw each slot came from
+ // (the per-draw give-up accounting needs that).
+ private long[] groupCandidates;
+ // Did this draw of the current group collect anything? Drives the give-up counter.
+ private final boolean[] drawCollectedAny;
+ private final int drawBatchCapacity;
+ // Every draw of a batch is consumed before the next refill; see refillLeafDrawBatch.
+ private int currentBatchSize;
// search predicate
private final ILSMIndexBatchPointCursor searchCursor;
@@ -128,16 +120,26 @@ public class ColumnBtreeSampleCursor extends EnforcedIndexCursor implements ITre
// Static upper bound for rejection sampling
private final int leafTupleCapacity;
- // Cached once: the per-draw / per-yield System.nanoTime() timing is consumed
- // only by the trace log in doClose(). Gating on this flag removes those
- // syscall-class calls from the hot path when trace logging is disabled.
+ // Cached so the per-draw System.nanoTime() calls leave the hot path when trace logging is off; the only
+ // consumer is doClose()'s trace line.
private final boolean traceTimingEnabled;
- // Debug and traceability
private long totalTimeTakenToFindRandomLeaf = 0;
private long totalTimeTakenToFindRandomTuples = 0;
private boolean endedPreemptively = false;
+ // Null in production scans (zero overhead); attached by the perf harness.
+ private SampleCursorStats stats;
+
+ public void setStats(SampleCursorStats stats) {
+ this.stats = stats;
+ }
+
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_4_8, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.GENERATED, notes = "Cost-breakdown stats accessor")
+ public SampleCursorStats getStats() {
+ return stats;
+ }
+
private ICachedPage page0 = null;
private int rootPageId;
private int[] leafPageIds = null;
@@ -157,7 +159,7 @@ public class ColumnBtreeSampleCursor extends EnforcedIndexCursor implements ITre
public ColumnBtreeSampleCursor(ColumnBTree columnBTree, ColumnBTreeReadLeafFrame leafFrame,
BTreeOpContext opContext, IColumnReadContext context, long componentSampleCardinality, long sampleSeed,
int index, ILSMIndexBatchPointCursor searchCursor, int maxLeafFindingAttempts, int leafDrawBatchSize,
- int maxLeafTupleCount, int samplesPerPage) {
+ int maxLeafTupleCount) {
this.bTree = columnBTree;
this.opCtx = opContext;
this.leafFrame = leafFrame;
@@ -173,21 +175,23 @@ public class ColumnBtreeSampleCursor extends EnforcedIndexCursor implements ITre
this.seenTupleIndexes = new LongOpenHashSet();
this.totalAccessCount = 0;
this.maxLeafFindingAttempts = maxLeafFindingAttempts;
- this.samplesPerPage = Math.max(1, samplesPerPage);
- // Collision probability ≈ samplesPerPage/tupleCount ≪ 1 for typical configs, so 5× is conservative.
- this.maxPageSlotRetries = this.samplesPerPage * 5;
- this.pageTupleCandidates = new int[this.maxPageSlotRetries];
this.leafTupleCapacity = maxLeafTupleCount;
this.traceTimingEnabled = LOGGER.isTraceEnabled();
- this.leafDrawBatchSize = (int) Math.max(leafDrawBatchSize, componentSampleCardinality);
- this.drawAcceptanceSamples = new double[this.leafDrawBatchSize];
- this.drawTupleStartSeeds = new int[this.leafDrawBatchSize];
- this.drawSortKeys = new long[this.leafDrawBatchSize];
- this.leafDrawView = new LeafDraw(-1, 0.0, 0);
-
- // FIX: Force a batch refill on the very first nextLeafDraw() call
- this.pendingLeafDrawIndex = this.leafDrawBatchSize;
+ // A draw yields at most one sample, so a batch beyond the outstanding shortfall is waste, and it must
+ // be consumed whole (refillLeafDrawBatch). The configured size stays an upper bound -- an I/O-locality
+ // knob.
+ long capacity = leafDrawBatchSize > 0 ? Math.min(leafDrawBatchSize, componentSampleCardinality)
+ : componentSampleCardinality;
+ this.drawBatchCapacity = (int) Math.max(1, Math.min(Integer.MAX_VALUE, capacity));
+ this.drawAcceptanceSamples = new double[this.drawBatchCapacity];
+ this.drawSortKeys = new long[this.drawBatchCapacity];
+ this.drawCollectedAny = new boolean[this.drawBatchCapacity];
+ this.groupCandidates = new long[16];
+
+ // Force a batch refill on the very first draw.
+ this.pendingLeafDrawIndex = 0;
+ this.currentBatchSize = 0;
}
@Override
@@ -213,19 +217,23 @@ public class ColumnBtreeSampleCursor extends EnforcedIndexCursor implements ITre
rootPageId = ((BTreeCursorInitialState) initialState).getPageId();
leafPageIds = bTree.enumerateLeafPageIds(rootPageId, opCtx, context);
- // Scale the give-up threshold to the component's tuple-population ceiling
- // (leaf pages x max tuples/leaf), clamped to int to avoid an overflowed
- // negative cap on huge components (where the target is far below the
- // population and the loop exits on reaching the target regardless).
- long populationCeiling = (long) leafPageIds.length * leafTupleCapacity;
- effectiveMaxLeafFindingAttempts = (int) Math.min(Integer.MAX_VALUE,
- Math.max(maxLeafFindingAttempts, POPULATION_ATTEMPT_MULTIPLIER * populationCeiling));
-
- collectedSamples = new long[(int) componentSampleCardinality];
+ // Scale to the TARGET, not the tuple population: a bound of MULTIPLIER * leafPages * maxLeafTupleCount
+ // is ~32x total tuples, so an unreachable target ground through that many attempts however small the
+ // target was. Harmless on row, fatal on column -- a column attempt re-inits the PK decoders and walks
+ // the PK column, so the same count is milliseconds there and hours here. Target-scaling makes it
+ // O(target). The leafPages floor lets a tiny target still reach every page; the clamp stops the product
+ // overflowing negative.
+ long attemptCeiling = SAMPLE_ATTEMPT_MULTIPLIER * Math.max(leafPageIds.length, componentSampleCardinality);
+ effectiveMaxLeafFindingAttempts =
+ (int) Math.min(Integer.MAX_VALUE, Math.max(maxLeafFindingAttempts, attemptCeiling));
+
+ collectedSamples = new long[(int) Math.min(Integer.MAX_VALUE, componentSampleCardinality)];
collectedCount = 0;
selectionDone = false;
yieldPos = 0;
prevYieldPageId = -1;
+ pendingLeafDrawIndex = 0;
+ currentBatchSize = 0;
}
@Override
@@ -241,106 +249,187 @@ public class ColumnBtreeSampleCursor extends EnforcedIndexCursor implements ITre
// Phase 1: PK-only selection (Batched & Sorted for sequential I/O)
// ──────────────────────────────────────────────────────────────────────
+ /**
+ * Collects up to {@code componentSampleCardinality} distinct live tuples.
+ * <p>
+ * <b>The result is an exactly uniform sample without replacement.</b> A draw picks a page uniformly
+ * ({@code 1/P}), accepts it with the Olken & Rotem fill correction ({@code n/C}), then probes one of the
+ * page's {@code n} slots ({@code 1/n}). The product {@code 1/(P·C)} is free of {@code n}: cancelling the
+ * page's fill is the whole point of the Olken step, and dropping it would over-represent tuples on
+ * partly-filled pages.
+ * <p>
+ * Uniformity then rests on every stopping rule being a function of <b>counts only</b>, since such rules
+ * commute with relabelling the live tuples. Truncating a pageId-sorted batch is the one rule that is not —
+ * it keeps precisely the lowest pageIds — hence full-batch consumption ({@link #refillLeafDrawBatch()}).
+ * Page grouping is likewise safe because it only reorders probes: the RNG is consumed identically, and the
+ * result is order-insensitive (set-based dedup, sorted before phase 2, count-based stopping).
+ * <p>
+ * <b>Taking several slots per page visit would break this</b>, and was removed: the tuples of one visit
+ * share a page, so the sample becomes a cluster sample over contiguous stretches of the key space —
+ * measured chi-square well past its critical value — while the per-page amortization that once justified it
+ * is now unconditional, since a batch's same-page draws share one primary-key pass regardless.
+ * <p>
+ * One caveat: a component without {@code maxLeafTupleCount} metadata ({@code leafTupleCapacity == 0}) cannot
+ * be fill-corrected, which re-weights tuples by {@code 1/n}.
+ * <p>
+ * The attempts cap is tested only at batch boundaries, so it can overshoot by one batch — count-based
+ * either way, so bounded extra work and no effect on uniformity.
+ */
private void runPhase1Selection() throws HyracksDataException {
- while (collectedCount < componentSampleCardinality && hasNextAttemptCount < effectiveMaxLeafFindingAttempts) {
- LeafDraw leafDraw = nextLeafDraw();
- if (leafDraw == null) {
- break;
- }
-
- totalAccessCount++;
-
- int selectedPageId = pinAndAcceptLeafPage0(leafDraw);
- if (selectedPageId == -1) {
- hasNextAttemptCount++;
- continue;
- }
-
- int collectedFromPage = collectFromPage(selectedPageId);
-
- if (collectedFromPage > 0) {
- hasNextAttemptCount = 0;
- } else {
- hasNextAttemptCount++;
+ while (true) {
+ if (pendingLeafDrawIndex >= currentBatchSize) {
+ // The ONLY legal stopping point: mid-batch would keep just the sorted batch's low-pageId
+ // prefix, a spatial filter rather than a sample.
+ if (hasNextAttemptCount >= effectiveMaxLeafFindingAttempts) {
+ break;
+ }
+ refillLeafDrawBatch();
+ if (currentBatchSize == 0) {
+ // Target reached, or the component has no leaf pages.
+ break;
+ }
}
+ visitNextPageGroup();
}
endedPreemptively = (collectedCount < componentSampleCardinality);
unpinCurrentPage0();
- // Sort by pageId (high 32 bits) then tupleIndex (low 32 bits) for Phase 2
+ // pageId (high 32) then tupleIndex (low 32), so phase 2 loads each page's columns once.
Arrays.sort(collectedSamples, 0, collectedCount);
if (LOGGER.isDebugEnabled()) {
- LOGGER.debug("ColumnBtreeSampleCursor Phase 1: collected {} samples from {} leaf pages", collectedCount,
- leafPageIds.length);
+ // endedPreemptively with attempts == cap means the target was unreachable (near-exhaustive or
+ // over-estimated); attempts well below cap means it was met.
+ LOGGER.debug(
+ "ColumnBtreeSampleCursor Phase 1: collected {}/{} samples from {} leaf pages, "
+ + "attempts {}/{}, endedPreemptively {}",
+ collectedCount, componentSampleCardinality, leafPageIds.length, hasNextAttemptCount,
+ effectiveMaxLeafFindingAttempts, endedPreemptively);
}
}
- private LeafDraw nextLeafDraw() {
- if (pendingLeafDrawIndex >= leafDrawBatchSize) {
- refillLeafDrawBatch();
- if (leafPageIds == null || leafPageIds.length == 0) {
- return null;
+ /**
+ * Consumes one <b>page group</b> — the maximal run of same-pageId draws, contiguous because the batch is
+ * pageId-sorted — with a single page0 pin, rewind and forward PK pass, instead of one per draw.
+ * <p>
+ * Two invariants keep this a pure reordering rather than a different sampling design
+ * ({@link #runPhase1Selection()}): a group never crosses a batch boundary and is always consumed whole, so
+ * the legal stopping points are unchanged; and the RNG is consumed in the same order and quantity, since
+ * draws are visited in ascending draw index and only accepted ones pull slots.
+ * <p>
+ * Per-draw give-up accounting is replayed after the pass rather than interleaved. Equivalent, because
+ * nothing between a group's first and last draw reads {@link #hasNextAttemptCount} — it is tested only at a
+ * batch boundary.
+ */
+ private void visitNextPageGroup() throws HyracksDataException {
+ int groupStart = pendingLeafDrawIndex;
+ // High 32 bits of the sort key ARE the pageId; the low 32 index back into the unsorted side arrays.
+ int pageId = (int) (drawSortKeys[groupStart] >>> 32);
+ int groupEnd = groupStart + 1;
+ while (groupEnd < currentBatchSize && (int) (drawSortKeys[groupEnd] >>> 32) == pageId) {
+ groupEnd++;
+ }
+ pendingLeafDrawIndex = groupEnd;
+ int groupSize = groupEnd - groupStart;
+ totalAccessCount += groupSize;
+ if (stats != null) {
+ stats.pageGroups++;
+ }
+
+ Arrays.fill(drawCollectedAny, 0, groupSize, false);
+ int tupleCount = pinLeafPage0(pageId);
+ if (tupleCount == 0) {
+ // An empty page rejects every draw that landed on it.
+ if (stats != null) {
+ stats.attempts += groupSize;
+ stats.pagesRejected += groupSize;
+ }
+ } else {
+ int candidateCount = drawGroupCandidates(groupStart, groupEnd, tupleCount);
+ if (candidateCount > 0) {
+ // Ascending by slot (then by draw ordinal), so the forward-only decoders never rewind.
+ Arrays.sort(groupCandidates, 0, candidateCount);
+ probeGroupCandidates(pageId, candidateCount);
+ } else {
+ // No draw survived fill-rejection: nothing to probe, so do not hold the page.
+ unpinCurrentPage0();
+ }
+ }
+
+ for (int drawOrdinal = 0; drawOrdinal < groupSize; drawOrdinal++) {
+ if (drawCollectedAny[drawOrdinal]) {
+ hasNextAttemptCount = 0;
+ } else {
+ hasNextAttemptCount++;
}
}
- // Sort key packs (pageId << 32 | drawIndex): high 32 bits ARE the pageId (no separate array),
- // low 32 bits index back into the unsorted SoA side arrays.
- long sortKey = drawSortKeys[pendingLeafDrawIndex++];
- int drawIndex = (int) sortKey;
- leafDrawView.pageId = (int) (sortKey >>> 32);
- leafDrawView.acceptanceSample = drawAcceptanceSamples[drawIndex];
- leafDrawView.tupleStartSeed = drawTupleStartSeeds[drawIndex];
- return leafDrawView;
}
+ /**
+ * Draws the next batch of leaf-page visits, pageId-sorted so the page0 pins go in ascending order.
+ * <p>
+ * <b>The batch is sized to the outstanding shortfall so it can be consumed whole.</b> Sorting is only a
+ * permutation, and a permutation changes nothing — <em>provided every draw is consumed</em>. Truncating a
+ * pageId-sorted batch keeps precisely the lowest pageIds, turning the sort into a spatial filter; a fixed
+ * 32768-draw batch guaranteed that, since the target was met within a short prefix. At {@code remaining}
+ * draws the target can only be reached on the last one, because a draw yields at most one sample — so the
+ * caller can consume all of it. Total draws are unchanged; only the sort windows shrink.
+ * <p>
+ * The configured {@code leafDrawBatchSize} therefore no longer binds unless set below the target. It is kept
+ * as the ceiling on the draw-buffer allocation, and a smaller batch is still consumed whole, so it can only
+ * change pin locality, never which tuples are sampled.
+ * <p>
+ * Sets {@link #currentBatchSize} to 0 when nothing more is needed (target met) or possible (no leaf pages).
+ */
private void refillLeafDrawBatch() {
pendingLeafDrawIndex = 0;
- if (leafPageIds == null || leafPageIds.length == 0 || leafDrawBatchSize <= 0) {
+ currentBatchSize = 0;
+ if (leafPageIds == null || leafPageIds.length == 0) {
+ return;
+ }
+ long remaining = componentSampleCardinality - collectedCount;
+ if (remaining <= 0) {
return;
}
- for (int i = 0; i < leafDrawBatchSize; i++) {
+ int batchSize = (int) Math.min(drawBatchCapacity, remaining);
+ for (int i = 0; i < batchSize; i++) {
int randomLeafIndex = randomNumGen.nextInt(leafPageIds.length);
int targetPageId = leafPageIds[randomLeafIndex];
drawAcceptanceSamples[i] = randomNumGen.nextDouble();
- drawTupleStartSeeds[i] = randomNumGen.nextInt();
drawSortKeys[i] = (((long) targetPageId) << 32) | (i & 0xffffffffL);
}
- // Sorting Phase 1 draws dramatically reduces random disk seeks for page0.
- // Primitive long[] sort avoids the boxed-object comparator dispatch.
- Arrays.sort(drawSortKeys);
+ // Cuts random page0 seeks; primitive sort avoids boxed comparator dispatch.
+ Arrays.sort(drawSortKeys, 0, batchSize);
+ currentBatchSize = batchSize;
}
- private int pinAndAcceptLeafPage0(LeafDraw leafDraw) throws HyracksDataException {
+ /**
+ * Pins the group's page0 (reusing the pin if it is already the one we hold) and returns its tuple count, or
+ * {@code 0} — having released the page — when the page holds no tuples.
+ * <p>
+ * The Olken & Rotem fill test is not here: it is per draw, not per page, so it lives in
+ * {@link #drawGroupCandidates(int, int, int)}. Splitting it out is what lets one pin serve a whole group.
+ */
+ private int pinLeafPage0(int pageId) throws HyracksDataException {
long nanos = traceTimingEnabled ? System.nanoTime() : 0L;
try {
- // Unpin previous page0 if we are moving to a new page
- if (page0 != null && prevYieldPageId != leafDraw.pageId) {
+ if (page0 != null && prevYieldPageId != pageId) {
unpinCurrentPage0();
}
if (page0 == null) {
- ICachedPage randomLeafPage =
- bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, leafDraw.pageId), context);
+ ICachedPage randomLeafPage = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), context);
leafFrame.setPage(randomLeafPage);
page0 = leafFrame.getPage();
- prevYieldPageId = leafDraw.pageId;
+ prevYieldPageId = pageId;
}
int tupleCount = leafFrame.getTupleCount();
if (tupleCount == 0) {
unpinCurrentPage0();
- return -1;
- }
-
- // Rejection sampling against static capacity
- double acceptProb = (double) tupleCount / leafTupleCapacity;
- if (leafDraw.acceptanceSample >= acceptProb) {
- unpinCurrentPage0();
- return -1;
}
-
- return leafDraw.pageId;
+ return tupleCount;
} finally {
if (traceTimingEnabled) {
totalTimeTakenToFindRandomLeaf += (System.nanoTime() - nanos);
@@ -349,73 +438,133 @@ public class ColumnBtreeSampleCursor extends EnforcedIndexCursor implements ITre
}
/**
- * Collects up to {@code samplesPerPage} live tuples from one accepted page using a single
- * forward-only pass over the PK level decoders.
+ * Applies the fill-proportional (Olken & Rotem) acceptance test to each draw of the group and gives every
+ * survivor one candidate slot, packed into {@link #groupCandidates} as {@code (tupleIndex << 32) | drawOrdinal}.
+ * <p>
+ * One slot per draw is what makes a tuple's per-draw probe probability {@code 1 / tupleCount}, cancelling the
+ * page's fill against the Olken acceptance probability ({@link #runPhase1Selection()}). Distinct draws are
+ * independent trials and may pick the same slot.
* <p>
- * The PK definition-level decoder is sequential and cannot seek backwards, so we pre-draw the
- * candidate tuple indices, sort them ascending, and probe them in order via
- * {@link AbstractColumnTupleReference#seekForwardPKOnly(int)}. This scans the page once
- * ({@code O(maxIndex)} level reads) instead of rewinding the decoder per probe. Antimatter,
- * collision and newer-component rejections simply advance the forward scan.
+ * <b>Every candidate is probed</b> — the intra-page form of the full-batch rule. Candidates are
+ * slot-ascending, so stopping at the first acceptable one would select the <em>minimum</em> of several
+ * uniform draws and pull every page towards its low slots.
*
- * @return the number of live tuples collected from this page
+ * @return the number of candidate slots written to {@link #groupCandidates}
*/
- private int collectFromPage(int selectedPageId) throws HyracksDataException {
- long nanos = traceTimingEnabled ? System.nanoTime() : 0L;
- try {
- int tupleCount = leafFrame.getTupleCount();
- int drawCount = Math.min(maxPageSlotRetries, tupleCount);
-
- // Pre-draw candidate tuple indices and sort ascending for a monotonic forward scan.
- for (int i = 0; i < drawCount; i++) {
- pageTupleCandidates[i] = randomNumGen.nextInt(tupleCount);
+ private int drawGroupCandidates(int groupStart, int groupEnd, int tupleCount) {
+ ensureGroupCandidateCapacity(groupEnd - groupStart);
+ // Unknown capacity (pre-upgrade component, no metadata) => accept every page. Mirrors
+ // DiskBTreeSampleCursor#pinAndAcceptLeafPage.
+ boolean applyFillRejection = leafTupleCapacity > 0;
+ double acceptProb = applyFillRejection ? (double) tupleCount / leafTupleCapacity : 1.0;
+
+ int candidateCount = 0;
+ // Ascending draw index: the order the sorted batch would have visited these anyway.
+ for (int i = groupStart; i < groupEnd; i++) {
+ int drawIndex = (int) drawSortKeys[i];
+ if (applyFillRejection && drawAcceptanceSamples[drawIndex] >= acceptProb) {
+ if (stats != null) {
+ stats.attempts++;
+ stats.pagesRejected++;
+ }
+ continue;
}
- Arrays.sort(pageTupleCandidates, 0, drawCount);
+ if (stats != null) {
+ stats.attempts++;
+ stats.pagesAccepted++;
+ }
+ int drawOrdinal = i - groupStart;
+ groupCandidates[candidateCount++] =
+ (((long) randomNumGen.nextInt(tupleCount)) << 32) | (drawOrdinal & 0xffffffffL);
+ }
+ return candidateCount;
+ }
- // Rewind the PK level decoders once for this page; subsequent seeks move forward only.
+ /**
+ * Probes the group's slot-ascending candidates in one forward-only pass, collecting the live ones.
+ * <p>
+ * The PK definition-level decoder is sequential and cannot seek backwards — hence the ascending order and
+ * the single {@link AbstractColumnTupleReference#startSamplingPage()} rewind. One
+ * {@code O(maxCandidateIndex)} scan serves the whole group. Rejections just advance the scan.
+ * <p>
+ * <b>Repeated slots must not be re-seeked.</b> Two draws may pick the same slot, and
+ * {@link AbstractColumnTupleReference#seekForwardPKOnly(int)} computes {@code startIndex - tupleIndex - 1},
+ * so a repeat would pass a skip of {@code -1} and desync the decoder. The reader is already there, so the
+ * repeat just re-runs the liveness decision.
+ */
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_4_8, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.ASSISTED, notes = "PK-seek and liveness timing, non-overlapping")
+ private void probeGroupCandidates(int pageId, int candidateCount) throws HyracksDataException {
+ long nanos = (traceTimingEnabled || stats != null) ? System.nanoTime() : 0L;
+ // Subtracted from the method-wide window before it lands in stats.pkSeekNanos: the liveness check runs
+ // inside this timed span, so without this pkSeekNanos would contain it and dilute livenessSharePct().
+ long livenessNanosThisCall = 0L;
+ try {
columnTupleRef.startSamplingPage();
+ int lastProbedIndex = -1;
- int collectedFromPage = 0;
- int prevTupleIndex = -1;
- for (int c = 0; c < drawCount && collectedFromPage < samplesPerPage
- && collectedCount < componentSampleCardinality; c++) {
- int tupleIndex = pageTupleCandidates[c];
- if (tupleIndex == prevTupleIndex) {
- // Duplicate draw within the sorted batch; the decoder cannot revisit it anyway.
- continue;
- }
- prevTupleIndex = tupleIndex;
+ // Every candidate is probed: stopping early biases towards low slots (drawGroupCandidates).
+ for (int c = 0; c < candidateCount; c++) {
+ long packed = groupCandidates[c];
+ int tupleIndex = (int) (packed >>> 32);
+ int drawOrdinal = (int) packed;
- columnTupleRef.seekForwardPKOnly(tupleIndex);
+ if (tupleIndex != lastProbedIndex) {
+ columnTupleRef.seekForwardPKOnly(tupleIndex);
+ lastProbedIndex = tupleIndex;
+ }
+ if (stats != null) {
+ stats.pkSeekCalls++;
+ }
if (frameTuple.isAntimatter()) {
continue;
}
- long pageTupleKey = getPageTupleKey(selectedPageId, tupleIndex);
+ long pageTupleKey = getPageTupleKey(pageId, tupleIndex);
if (seenTupleIndexes.contains(pageTupleKey)) {
continue;
}
- // Newer-component check (uses PK fields from page0)
+ // Newer-component liveness, from page0's PK fields alone.
searchKeys.clear();
foundIndexes.clear();
searchKeys.add(frameTuple);
batchPredicate.reset(searchKeys);
searchCursor.setPredicate(batchPredicate);
+ long livenessStart = stats != null ? System.nanoTime() : 0L;
searchCursor.hasNextWithPredicate(foundIndexes);
+ if (stats != null) {
+ long livenessElapsed = System.nanoTime() - livenessStart;
+ stats.livenessNanos += livenessElapsed;
+ livenessNanosThisCall += livenessElapsed;
+ stats.livenessCalls++;
+ stats.livenessKeys++;
+ }
if (!foundIndexes.isEmpty()) {
continue;
}
seenTupleIndexes.add(pageTupleKey);
collectedSamples[collectedCount++] = pageTupleKey;
- collectedFromPage++;
+ drawCollectedAny[drawOrdinal] = true;
}
- return collectedFromPage;
} finally {
if (traceTimingEnabled) {
totalTimeTakenToFindRandomTuples += (System.nanoTime() - nanos);
}
+ if (stats != null) {
+ stats.pkSeekNanos += System.nanoTime() - nanos - livenessNanosThisCall;
+ }
+ }
+ }
+
+ /**
+ * Grows {@link #groupCandidates}. One slot per draw and a group cannot exceed the batch capacity, so this
+ * settles after the first few groups rather than growing without limit.
+ */
+ private void ensureGroupCandidateCapacity(long needed) {
+ if (groupCandidates.length < needed) {
+ long grown = Math.max(needed, groupCandidates.length * 2L);
+ groupCandidates = new long[(int) Math.min(Integer.MAX_VALUE, grown)];
}
}
@@ -423,37 +572,32 @@ public class ColumnBtreeSampleCursor extends EnforcedIndexCursor implements ITre
// Phase 2: Sorted column collection — one mega-page load per page
// ──────────────────────────────────────────────────────────────────────
- @AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_5, tool = AiProvenance.Tool.CLAUDE_CODE_CLI, contributionKind = AiProvenance.ContributionKind.ASSISTED, notes = "Release the final mega-leaf's pages when Phase 2 is exhausted instead of holding them "
- + "until doClose()")
private boolean yieldNextFromPhase2() throws HyracksDataException {
if (yieldPos >= collectedCount) {
- /*
- * Phase 2 releases a mega-leaf's pages only when the page id changes, so when the samples run out
- * the last page's column set is still pinned. Release it here rather than holding a full mega-leaf
- * of frames until doClose(). releasePages() is idempotent: context.release/unpinAll and
- * unpinColumnsPages clear their page lists, and unpinCurrentPage0 no-ops on a null page0.
- */
+ // Pages are released on page-id change, so the last mega-leaf is still pinned here. Release it
+ // rather than hold a full mega-leaf of frames until doClose(). releasePages() is idempotent.
releasePages();
return false;
}
- long nanos = traceTimingEnabled ? System.nanoTime() : 0L;
+ long nanos = (traceTimingEnabled || stats != null) ? System.nanoTime() : 0L;
try {
long packed = collectedSamples[yieldPos];
int pageId = (int) (packed >>> 32);
int tupleIdx = (int) packed;
if (pageId != prevYieldPageId) {
- // Release previous column pages & page0
context.release(bufferCache);
unpinCurrentPage0();
- // Pin the new page0
ICachedPage newPage = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), context);
+ if (stats != null) {
+ stats.phase2PagePins++;
+ }
leafFrame.setPage(newPage);
page0 = leafFrame.getPage();
- // Full column load
+ // Full column load -- the point of deferring to phase 2.
context.preparePageZeroSegments(leafFrame, bufferCache, fileId);
frameTuple.newPage();
context.prepareColumns(leafFrame, bufferCache, fileId);
@@ -467,10 +611,12 @@ public class ColumnBtreeSampleCursor extends EnforcedIndexCursor implements ITre
yieldPos++;
return true;
} finally {
- // Track Phase 2 materialization time here
if (traceTimingEnabled) {
totalTimeTakenToFindRandomTuples += (System.nanoTime() - nanos);
}
+ if (stats != null) {
+ stats.phase2Nanos += System.nanoTime() - nanos;
+ }
}
}
@@ -524,8 +670,9 @@ public class ColumnBtreeSampleCursor extends EnforcedIndexCursor implements ITre
yieldPos = 0;
prevYieldPageId = -1;
- // FIX: Force a batch refill on the next reuse
- pendingLeafDrawIndex = leafDrawBatchSize;
+ // Force a batch refill on the next reuse
+ pendingLeafDrawIndex = 0;
+ currentBatchSize = 0;
sampledCount = 0;
hasNextAttemptCount = 0;
@@ -546,13 +693,10 @@ public class ColumnBtreeSampleCursor extends EnforcedIndexCursor implements ITre
}
@Override
- @AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_4_8, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.ASSISTED, notes = "Pass read context on unpin so onUnpin/afterRead releases the read latch acquired by pin (fixes cloud isClean leak)")
public void unpin(ICachedPage page) throws HyracksDataException {
- // Must pass the same read context used in pin(int): pin() goes through
- // context.onPin -> CloudCachedPage.beforeRead (read latch), so unpin must go
- // through context.onUnpin -> afterRead to release that latch. Unpinning without
- // the context skips onUnpin and leaks the read latch (fails BufferCache.isClean
- // under cloud storage).
+ // Same context as pin(int): pin goes through context.onPin -> beforeRead (read latch), so unpin must go
+ // through onUnpin -> afterRead to release it. Without the context the latch leaks and cloud storage
+ // fails BufferCache.isClean.
bufferCache.unpin(page, context);
}
@@ -561,16 +705,4 @@ public class ColumnBtreeSampleCursor extends EnforcedIndexCursor implements ITre
return bufferCache.getPageSize();
}
- // Mutable single-instance view over the SoA draw arrays, repointed per draw.
- private static final class LeafDraw {
- private int pageId;
- private double acceptanceSample;
- private int tupleStartSeed;
-
- private LeafDraw(int pageId, double acceptanceSample, int tupleStartSeed) {
- this.pageId = pageId;
- this.acceptanceSample = acceptanceSample;
- this.tupleStartSeed = tupleStartSeed;
- }
- }
}
\ No newline at end of file
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/SampleCursorStats.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/SampleCursorStats.java
new file mode 100644
index 0000000000..0a046d12ac
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/SampleCursorStats.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.btree.column.impls.btree;
+
+public final class SampleCursorStats {
+
+ public long livenessNanos;
+ public long livenessCalls;
+ public long livenessKeys;
+ public long pkSeekNanos;
+ public long pkSeekCalls;
+ public long phase2Nanos;
+ public long phase2PagePins;
+ public long pagesAccepted;
+ public long pagesRejected;
+ public long attempts;
+ /**
+ * Page groups visited in phase 1 — one per {@code startSamplingPage()} rewind and forward PK pass, i.e. the
+ * count this cursor's cost is actually proportional to. Distinct from {@link #attempts}, which counts
+ * <em>draws</em>: a batch is sorted by pageId, so the draws that landed on one page are consumed as a single
+ * group sharing one pass. {@code attempts / pageGroups} is therefore the amortization factor achieved.
+ */
+ public long pageGroups;
+
+ public void reset() {
+ livenessNanos = 0;
+ livenessCalls = 0;
+ livenessKeys = 0;
+ pkSeekNanos = 0;
+ pkSeekCalls = 0;
+ phase2Nanos = 0;
+ phase2PagePins = 0;
+ pagesAccepted = 0;
+ pagesRejected = 0;
+ attempts = 0;
+ pageGroups = 0;
+ }
+
+ public double livenessSharePct() {
+ long total = livenessNanos + pkSeekNanos + phase2Nanos;
+ return total == 0 ? 0.0 : (100.0 * livenessNanos / total);
+ }
+
+ public String format() {
+ return String.format(
+ "liveness %.1fms (%d calls, %d keys, %.1f%%), pkSeek %.1fms (%d calls), "
+ + "phase2 %.1fms (%d pins), pages %d/%d acc/rej, attempts %d, pageGroups %d",
+ livenessNanos / 1e6, livenessCalls, livenessKeys, livenessSharePct(), pkSeekNanos / 1e6, pkSeekCalls,
+ phase2Nanos / 1e6, phase2PagePins, pagesAccepted, pagesRejected, attempts, pageGroups);
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/LSMColumnBTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/LSMColumnBTree.java
index 5e7423c8e8..1fd803add4 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/LSMColumnBTree.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/LSMColumnBTree.java
@@ -34,7 +34,6 @@ import org.apache.hyracks.storage.am.lsm.btree.column.utils.ColumnUtil;
import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTree;
import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeBatchPointSearchCursor;
import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeRangeSearchCursor;
-import org.apache.hyracks.storage.am.lsm.btree.impls.LSMIndexSampleCursor;
import org.apache.hyracks.storage.am.lsm.common.api.IComponentMetadata;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackFactory;
@@ -143,9 +142,8 @@ public class LSMColumnBTree extends LSMBTree {
return new LSMColumnBatchPointSearchCursor(opCtx);
}
- @Override
- public LSMIndexSampleCursor createSampleCollectorCursor(ILSMIndexOperationContext opContext) {
- return new LSMIndexSampleCursor(opContext, new LSMColumnBatchPointSearchCursor(opContext));
+ public LSMColumnSampleLivenessSearchCursor createSampleLivenessSearchCursor(ILSMIndexOperationContext opCtx) {
+ return new LSMColumnSampleLivenessSearchCursor(opCtx);
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/LSMColumnSampleLivenessSearchCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/LSMColumnSampleLivenessSearchCursor.java
new file mode 100644
index 0000000000..86ff2b7a0c
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/lsm/LSMColumnSampleLivenessSearchCursor.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.btree.column.impls.lsm;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.btree.impls.BTree;
+import org.apache.hyracks.storage.am.btree.impls.BTree.BTreeAccessor;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexCursor;
+import org.apache.hyracks.storage.am.common.impls.NoOpIndexAccessParameters;
+import org.apache.hyracks.storage.am.lsm.btree.column.api.projection.IColumnProjectionInfo;
+import org.apache.hyracks.storage.am.lsm.btree.column.cloud.buffercache.IColumnReadContext;
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.ColumnBTree;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent.LSMComponentType;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
+
+public class LSMColumnSampleLivenessSearchCursor extends LSMColumnBatchPointSearchCursor {
+
+ public LSMColumnSampleLivenessSearchCursor(ILSMIndexOperationContext opCtx) {
+ super(opCtx);
+ }
+
+ @Override
+ protected BTreeAccessor createAccessor(LSMComponentType type, BTree btree, int index) throws HyracksDataException {
+ if (type == LSMComponentType.MEMORY) {
+ // Row-format component: nothing column-specific to reduce. Sampling never reaches this branch (the
+ // sample cursor's search components are all on-disk), but delegating keeps the class total.
+ return super.createAccessor(type, btree, index);
+ }
+ ColumnBTree columnBTree = (ColumnBTree) btree;
+ LSMColumnBTreeOpContext columnOpCtx = (LSMColumnBTreeOpContext) opCtx;
+ IColumnProjectionInfo projectionInfo = columnOpCtx.createProjectionInfo();
+ if (projectionInfo != null) {
+ projectionInfo = projectionInfo.createExistenceOnlyProjectionInfo();
+ }
+ IColumnReadContext context = columnOpCtx.createPageZeroContext(projectionInfo);
+ return columnBTree.createAccessor(NoOpIndexAccessParameters.INSTANCE, index, projectionInfo, context);
+ }
+
+ @Override
+ protected ITreeIndexCursor createCursor(LSMComponentType type, BTreeAccessor btreeAccessor) {
+ if (btreeAccessor instanceof ColumnBTree.ColumnBTreeAccessor) {
+ return ((ColumnBTree.ColumnBTreeAccessor) btreeAccessor).createExistencePointCursor();
+ }
+ // Memory (row-format) component; see createAccessor above.
+ return super.createCursor(type, btreeAccessor);
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/BTreeSampleCollectorOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/BTreeSampleCollectorOperatorDescriptor.java
index f6f7c203f1..2f92dfc6e6 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/BTreeSampleCollectorOperatorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/BTreeSampleCollectorOperatorDescriptor.java
@@ -39,7 +39,6 @@ public class BTreeSampleCollectorOperatorDescriptor extends BTreeSearchOperatorD
private final long sampleSeed;
private final int maxSampleLeafAttempts;
private final int sampleLeafDrawBatchSize;
- private final int columnSamplesPerPage;
public BTreeSampleCollectorOperatorDescriptor(IOperatorDescriptorRegistry spec, RecordDescriptor outRecDesc,
int[] lowKeyFields, int[] highKeyFields, boolean lowKeyInclusive, boolean highKeyInclusive,
@@ -51,7 +50,7 @@ public class BTreeSampleCollectorOperatorDescriptor extends BTreeSearchOperatorD
byte[] searchCallbackProceedResultTrueValue, ITupleProjectorFactory tupleProjectorFactory,
ITuplePartitionerFactory tuplePartitionerFactory, int[][] partitionsMap,
int sampleCardinalityTargetPerPartition, long sampleSeed, int maxSampleLeafAttempts,
- int sampleLeafDrawBatchSize, int columnSamplesPerPage) {
+ int sampleLeafDrawBatchSize) {
super(spec, outRecDesc, lowKeyFields, highKeyFields, lowKeyInclusive, highKeyInclusive, indexHelperFactory,
retainInput, retainMissing, missingWriterFactory, searchCallbackFactory, minFilterFieldIndexes,
maxFilterFieldIndexes, appendIndexFilter, nonFilterWriterFactory, tupleFilterFactory, outputLimit,
@@ -61,7 +60,6 @@ public class BTreeSampleCollectorOperatorDescriptor extends BTreeSearchOperatorD
this.sampleSeed = sampleSeed;
this.maxSampleLeafAttempts = maxSampleLeafAttempts;
this.sampleLeafDrawBatchSize = sampleLeafDrawBatchSize;
- this.columnSamplesPerPage = columnSamplesPerPage;
}
@Override
@@ -72,7 +70,6 @@ public class BTreeSampleCollectorOperatorDescriptor extends BTreeSearchOperatorD
lowKeyInclusive, highKeyInclusive, minFilterFieldIndexes, maxFilterFieldIndexes, indexHelperFactory,
retainInput, retainMissing, missingWriterFactory, searchCallbackFactory, tupleFilterFactory,
outputLimit, tupleProjectorFactory, tuplePartitionerFactory, partitionsMap,
- sampleCardinalityTargetPerPartition, sampleSeed, maxSampleLeafAttempts, sampleLeafDrawBatchSize,
- columnSamplesPerPage);
+ sampleCardinalityTargetPerPartition, sampleSeed, maxSampleLeafAttempts, sampleLeafDrawBatchSize);
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/BTreeSampleCollectorOperatorDescriptorNodePushable.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/BTreeSampleCollectorOperatorDescriptorNodePushable.java
index 2f7d3e6fa6..ab2a8d3a7b 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/BTreeSampleCollectorOperatorDescriptorNodePushable.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/BTreeSampleCollectorOperatorDescriptorNodePushable.java
@@ -47,7 +47,6 @@ public class BTreeSampleCollectorOperatorDescriptorNodePushable extends BTreeSea
private final long sampleSeed;
private final int maxSampleLeafAttempts;
private final int sampleLeafDrawBatchSize;
- private final int columnSamplesPerPage;
private final IHyracksTaskContext ctx;
public BTreeSampleCollectorOperatorDescriptorNodePushable(IHyracksTaskContext ctx, int partition,
@@ -58,7 +57,7 @@ public class BTreeSampleCollectorOperatorDescriptorNodePushable extends BTreeSea
ITupleFilterFactory tupleFilterFactory, long outputLimit, ITupleProjectorFactory tupleProjectorFactory,
ITuplePartitionerFactory tuplePartitionerFactory, int[][] partitionsMap,
int sampleCardinalityTargetPerPartition, long sampleSeed, int maxSampleLeafAttempts,
- int sampleLeafDrawBatchSize, int columnSamplesPerPage) throws HyracksDataException {
+ int sampleLeafDrawBatchSize) throws HyracksDataException {
super(ctx, partition, inputRecDesc, lowKeyFields, highKeyFields, lowKeyInclusive, highKeyInclusive,
minFilterKeyFields, maxFilterKeyFields, indexHelperFactory, retainInput, retainMissing,
missingWriterFactory, searchCallbackFactory, false, null, tupleFilterFactory, outputLimit, false, null,
@@ -67,7 +66,6 @@ public class BTreeSampleCollectorOperatorDescriptorNodePushable extends BTreeSea
this.sampleSeed = sampleSeed;
this.maxSampleLeafAttempts = maxSampleLeafAttempts;
this.sampleLeafDrawBatchSize = sampleLeafDrawBatchSize;
- this.columnSamplesPerPage = columnSamplesPerPage;
this.ctx = ctx;
}
@@ -78,7 +76,6 @@ public class BTreeSampleCollectorOperatorDescriptorNodePushable extends BTreeSea
iap.getParameters().put(LSMIndexSampleCursor.SAMPLE_SEED, sampleSeed);
iap.getParameters().put(LSMIndexSampleCursor.SAMPLE_MAX_LEAF_ATTEMPTS, maxSampleLeafAttempts);
iap.getParameters().put(LSMIndexSampleCursor.SAMPLE_LEAF_DRAW_BATCH_SIZE, sampleLeafDrawBatchSize);
- iap.getParameters().put(LSMIndexSampleCursor.SAMPLE_COLUMN_SAMPLES_PER_PAGE, columnSamplesPerPage);
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
index 492e3057ff..74ea5bdd3b 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
@@ -250,7 +250,11 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
}
public LSMIndexSampleCursor createSampleCollectorCursor(ILSMIndexOperationContext opContext) {
- return new LSMIndexSampleCursor(opContext);
+ return new LSMIndexSampleCursor(opContext, createSampleLivenessSearchCursor(opContext));
+ }
+
+ public LSMBTreeBatchPointSearchCursor createSampleLivenessSearchCursor(ILSMIndexOperationContext opCtx) {
+ return new LSMBTreeBatchPointSearchCursor(opCtx);
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMIndexSampleCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMIndexSampleCursor.java
index 3fab1f60a0..fa3ff69d11 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMIndexSampleCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMIndexSampleCursor.java
@@ -53,7 +53,6 @@ public class LSMIndexSampleCursor extends EnforcedIndexCursor implements ILSMInd
public static final String SAMPLE_SEED = "SAMPLE_SEED";
public static final String SAMPLE_MAX_LEAF_ATTEMPTS = "SAMPLE_MAX_LEAF_ATTEMPTS";
public static final String SAMPLE_LEAF_DRAW_BATCH_SIZE = "SAMPLE_LEAF_DRAW_BATCH_SIZE";
- public static final String SAMPLE_COLUMN_SAMPLES_PER_PAGE = "SAMPLE_COLUMN_SAMPLES_PER_PAGE";
private static final Logger LOGGER = LogManager.getLogger();
// The theta sketch may have been written across multiple metadata pages; read it back via the chunking reader.
private static final ChunkedComponentMetadataReaderWriter THETA_SKETCH_RW =
@@ -77,7 +76,6 @@ public class LSMIndexSampleCursor extends EnforcedIndexCursor implements ILSMInd
private final long sampleSeed;
private int maxSampleLeafAttempts;
private int sampleLeafDrawBatchSize;
- private int columnSamplesPerPage;
// Number of LIVE tuples sampled so far.
private int sampledCount;
private int currentDiskComponentIndex;
@@ -108,7 +106,6 @@ public class LSMIndexSampleCursor extends EnforcedIndexCursor implements ILSMInd
// Read sampling knobs passed via the access parameters (sourced from StorageProperties at job build time).
maxSampleLeafAttempts = (int) opCtx.getParameters().get(SAMPLE_MAX_LEAF_ATTEMPTS);
sampleLeafDrawBatchSize = (int) opCtx.getParameters().get(SAMPLE_LEAF_DRAW_BATCH_SIZE);
- columnSamplesPerPage = (int) opCtx.getParameters().get(SAMPLE_COLUMN_SAMPLES_PER_PAGE);
// Reset state from any previous scan
sampledCount = 0;
@@ -164,7 +161,7 @@ public class LSMIndexSampleCursor extends EnforcedIndexCursor implements ILSMInd
private ITreeIndexCursor createCursor(DiskBTree.DiskBTreeAccessor bTreeAccessor, long componentSampleCardinality,
long sampleSeed, ILSMIndexBatchPointCursor searchCursor, int maxLeafTupleCount) {
return bTreeAccessor.createSampleCursor(componentSampleCardinality, sampleSeed, searchCursor,
- maxSampleLeafAttempts, sampleLeafDrawBatchSize, maxLeafTupleCount, columnSamplesPerPage,
+ maxSampleLeafAttempts, sampleLeafDrawBatchSize, maxLeafTupleCount,
AntimatterAwareTupleAcceptor.INSTANCE);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/pom.xml b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/pom.xml
index 788ccf3fe5..a7f6107b97 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/pom.xml
@@ -71,6 +71,12 @@
<version>${project.version}</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-storage-am-lsm-btree-column</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>org.apache.hyracks</groupId>
<artifactId>hyracks-storage-am-common</artifactId>
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/DiskBTreeSampleUniformityTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/DiskBTreeSampleUniformityTest.java
new file mode 100644
index 0000000000..e37e794885
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/DiskBTreeSampleUniformityTest.java
@@ -0,0 +1,354 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.lsm.btree;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import org.apache.hyracks.api.dataflow.value.ITypeTraits;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.accessors.IntegerBinaryComparatorFactory;
+import org.apache.hyracks.data.std.primitive.IntegerPointable;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleReference;
+import org.apache.hyracks.dataflow.common.utils.TupleUtils;
+import org.apache.hyracks.storage.am.btree.impls.BTreeOpContext;
+import org.apache.hyracks.storage.am.btree.impls.DiskBTree;
+import org.apache.hyracks.storage.am.btree.impls.RangePredicate;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexCursor;
+import org.apache.hyracks.storage.am.common.impls.NoOpIndexAccessParameters;
+import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
+import org.apache.hyracks.storage.am.lsm.btree.impls.AntimatterAwareTupleAcceptor;
+import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTree;
+import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeBatchPointSearchCursor;
+import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeOpContext;
+import org.apache.hyracks.storage.am.lsm.btree.util.LSMBTreeTestHarness;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
+import org.apache.hyracks.storage.am.lsm.common.impls.DiskComponentMetadata;
+import org.apache.hyracks.storage.common.buffercache.context.read.DefaultBufferCacheReadContextProvider;
+import org.apache.hyracks.util.annotations.AiProvenance;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Uniformity guard for {@code DiskBTreeSampleCursor}, the <b>row</b>-format sample cursor; {@code
+ * ColumnSampleUniformityTest} is the column analogue. It exists because the row cursor took the same uniformity
+ * fixes (full-batch consumption, {@code Random.nextInt(bound)} over a {@code %} fold) while its correctness
+ * rested on an analytical argument and functional tests that only checked "no deleted row leaked".
+ * <p>
+ * The sample must be uniform over the component's <em>live</em> tuples, because the CBO derives NDV and
+ * selectivity from it; clustered in primary-key order it is silently wrong for every column correlated with the
+ * key. The pre-fix failure was not subtle — the cursor drained the lowest page ids of a pageId-sorted batch and
+ * stopped at the target, making the "sample" a contiguous run of the key space.
+ * <p>
+ * Chi-square goodness-of-fit on {@link #NUM_SEEDS} pooled runs over {@link #NUM_BINS} bins, at the unscaled
+ * {@link #CHI_SQUARE_CRITICAL} (19 df, alpha = 0.001). Two deliberate details:
+ * <ul>
+ * <li><b>Per-bin expectations are proportional to each bin's live keys</b>, not a flat {@code total / NUM_BINS}.
+ * They agree exactly at this fixture; the proportional form guards against a future fixture whose uneven bins
+ * would otherwise inflate the statistic for reasons unrelated to the sampler.</li>
+ * <li><b>The sampling fraction is appreciable</b> and the cursor samples without replacement, so cell variance
+ * sits below multinomial by {@code 1 - f}. That makes the test conservative under the null and costs nothing
+ * against the alternative it catches, which is off by orders of magnitude, not a variance factor.</li>
+ * </ul>
+ * <p>
+ * <b>Sizing.</b> {@link #NUM_KEYS} keys flushed in rounds then merged — so the shared harness's
+ * 50-page mutable components do not bound the result — give 118 leaf pages at 256-byte disk pages, roughly six
+ * per bin. A handful of pages could not distinguish clustering from noise, since one page would span several
+ * bins.
+ * <p>
+ * <b>Shown non-vacuous:</b> reinstating the pre-fix mid-batch stop moves the pooled statistic from 13.98 to
+ * <b>42300.00</b>, with all 7200 samples in the lowest 3 bins. Recorded rather than automated — reproducing it
+ * means editing production code.
+ */
+@AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_4_8, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.TEST_GENERATED, notes = "Chi-square uniformity guard over the live key space for the row sample cursor")
+@SuppressWarnings("rawtypes")
+public class DiskBTreeSampleUniformityTest {
+
+ private static final int FIELD_COUNT = 2;
+ private static final int KEY_FIELD_COUNT = 1;
+ private static final ITypeTraits[] TYPE_TRAITS = { IntegerPointable.TYPE_TRAITS, IntegerPointable.TYPE_TRAITS };
+ private static final IBinaryComparatorFactory[] CMP_FACTORIES = { IntegerBinaryComparatorFactory.INSTANCE };
+ private static final int[] BLOOM_FILTER_KEY_FIELDS = { 0 };
+
+ /** Keys in the sampled (oldest) component. Sized for leaf pages, see the class javadoc. */
+ private static final int NUM_KEYS = 2000;
+ /**
+ * Each round must fit one mutable component (50 pages of 256 B in the shared harness). The rounds are merged
+ * afterwards, so this bounds memory, not the component's size.
+ */
+ private static final int KEYS_PER_ROUND = 500;
+ /** Percentage of the key space shadowed by newer components; mirrors the column fixture. */
+ private static final int SHADOW_PCT = 55;
+ private static final int NUM_SHADOW_COMPONENTS = 3;
+ private static final int NUM_BINS = 20;
+ private static final int NUM_SEEDS = 60;
+ private static final int TARGET = 120;
+ /** Production default of {@code STORAGE_SAMPLE_LEAF_DRAW_BATCH_SIZE} — far above the target, as in production. */
+ private static final int LEAF_DRAW_BATCH_SIZE = 32768;
+ /** Production default of {@code STORAGE_MAX_SAMPLE_LEAF_ATTEMPTS}; the cursor scales it up itself. */
+ private static final int MAX_LEAF_ATTEMPTS = 500;
+ /** 19 df, alpha = 0.001, unscaled. Deliberately loose: guards gross bias, not small-sample noise. */
+ private static final double CHI_SQUARE_CRITICAL = 43.82;
+ /**
+ * Too few leaf pages and clustering is indistinguishable from noise, one page spanning several bins.
+ * Measured at 118; set below that so page-packing changes do not fail spuriously, but far above the ~{@link
+ * #NUM_BINS} pages at which the test goes blind.
+ */
+ private static final int EXPECTED_MIN_LEAF_PAGES = 80;
+
+ private final LSMBTreeTestHarness harness = new LSMBTreeTestHarness();
+ private LSMBTree lsmBtree;
+
+ @Before
+ public void setUp() throws HyracksDataException {
+ harness.setUp();
+ lsmBtree = LSMBTreeExamplesTest.createTreeIndex(harness, TYPE_TRAITS, CMP_FACTORIES, BLOOM_FILTER_KEY_FIELDS,
+ null, null, null, null);
+ lsmBtree.create();
+ lsmBtree.activate();
+ buildComponents();
+ }
+
+ @After
+ public void tearDown() throws HyracksDataException {
+ try {
+ lsmBtree.deactivate();
+ lsmBtree.destroy();
+ } finally {
+ harness.tearDown();
+ }
+ }
+
+ /**
+ * Builds the sampled component (all {@link #NUM_KEYS} keys, flushed in rounds then merged into one disk
+ * component) followed by {@link #NUM_SHADOW_COMPONENTS} newer components that delete-and-reinsert the first
+ * {@code NUM_KEYS * SHADOW_PCT / 100} keys. Those keys are therefore present in a newer component and must not
+ * be emitted when the oldest component is sampled — exactly the shape the column fixture uses.
+ */
+ private void buildComponents() throws HyracksDataException {
+ ILSMIndexAccessor accessor = (ILSMIndexAccessor) lsmBtree.createAccessor(NoOpIndexAccessParameters.INSTANCE);
+ for (int from = 0; from < NUM_KEYS; from += KEYS_PER_ROUND) {
+ int to = Math.min(NUM_KEYS, from + KEYS_PER_ROUND);
+ for (int key = from; key < to; key++) {
+ insert(accessor, key);
+ }
+ accessor.scheduleFlush();
+ }
+ // Merge the rounds into ONE disk component: that component is what gets sampled, and its leaf-page count is
+ // what gives the test the resolution to see clustering. The merge also bulk-loads it, which is what records
+ // MAX_LEAF_TUPLE_COUNT_KEY, so the Olken fill-rejection denominator below is a real value.
+ List<ILSMDiskComponent> rounds = new ArrayList<>(lsmBtree.getDiskComponents());
+ Assert.assertTrue("expected several flushed rounds to merge", rounds.size() > 1);
+ accessor.scheduleMerge(rounds);
+ Assert.assertEquals("the sampled component must be a single merged component", 1,
+ lsmBtree.getDiskComponents().size());
+
+ int shadowBoundary = getShadowBoundary();
+ int perComponent = (shadowBoundary + NUM_SHADOW_COMPONENTS - 1) / NUM_SHADOW_COMPONENTS;
+ for (int component = 0; component < NUM_SHADOW_COMPONENTS; component++) {
+ int from = component * perComponent;
+ int to = Math.min(shadowBoundary, from + perComponent);
+ for (int key = from; key < to; key++) {
+ delete(accessor, key);
+ insert(accessor, key);
+ }
+ accessor.scheduleFlush();
+ }
+ Assert.assertEquals("expected one sampled component plus the shadow components", 1 + NUM_SHADOW_COMPONENTS,
+ lsmBtree.getDiskComponents().size());
+ }
+
+ /** Keys in {@code [0, shadowBoundary)} are shadowed by a newer component; keys at or above it stay live. */
+ private static int getShadowBoundary() {
+ return NUM_KEYS * SHADOW_PCT / 100;
+ }
+
+ @Test
+ public void sampleIsUniformOverLiveKeys() throws Exception {
+ int shadowBoundary = getShadowBoundary();
+ int liveCount = NUM_KEYS - shadowBoundary;
+ int leafPages = getSampledComponentLeafPageCount();
+ Assert.assertTrue("a component with " + leafPages + " leaf pages is too coarse to detect spatial clustering",
+ leafPages >= EXPECTED_MIN_LEAF_PAGES);
+
+ long[] binCounts = new long[NUM_BINS];
+ long total = 0;
+ int minEmitted = Integer.MAX_VALUE;
+ int maxEmitted = 0;
+ for (int seed = 0; seed < NUM_SEEDS; seed++) {
+ int[] keys = runSample(TARGET, seed);
+ minEmitted = Math.min(minEmitted, keys.length);
+ maxEmitted = Math.max(maxEmitted, keys.length);
+ for (int key : keys) {
+ Assert.assertTrue("shadowed key leaked into the sample: " + key, key >= shadowBoundary);
+ Assert.assertTrue("out-of-range key in the sample: " + key, key < NUM_KEYS);
+ binCounts[binOf(key, shadowBoundary, liveCount)]++;
+ total++;
+ }
+ }
+ Assert.assertTrue("no samples collected; cannot assess uniformity", total > 0);
+
+ double chiSquare = 0.0;
+ for (int bin = 0; bin < NUM_BINS; bin++) {
+ // Expectation proportional to the live keys the bin actually spans (see the class javadoc): the live
+ // population is not divisible by NUM_BINS, so equal expectations would themselves inflate the statistic.
+ double expected = (double) total * liveKeysInBin(bin, liveCount) / liveCount;
+ double delta = binCounts[bin] - expected;
+ chiSquare += delta * delta / expected;
+ }
+
+ System.out.println("=".repeat(96));
+ System.out.printf("row sample uniformity: keys=%d liveKeys=%d leafPages=%d target=%d seeds=%d%n", NUM_KEYS,
+ liveCount, leafPages, TARGET, NUM_SEEDS);
+ System.out.printf(" emitted per run: min=%d max=%d (requested %d)%n", minEmitted, maxEmitted, TARGET);
+ System.out.printf(" pooled=%d chiSquare=%.2f (critical %.2f, %d df)%n", total, chiSquare, CHI_SQUARE_CRITICAL,
+ NUM_BINS - 1);
+ for (int bin = 0; bin < NUM_BINS; bin++) {
+ int from = shadowBoundary + (int) ((long) bin * liveCount / NUM_BINS);
+ int to = shadowBoundary + (int) ((long) (bin + 1) * liveCount / NUM_BINS);
+ double expected = (double) total * liveKeysInBin(bin, liveCount) / liveCount;
+ int barLength = (int) Math.round(40 * binCounts[bin] / expected);
+ System.out.printf(" bin %2d [%5d,%5d) %7d %s%n", bin, from, to, binCounts[bin],
+ "#".repeat(Math.min(barLength, 200)));
+ }
+ System.out.println("=".repeat(96));
+
+ Assert.assertEquals("every run should reach the requested sample size", TARGET, minEmitted);
+ Assert.assertTrue("sample distribution is not uniform over live keys: chiSquare=" + chiSquare
+ + " exceeds critical " + CHI_SQUARE_CRITICAL, chiSquare < CHI_SQUARE_CRITICAL);
+ }
+
+ /** Bin index of a live key, using the same equal-width partition of the live range as {@link #liveKeysInBin}. */
+ private static int binOf(int key, int shadowBoundary, int liveCount) {
+ int bin = (int) ((long) (key - shadowBoundary) * NUM_BINS / liveCount);
+ return Math.min(bin, NUM_BINS - 1);
+ }
+
+ /** Number of live keys the given bin spans. Bins differ by one key when {@code liveCount % NUM_BINS != 0}. */
+ private static int liveKeysInBin(int bin, int liveCount) {
+ return (int) ((long) (bin + 1) * liveCount / NUM_BINS) - (int) ((long) bin * liveCount / NUM_BINS);
+ }
+
+ /**
+ * Drives the real row sample cursor over the <b>oldest</b> disk component with the liveness search pointed at all
+ * newer components — the wiring {@code LSMIndexSampleCursor} uses for the last component.
+ *
+ * @return the primary keys emitted, in emission order
+ */
+ private int[] runSample(int targetCardinality, long seed) throws HyracksDataException {
+ LSMBTreeOpContext opCtx = lsmBtree.createOpContext(NoOpIndexAccessParameters.INSTANCE);
+ opCtx.setOperation(IndexOperation.SEARCH);
+ lsmBtree.getOperationalComponents(opCtx);
+ List<ILSMComponent> components = opCtx.getComponentHolder(); // newest first
+ int oldestIndex = components.size() - 1;
+ ILSMComponent oldest = components.get(oldestIndex);
+ List<ILSMComponent> newer = new ArrayList<>(components.subList(0, oldestIndex));
+
+ RangePredicate searchPred = new RangePredicate(null, null, true, true, null, null);
+ opCtx.getSearchInitialState().reset(searchPred, newer);
+ LSMBTreeBatchPointSearchCursor searchCursor = new LSMBTreeBatchPointSearchCursor(opCtx);
+ searchCursor.open(opCtx.getSearchInitialState(), searchPred);
+
+ DiskBTree oldestBtree = (DiskBTree) oldest.getIndex();
+ DiskBTree.DiskBTreeAccessor sampleAccessor =
+ (DiskBTree.DiskBTreeAccessor) oldestBtree.createAccessor(NoOpIndexAccessParameters.INSTANCE);
+ ITreeIndexCursor sampleCursor =
+ sampleAccessor.createSampleCursor(targetCardinality, seed, searchCursor, MAX_LEAF_ATTEMPTS,
+ LEAF_DRAW_BATCH_SIZE, maxLeafTupleCountOf(oldest), AntimatterAwareTupleAcceptor.INSTANCE);
+
+ int[] keys = new int[Math.max(1, targetCardinality)];
+ int emitted = 0;
+ try {
+ sampleAccessor.diskSampleScan(sampleCursor);
+ while (sampleCursor.hasNext()) {
+ sampleCursor.next();
+ keys[emitted++] = IntegerPointable.getInteger(sampleCursor.getTuple().getFieldData(0),
+ sampleCursor.getTuple().getFieldStart(0));
+ }
+ } finally {
+ sampleCursor.close();
+ sampleCursor.destroy();
+ searchCursor.close();
+ searchCursor.destroy();
+ }
+ int[] result = new int[emitted];
+ System.arraycopy(keys, 0, result, 0, emitted);
+ return result;
+ }
+
+ /**
+ * Leaf-page count of the oldest (sampled) component — the population the cursor draws uniformly over, and hence
+ * the resolution at which this test can see clustering. Read via the same enumeration the cursor itself uses.
+ */
+ private int getSampledComponentLeafPageCount() throws HyracksDataException {
+ LSMBTreeOpContext opCtx = lsmBtree.createOpContext(NoOpIndexAccessParameters.INSTANCE);
+ opCtx.setOperation(IndexOperation.SEARCH);
+ lsmBtree.getOperationalComponents(opCtx);
+ List<ILSMComponent> components = opCtx.getComponentHolder();
+ DiskBTree oldestBtree = (DiskBTree) components.get(components.size() - 1).getIndex();
+ DiskBTree.DiskBTreeAccessor accessor =
+ (DiskBTree.DiskBTreeAccessor) oldestBtree.createAccessor(NoOpIndexAccessParameters.INSTANCE);
+ try {
+ BTreeOpContext btreeOpCtx = accessor.getOpContext();
+ btreeOpCtx.reset();
+ return oldestBtree.enumerateLeafPageIdsCompact(oldestBtree.getRootPageId(), btreeOpCtx,
+ DefaultBufferCacheReadContextProvider.DEFAULT).size();
+ } finally {
+ accessor.destroy();
+ }
+ }
+
+ /**
+ * The component's recorded max leaf tuple count — the denominator of the cursor's Olken & Rotem
+ * fill-rejection probability, read exactly as {@code LSMIndexSampleCursor} reads it. Passing the real value
+ * matters here: with fill-rejection disabled the partially-filled last leaf page would have its tuples
+ * over-weighted, which this test would (correctly) report as non-uniformity of the fixture rather than of the
+ * cursor.
+ */
+ private static int maxLeafTupleCountOf(ILSMComponent component) throws HyracksDataException {
+ DiskComponentMetadata metadata = (DiskComponentMetadata) component.getMetadata();
+ ArrayBackedValueStorage ref = new ArrayBackedValueStorage();
+ if (metadata.get(DiskComponentMetadata.MAX_LEAF_TUPLE_COUNT_KEY, ref) && ref.getLength() >= Integer.BYTES) {
+ return IntegerPointable.getInteger(ref.getByteArray(), ref.getStartOffset());
+ }
+ return 0;
+ }
+
+ private void insert(ILSMIndexAccessor accessor, int key) throws HyracksDataException {
+ ArrayTupleBuilder tb = new ArrayTupleBuilder(FIELD_COUNT);
+ ArrayTupleReference tuple = new ArrayTupleReference();
+ TupleUtils.createIntegerTuple(tb, tuple, key, key);
+ accessor.insert(tuple);
+ }
+
+ private void delete(ILSMIndexAccessor accessor, int key) throws HyracksDataException {
+ ArrayTupleBuilder tb = new ArrayTupleBuilder(KEY_FIELD_COUNT);
+ ArrayTupleReference tuple = new ArrayTupleReference();
+ TupleUtils.createIntegerTuple(tb, tuple, key);
+ accessor.delete(tuple);
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMSampleFragmentationContaminationTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMSampleFragmentationContaminationTest.java
index 3acf24144a..3f356fc8c4 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMSampleFragmentationContaminationTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMSampleFragmentationContaminationTest.java
@@ -144,9 +144,9 @@ public class LSMSampleFragmentationContaminationTest {
(DiskBTree.DiskBTreeAccessor) olderBtree.createAccessor(NoOpIndexAccessParameters.INSTANCE);
// Aim to collect NUM_LIVE samples; maxLeafTupleCount=0 disables fill-rejection (not under test here).
- ITreeIndexCursor sampleCursor = sampleAccessor.createSampleCursor(NUM_LIVE, SAMPLE_SEED, searchCursor,
- /*maxLeafAttempts*/ 50000, /*leafDrawBatchSize*/ NUM_KEYS, /*maxLeafTupleCount*/ 0,
- /*samplesPerPage*/ 0, AntimatterAwareTupleAcceptor.INSTANCE);
+ ITreeIndexCursor sampleCursor =
+ sampleAccessor.createSampleCursor(NUM_LIVE, SAMPLE_SEED, searchCursor, /*maxLeafAttempts*/ 50000,
+ /*leafDrawBatchSize*/ NUM_KEYS, /*maxLeafTupleCount*/ 0, AntimatterAwareTupleAcceptor.INSTANCE);
int emitted = 0;
int leakedDeleted = 0;
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/SampleCursorStatsTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/SampleCursorStatsTest.java
new file mode 100644
index 0000000000..d8fb5d2503
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/SampleCursorStatsTest.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.btree;
+
+import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.SampleCursorStats;
+import org.apache.hyracks.util.annotations.AiProvenance;
+import org.junit.Assert;
+import org.junit.Test;
+
+@AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_4_8, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.TEST_GENERATED, notes = "Unit tests for SampleCursorStats' own accounting arithmetic (share computation, zero-denominator, reset)")
+public class SampleCursorStatsTest {
+
+ @Test
+ public void livenessShareIsComputedFromThePhaseTimings() {
+ SampleCursorStats stats = new SampleCursorStats();
+ stats.livenessNanos = 800;
+ stats.pkSeekNanos = 100;
+ stats.phase2Nanos = 100;
+ Assert.assertEquals(80.0, stats.livenessSharePct(), 0.0001);
+ }
+
+ @Test
+ public void livenessShareIsZeroWhenNothingWasMeasured() {
+ Assert.assertEquals(0.0, new SampleCursorStats().livenessSharePct(), 0.0001);
+ }
+
+ @Test
+ public void resetClearsEveryCounter() {
+ SampleCursorStats stats = new SampleCursorStats();
+ stats.livenessNanos = 5;
+ stats.livenessCalls = 5;
+ stats.livenessKeys = 5;
+ stats.pkSeekNanos = 5;
+ stats.pkSeekCalls = 5;
+ stats.phase2Nanos = 5;
+ stats.phase2PagePins = 5;
+ stats.pagesAccepted = 5;
+ stats.pagesRejected = 5;
+ stats.attempts = 5;
+ stats.reset();
+ Assert.assertEquals(0, stats.livenessNanos);
+ Assert.assertEquals(0, stats.livenessCalls);
+ Assert.assertEquals(0, stats.livenessKeys);
+ Assert.assertEquals(0, stats.pkSeekNanos);
+ Assert.assertEquals(0, stats.pkSeekCalls);
+ Assert.assertEquals(0, stats.phase2Nanos);
+ Assert.assertEquals(0, stats.phase2PagePins);
+ Assert.assertEquals(0, stats.pagesAccepted);
+ Assert.assertEquals(0, stats.pagesRejected);
+ Assert.assertEquals(0, stats.attempts);
+ Assert.assertEquals(0.0, stats.livenessSharePct(), 0.0001);
+ }
+}