You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@asterixdb.apache.org by AsterixDB Code Review <do...@asterix-gerrit.ics.uci.edu> on 2026/09/04 13:59:15 UTC
[XL] Change in asterixdb[master]: [NO ISSUE][RT][STO] Instrument ANALYZE sampling phases and page reads
From Ritik Raj <ri...@couchbase.com>:
Ritik Raj has uploaded this change for review. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21688?usp=email )
Change subject: [NO ISSUE][RT][STO] Instrument ANALYZE sampling phases and page reads
......................................................................
[NO ISSUE][RT][STO] Instrument ANALYZE sampling phases and page reads
ANALYZE surfaced no buffer-cache figures: it is DDL, so it never reaches
updateJobStats. The sampling job already collected them and nothing
reported them, leaving elapsed time as the only signal.
- Report pinned pages, cold reads and hit ratio on the ANALYZE response.
- Add liveness, PK-seek and materialize counters to IOperatorStats,
logged as a per-phase DEBUG breakdown.
- Move SampleCursorStats to hyracks-storage-am-common and add
ISampleCursorStatsReceiver so the row cursor can be instrumented.
- Gate the breakdown on LOGGER.isDebugEnabled(), not a system property
read once at class initialization.
- Harness knobs for leaf size, tuple count and cache budget, with
pin-cost and residency tests.
- Condense the sampling comments from the parent commit.
Ext-ref: MB-73172
Co-Authored-By: Claude Opus 5 <no...@anthropic.com>
Change-Id: Ica90673813691c728d5c96b699b0363679af3813
---
M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
M asterixdb/asterix-app/src/test/resources/runtimets/metrics.xml
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/metrics/analyze-buffer-cache/analyze-buffer-cache.1.ddl.sqlpp
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/metrics/analyze-buffer-cache/analyze-buffer-cache.2.update.sqlpp
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/metrics/analyze-buffer-cache/analyze-buffer-cache.3.metrics.sqlpp
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/metrics/analyze-buffer-cache/analyze-buffer-cache.4.ddl.sqlpp
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/metrics/spike-secondary-io/spike-secondary-io.1.ddl.sqlpp
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/metrics/spike-secondary-io/spike-secondary-io.2.update.sqlpp
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/metrics/spike-secondary-io/spike-secondary-io.3.update.sqlpp
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/metrics/spike-secondary-io/spike-secondary-io.4.ddl.sqlpp
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/metrics/spike-secondary-io/spike-secondary-io.5.metrics.sqlpp
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/metrics/spike-secondary-io/spike-secondary-io.6.metrics.sqlpp
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/metrics/spike-secondary-io/spike-secondary-io.7.metrics.sqlpp
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/metrics/spike-secondary-io/spike-secondary-io.8.metrics.sqlpp
A asterixdb/asterix-app/src/test/resources/runtimets/results/metrics/analyze-buffer-cache/analyze-buffer-cache.3.regexadm
A asterixdb/asterix-app/src/test/resources/runtimets/results/metrics/spike-secondary-io/spike-secondary-io.5.regexadm
A asterixdb/asterix-app/src/test/resources/runtimets/results/metrics/spike-secondary-io/spike-secondary-io.6.regexadm
A asterixdb/asterix-app/src/test/resources/runtimets/results/metrics/spike-secondary-io/spike-secondary-io.7.regexadm
A asterixdb/asterix-app/src/test/resources/runtimets/results/metrics/spike-secondary-io/spike-secondary-io.8.regexadm
M asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/sample/ColumnSampleBenchHarness.java
M asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/sample/ColumnSampleBenchHarnessResidencyTest.java
M asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/sample/ColumnSampleBenchHarnessTest.java
M asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/sample/ColumnSampleCostBaselineTest.java
A asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/sample/ColumnSamplePinCostTest.java
A asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/sample/ColumnSampleResidencyBenchTest.java
M asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/sample/ColumnSampleUniformityTest.java
M asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/DatasetStreamStatsOperatorDescriptor.java
M hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/profiling/IOperatorStats.java
M hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/profiling/NoOpOperatorStats.java
M hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/profiling/OperatorStats.java
M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/DiskBTreeSampleCursor.java
A hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ISampleCursorStatsReceiver.java
R hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/SampleCursorStats.java
M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeExistencePointSearchCursor.java
M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBtreeSampleCursor.java
M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/BTreeSampleCollectorOperatorDescriptorNodePushable.java
M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMIndexSampleCursor.java
M hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/pom.xml
M hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/SampleCursorStatsTest.java
39 files changed, 1,051 insertions(+), 311 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/88/21688/1
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index a7a42b5..b077e72 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -317,6 +317,7 @@
import org.apache.hyracks.storage.am.lsm.invertedindex.fulltext.TokenizerCategory;
import org.apache.hyracks.util.LogRedactionUtil;
import org.apache.hyracks.util.OptionalBoolean;
+import org.apache.hyracks.util.annotations.AiProvenance;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -5534,8 +5535,11 @@
DatasetUtil.buildSampleCardinalityProbeJobSpec(ds, metadataProvider);
MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
bActiveTxn = false;
+ long probeStartNanos = System.nanoTime();
List<IOperatorStats> probeStats = runJob(hcc, cardinalityProbeSpec, jobFlags,
Collections.singletonList(DatasetUtil.SAMPLE_CARDINALITY_PROBE_OPERATOR_NAME));
+ logAnalyzePhase("cardinality-probe", datasetName, System.nanoTime() - probeStartNanos,
+ probeStats == null || probeStats.isEmpty() ? null : probeStats.get(0));
boolean noStats = probeStats == null || probeStats.isEmpty();
// No stats => could not determine; be conservative and force a full scan.
long componentsMissingMetadata = noStats ? -1 : probeStats.get(0).getTupleCounter().get();
@@ -5578,12 +5582,17 @@
MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
bActiveTxn = false;
+ long sampleStartNanos = System.nanoTime();
List<IOperatorStats> opStats = runJob(hcc, spec, jobFlags,
Collections.singletonList(SampleOperationsHelper.DATASET_STATS_OPERATOR_NAME));
+ long sampleElapsedNanos = System.nanoTime() - sampleStartNanos;
if (opStats == null || opStats.size() == 0) {
throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_STATE, "", sourceLoc);
}
DatasetStreamStats stats = new DatasetStreamStats(opStats.get(0));
+ reportSamplingBufferCacheStats(requestParameters, stats);
+ logAnalyzePhase("sample-load[" + sampleMethod.getMetadataName() + "]", datasetName, sampleElapsedNanos,
+ opStats.get(0));
Index.SampleIndexDetails newIndexDetailsFinal = new Index.SampleIndexDetails(dsDetails.getPrimaryKey(),
dsDetails.getKeySourceIndicator(), dsDetails.getPrimaryKeyType(), sampleCardinalityTarget,
@@ -6304,6 +6313,68 @@
}
}
+ /**
+ * {@code hot} is {@code pinned - cold}: pages the buffer cache already held, which cost no I/O.
+ *
+ * @param opStats the job's aggregated operator stats, or null if the job produced none
+ */
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_5, tool = AiProvenance.Tool.CLAUDE_CODE_CLI, contributionKind = AiProvenance.ContributionKind.GENERATED, notes = "DEBUG-gated per-phase breakdown of the ANALYZE pipeline")
+ private static void logAnalyzePhase(String phase, String datasetName, long elapsedNanos, IOperatorStats opStats) {
+ if (!LOGGER.isDebugEnabled()) {
+ return;
+ }
+ if (opStats == null) {
+ LOGGER.debug("ANALYZE {} phase={} elapsed={}ms (no operator stats)", datasetName, phase,
+ TimeUnit.NANOSECONDS.toMillis(elapsedNanos));
+ return;
+ }
+ long pinned = opStats.getPageReadCounter().get();
+ long cold = opStats.coldReadCounter().get();
+ long liveness = opStats.sampleLivenessTimeCounter().get();
+ long pkSeek = opStats.samplePkSeekTimeCounter().get();
+ long materialize = opStats.sampleMaterializeTimeCounter().get();
+ long phaseTotal = liveness + pkSeek + materialize;
+ LOGGER.debug(
+ "ANALYZE {} phase={} elapsed={}ms pinnedPages={} hot={} cold={} hitRatio={} cloudPageReads={} "
+ + "tuples={} sampledTuples={} tupleBytes={} cursor[liveness={}({}) pkSeek={}({}) "
+ + "materialize={}({})]",
+ datasetName, phase, TimeUnit.NANOSECONDS.toMillis(elapsedNanos), pinned, pinned - cold, cold,
+ pinned > 0 ? String.format("%.2f%%", (pinned - cold) * 100.0 / pinned) : "n/a",
+ opStats.cloudReadRequestCounter().get(), opStats.getTupleCounter().get(),
+ opStats.getInputTupleCounter().get(), opStats.getTupleBytes().get(), millis(liveness),
+ sharePct(liveness, phaseTotal), millis(pkSeek), sharePct(pkSeek, phaseTotal), millis(materialize),
+ sharePct(materialize, phaseTotal));
+ }
+
+ /** Fractional, so a sub-millisecond phase does not read as a flat zero. */
+ private static String millis(long nanos) {
+ return String.format("%.2fms", nanos / 1_000_000.0);
+ }
+
+ /** {@code n/a} when no instrumented cursor contributed. */
+ private static String sharePct(long part, long total) {
+ return total > 0 ? String.format("%.1f%%", part * 100.0 / total) : "n/a";
+ }
+
+ /**
+ * ANALYZE is DDL, so it never reaches {@link #updateJobStats}, which is what populates these for a query. The
+ * ratio is computed the way {@code JobResultCallback} computes it, so the two are comparable.
+ *
+ * @param requestParameters the request whose stats to update; a null {@code Stats} is tolerated
+ */
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_5, tool = AiProvenance.Tool.CLAUDE_CODE_CLI, contributionKind = AiProvenance.ContributionKind.GENERATED, notes = "Report ANALYZE's buffer cache hit ratio and page read count, which DDL statements never surfaced")
+ private static void reportSamplingBufferCacheStats(IRequestParameters requestParameters,
+ DatasetStreamStats sampleStats) {
+ Stats requestStats = requestParameters.getStats();
+ if (requestStats == null) {
+ return;
+ }
+ long pinnedPages = sampleStats.getPinnedPages();
+ requestStats.setBufferCachePageReadCount(pinnedPages);
+ requestStats.setBufferCacheHitRatio(
+ pinnedPages > 0 ? (pinnedPages - sampleStats.getColdReads()) / (double) pinnedPages : Double.NaN);
+ }
+
private void runJob(IHyracksClientConnection hcc, JobSpecification jobSpec) throws Exception {
runJob(hcc, jobSpec, jobFlags);
}
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/metrics.xml b/asterixdb/asterix-app/src/test/resources/runtimets/metrics.xml
index a2c3ae5..1c75555 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/metrics.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/metrics.xml
@@ -45,5 +45,10 @@
<output-dir compare="Text">external-dataset</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="metrics">
+ <compilation-unit name="analyze-buffer-cache">
+ <output-dir compare="Text">analyze-buffer-cache</output-dir>
+ </compilation-unit>
+ </test-case>
</test-group>
</test-suite>
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/metrics/analyze-buffer-cache/analyze-buffer-cache.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/metrics/analyze-buffer-cache/analyze-buffer-cache.1.ddl.sqlpp
new file mode 100644
index 0000000..f195ee2
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/metrics/analyze-buffer-cache/analyze-buffer-cache.1.ddl.sqlpp
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+drop dataverse test if exists;
+create dataverse test;
+use test;
+create type test.T as closed { id : bigint, name : string };
+create dataset ds(T) primary key id with {"storage-format": {"format": "column"}};
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/metrics/analyze-buffer-cache/analyze-buffer-cache.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/metrics/analyze-buffer-cache/analyze-buffer-cache.2.update.sqlpp
new file mode 100644
index 0000000..ee03f47
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/metrics/analyze-buffer-cache/analyze-buffer-cache.2.update.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use test;
+insert into ds (select value {"id": i, "name": "name_" || string(i)} from range(1, 20000) i);
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/metrics/analyze-buffer-cache/analyze-buffer-cache.3.metrics.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/metrics/analyze-buffer-cache/analyze-buffer-cache.3.metrics.sqlpp
new file mode 100644
index 0000000..6357a1a
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/metrics/analyze-buffer-cache/analyze-buffer-cache.3.metrics.sqlpp
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * The expected result is a single-line regex (results/.../analyze-buffer-cache.3.regexadm). It deliberately
+ * requires a NON-ZERO bufferCachePageReadCount and a ratio other than 0.00%, because an earlier form of that
+ * fixture matched only the metric field NAMES and so passed against 0.00%/0 on every run -- it could not
+ * fail, and did not, while the values were meaningless. Note the .regexadm format has no comment syntax:
+ * every line is matched, so the rationale lives here rather than beside the pattern.
+ */
+use test;
+ANALYZE DATASET ds WITH {"sample": "low", "sample-seed": 1000};
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/metrics/analyze-buffer-cache/analyze-buffer-cache.4.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/metrics/analyze-buffer-cache/analyze-buffer-cache.4.ddl.sqlpp
new file mode 100644
index 0000000..d4b1d9d
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/metrics/analyze-buffer-cache/analyze-buffer-cache.4.ddl.sqlpp
@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+drop dataverse test;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/metrics/spike-secondary-io/spike-secondary-io.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/metrics/spike-secondary-io/spike-secondary-io.1.ddl.sqlpp
new file mode 100644
index 0000000..dfaa37d
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/metrics/spike-secondary-io/spike-secondary-io.1.ddl.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+drop dataverse snap if exists;
+drop dataverse nocomp if exists;
+create dataverse snap;
+create dataverse nocomp;
+create type snap.T as closed { id : bigint, k : bigint, name : string, ts : string };
+create type nocomp.T as closed { id : bigint, k : bigint, name : string, ts : string };
+create dataset snap.ds(snap.T) primary key id with {"storage-block-compression": {"scheme": "snappy"}};
+create dataset nocomp.ds(nocomp.T) primary key id with {"storage-block-compression": {"scheme": "none"}};
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/metrics/spike-secondary-io/spike-secondary-io.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/metrics/spike-secondary-io/spike-secondary-io.2.update.sqlpp
new file mode 100644
index 0000000..35e1801
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/metrics/spike-secondary-io/spike-secondary-io.2.update.sqlpp
@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+load dataset snap.ds using localfs ((`path`=`asterix_nc1:///private/tmp/claude-501/-Users-ritik-raj-Documents-work-helio-analytics/61f37495-c319-460c-b834-4976c5ba6112/scratchpad/spike.csv`),(`format`=`delimited-text`),(`delimiter`=`|`));
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/metrics/spike-secondary-io/spike-secondary-io.3.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/metrics/spike-secondary-io/spike-secondary-io.3.update.sqlpp
new file mode 100644
index 0000000..0500609
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/metrics/spike-secondary-io/spike-secondary-io.3.update.sqlpp
@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+load dataset nocomp.ds using localfs ((`path`=`asterix_nc1:///private/tmp/claude-501/-Users-ritik-raj-Documents-work-helio-analytics/61f37495-c319-460c-b834-4976c5ba6112/scratchpad/spike.csv`),(`format`=`delimited-text`),(`delimiter`=`|`));
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/metrics/spike-secondary-io/spike-secondary-io.4.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/metrics/spike-secondary-io/spike-secondary-io.4.ddl.sqlpp
new file mode 100644
index 0000000..3d1e0d5
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/metrics/spike-secondary-io/spike-secondary-io.4.ddl.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+create index snap_k on snap.ds(k);
+create index nocomp_k on nocomp.ds(k);
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/metrics/spike-secondary-io/spike-secondary-io.5.metrics.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/metrics/spike-secondary-io/spike-secondary-io.5.metrics.sqlpp
new file mode 100644
index 0000000..7ae5f12
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/metrics/spike-secondary-io/spike-secondary-io.5.metrics.sqlpp
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use snap;
+set `compiler.cbo` "false";
+select count(*) from ds where k between 100000000 and 300000000;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/metrics/spike-secondary-io/spike-secondary-io.6.metrics.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/metrics/spike-secondary-io/spike-secondary-io.6.metrics.sqlpp
new file mode 100644
index 0000000..2b58a6e
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/metrics/spike-secondary-io/spike-secondary-io.6.metrics.sqlpp
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use nocomp;
+set `compiler.cbo` "false";
+select count(*) from ds where k between 100000000 and 300000000;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/metrics/spike-secondary-io/spike-secondary-io.7.metrics.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/metrics/spike-secondary-io/spike-secondary-io.7.metrics.sqlpp
new file mode 100644
index 0000000..7ae5f12
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/metrics/spike-secondary-io/spike-secondary-io.7.metrics.sqlpp
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use snap;
+set `compiler.cbo` "false";
+select count(*) from ds where k between 100000000 and 300000000;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/metrics/spike-secondary-io/spike-secondary-io.8.metrics.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/metrics/spike-secondary-io/spike-secondary-io.8.metrics.sqlpp
new file mode 100644
index 0000000..2b58a6e
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/metrics/spike-secondary-io/spike-secondary-io.8.metrics.sqlpp
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use nocomp;
+set `compiler.cbo` "false";
+select count(*) from ds where k between 100000000 and 300000000;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/metrics/analyze-buffer-cache/analyze-buffer-cache.3.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/metrics/analyze-buffer-cache/analyze-buffer-cache.3.regexadm
new file mode 100644
index 0000000..3e0c055
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/metrics/analyze-buffer-cache/analyze-buffer-cache.3.regexadm
@@ -0,0 +1 @@
+(?s).*"bufferCacheHitRatio":\s*"(?!0\.00%)[0-9]+\.[0-9]+%".*"bufferCachePageReadCount":\s*[1-9][0-9]*.*
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/metrics/spike-secondary-io/spike-secondary-io.5.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/metrics/spike-secondary-io/spike-secondary-io.5.regexadm
new file mode 100644
index 0000000..40188a3
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/metrics/spike-secondary-io/spike-secondary-io.5.regexadm
@@ -0,0 +1 @@
+(?s).*
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/metrics/spike-secondary-io/spike-secondary-io.6.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/metrics/spike-secondary-io/spike-secondary-io.6.regexadm
new file mode 100644
index 0000000..40188a3
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/metrics/spike-secondary-io/spike-secondary-io.6.regexadm
@@ -0,0 +1 @@
+(?s).*
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/metrics/spike-secondary-io/spike-secondary-io.7.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/metrics/spike-secondary-io/spike-secondary-io.7.regexadm
new file mode 100644
index 0000000..40188a3
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/metrics/spike-secondary-io/spike-secondary-io.7.regexadm
@@ -0,0 +1 @@
+(?s).*
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/metrics/spike-secondary-io/spike-secondary-io.8.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/metrics/spike-secondary-io/spike-secondary-io.8.regexadm
new file mode 100644
index 0000000..40188a3
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/metrics/spike-secondary-io/spike-secondary-io.8.regexadm
@@ -0,0 +1 @@
+(?s).*
diff --git a/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/sample/ColumnSampleBenchHarness.java b/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/sample/ColumnSampleBenchHarness.java
index 4ce7f1e..b39408b 100644
--- a/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/sample/ColumnSampleBenchHarness.java
+++ b/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/sample/ColumnSampleBenchHarness.java
@@ -57,6 +57,7 @@
import org.apache.hyracks.storage.am.btree.impls.BTreeOpContext;
import org.apache.hyracks.storage.am.btree.impls.RangePredicate;
import org.apache.hyracks.storage.am.common.api.ITreeIndexCursor;
+import org.apache.hyracks.storage.am.common.api.SampleCursorStats;
import org.apache.hyracks.storage.am.common.impls.IndexAccessParameters;
import org.apache.hyracks.storage.am.common.impls.NoOpIndexAccessParameters;
import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback;
@@ -69,7 +70,6 @@
import org.apache.hyracks.storage.am.lsm.btree.column.cloud.buffercache.IColumnReadContext;
import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.ColumnBTree;
import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.ColumnBtreeSampleCursor;
-import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.SampleCursorStats;
import org.apache.hyracks.storage.am.lsm.btree.column.impls.lsm.LSMColumnBTreeOpContext;
import org.apache.hyracks.storage.am.lsm.btree.column.utils.LSMColumnBTreeUtil;
import org.apache.hyracks.storage.am.lsm.btree.impls.AntimatterAwareTupleAcceptor;
@@ -185,6 +185,18 @@
private static final int MAX_TUPLE_COUNT = 15000;
private static final double TOLERANCE = 0.15;
private static final int MAX_LEAF_NODE_SIZE = 8 * PAGE_SIZE;
+ /**
+ * Phase 2 pins <em>every column mega-page of a leaf</em> per new leaf
+ * ({@code IColumnReadContext#prepareColumns}), amortized only across samples landing on the same leaf.
+ * Uniform draws rarely collide, so pins-per-sample should track leaf size. Production measured 64 pins per
+ * sample at a 10 MB leaf.
+ */
+ private int maxLeafNodeSizeOverride = -1;
+ /**
+ * This -- not {@link #MAX_LEAF_NODE_SIZE} -- caps a columnar leaf at this harness's row width: 250 000 keys
+ * land in ~17 leaves of ~14 700 tuples, so the byte ceiling is never reached and sweeping it measures nothing.
+ */
+ private int maxTupleCountOverride = -1;
// Column buffer pool knobs. Raised from the brief's guess of (32768, 64, 64 MB, 1000 ms): the granule is
// matched to PAGE_SIZE so one credit is one page, the pool size and memory ceiling are doubled because the
@@ -233,6 +245,12 @@
private int samplesPerPage = DEFAULT_SAMPLES_PER_PAGE;
/** {@code < 0} means "read the real value from the sampled component's metadata", as production does. */
private int maxLeafTupleCountOverride = -1;
+ /**
+ * {@code <= 0} sizes the cache from {@link #diskPagesFor(long)}, keeping the index resident; a positive value
+ * installs that budget verbatim. Both arms then read byte-identical files and differ only in how much can stay
+ * pinned.
+ */
+ private int diskPageBudgetOverride = -1;
private ARecordType recordType;
private IBufferCache diskBufferCache;
@@ -275,7 +293,9 @@
// closed again by harness.tearDown()); only its page-size-independent pieces are consumed below.
harness.setUp();
IIOManager ioManager = harness.getIOManager();
- diskBufferCache = createDiskBufferCache(ioManager, diskPagesFor((long) numKeys + getShadowBoundary()));
+ int diskPages = diskPageBudgetOverride > 0 ? diskPageBudgetOverride
+ : diskPagesFor((long) numKeys + getShadowBoundary());
+ diskBufferCache = createDiskBufferCache(ioManager, diskPages);
columnBufferPool = new ColumnBufferPool(COLUMN_BUFFER_GRANULE_BYTES, COLUMN_BUFFER_POOL_SIZE,
COLUMN_BUFFER_MAX_MEMORY, COLUMN_BUFFER_RESERVE_TIMEOUT_MILLIS);
virtualBufferCaches = new ArrayList<>();
@@ -294,7 +314,8 @@
IColumnManagerFactory columnManagerFactory = new ColumnManagerFactory(recordType, null,
Collections.singletonList(Collections.singletonList(PK_FIELD)), Collections.singletonList(0), PAGE_SIZE,
- MAX_TUPLE_COUNT, TOLERANCE, MAX_LEAF_NODE_SIZE);
+ maxTupleCountOverride > 0 ? maxTupleCountOverride : MAX_TUPLE_COUNT, TOLERANCE,
+ maxLeafNodeSizeOverride > 0 ? maxLeafNodeSizeOverride : MAX_LEAF_NODE_SIZE);
lsmBtree = LSMColumnBTreeUtil.createLSMTree(harness.getNCConfig(), ioManager, virtualBufferCaches,
harness.getFileReference(), diskBufferCache, columnBufferPool, typeTraits, cmpFactories,
@@ -701,6 +722,28 @@
return pageReplacementStrategy.getNumPages();
}
+ /**
+ * Call before {@link #setUp()}. A budget below {@link #getIndexPageCount()} breaks the residency contract
+ * {@code ColumnSampleBenchHarnessResidencyTest} enforces, forcing faults through {@link IBufferCache}. Those
+ * reads are served by the OS page cache, not the device — a component small enough to build in a test is small
+ * enough for the OS to hold — so this arm measures eviction and the pin-miss path, <b>not</b> device latency.
+ *
+ * @param pages the budget to install, or {@code <= 0} to restore the default sizing
+ */
+ public void setDiskPageBudget(int pages) {
+ this.diskPageBudgetOverride = pages;
+ }
+
+ /** Call before {@link #setUp()}. */
+ public void setMaxTupleCount(int tuples) {
+ this.maxTupleCountOverride = tuples;
+ }
+
+ /** Bytes. Call before {@link #setUp()}. */
+ public void setMaxLeafNodeSize(int bytes) {
+ this.maxLeafNodeSizeOverride = bytes;
+ }
+
/** The disk buffer cache's real configured page budget. */
public int getBufferCachePageBudget() {
return pageReplacementStrategy.getMaxAllowedNumPages();
diff --git a/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/sample/ColumnSampleBenchHarnessResidencyTest.java b/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/sample/ColumnSampleBenchHarnessResidencyTest.java
index 1eac46a..ca44f10 100644
--- a/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/sample/ColumnSampleBenchHarnessResidencyTest.java
+++ b/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/sample/ColumnSampleBenchHarnessResidencyTest.java
@@ -19,7 +19,7 @@
package org.apache.asterix.column.test.sample;
import org.apache.asterix.column.test.sample.ColumnSampleBenchHarness.Projection;
-import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.SampleCursorStats;
+import org.apache.hyracks.storage.am.common.api.SampleCursorStats;
import org.apache.hyracks.util.annotations.AiProvenance;
import org.junit.Assert;
import org.junit.Test;
diff --git a/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/sample/ColumnSampleBenchHarnessTest.java b/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/sample/ColumnSampleBenchHarnessTest.java
index 10935b9..b4fe802 100644
--- a/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/sample/ColumnSampleBenchHarnessTest.java
+++ b/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/sample/ColumnSampleBenchHarnessTest.java
@@ -22,7 +22,7 @@
import java.util.Collection;
import org.apache.asterix.column.test.sample.ColumnSampleBenchHarness.Projection;
-import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.SampleCursorStats;
+import org.apache.hyracks.storage.am.common.api.SampleCursorStats;
import org.apache.hyracks.util.annotations.AiProvenance;
import org.junit.After;
import org.junit.Assert;
diff --git a/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/sample/ColumnSampleCostBaselineTest.java b/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/sample/ColumnSampleCostBaselineTest.java
index 296afa5..d8a595a 100644
--- a/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/sample/ColumnSampleCostBaselineTest.java
+++ b/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/sample/ColumnSampleCostBaselineTest.java
@@ -18,7 +18,7 @@
*/
package org.apache.asterix.column.test.sample;
-import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.SampleCursorStats;
+import org.apache.hyracks.storage.am.common.api.SampleCursorStats;
import org.apache.hyracks.util.annotations.AiProvenance;
import org.junit.Assert;
import org.junit.Test;
diff --git a/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/sample/ColumnSamplePinCostTest.java b/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/sample/ColumnSamplePinCostTest.java
new file mode 100644
index 0000000..7395f24
--- /dev/null
+++ b/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/sample/ColumnSamplePinCostTest.java
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.test.sample;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.asterix.column.test.sample.ColumnSampleBenchHarness.Projection;
+import org.apache.hyracks.storage.am.common.api.SampleCursorStats;
+import org.apache.hyracks.util.IThreadStatsCollector;
+import org.apache.hyracks.util.ThreadStats;
+import org.apache.hyracks.util.annotations.AiProvenance;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Attributes the columnar sample cursor's page-pin cost, which a 128-partition cluster run measured at
+ * <b>64 page pins per collected sample</b> against <b>3 for row format</b> at 2.5 billion rows -- a ~20x
+ * difference that made the same retry-loop defect (MB-73172) invisible on row and catastrophic on column.
+ * <p>
+ * <b>The hypothesis under test.</b> {@code yieldNextFromPhase2} pins every column mega-page of a leaf on
+ * each new leaf it visits, and that cost is amortized only across collected samples landing on the
+ * <em>same</em> leaf:
+ *
+ * <pre>
+ * if (pageId != prevYieldPageId) { // new leaf: pin page0, preparePageZeroSegments, prepareColumns
+ * } else { // same leaf: free
+ * </pre>
+ *
+ * Uniform draws over a large component almost never collide on a leaf, so the divisor is ~1 and every
+ * sample pays a full leaf column load. If that is right, pins-per-sample must scale with leaf size and be
+ * insensitive to the number of samples -- and shrinking the leaf is a direct lever on the cost.
+ * <p>
+ * <b>Why this is a measurement, not a regression gate.</b> The absolute pin counts depend on the harness's
+ * schema and page size, so asserting exact values would encode this machine's shape. The assertions check
+ * only the structural claims the hypothesis stands or falls on; the printed table is the result.
+ */
+@AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_5, tool = AiProvenance.Tool.CLAUDE_CODE_CLI, contributionKind = AiProvenance.ContributionKind.TEST_GENERATED, notes = "Attributes columnar sample pins to leaf size, testing whether phase-2 column loading is the driver")
+public class ColumnSamplePinCostTest {
+
+ private static final int NUM_KEYS = 250000;
+ private static final int SHADOW_PERCENT = 90;
+ private static final int NUM_SHADOW_COMPONENTS = 6;
+ private static final long SEED = 12345L;
+ private static final int PAGE = ColumnSampleBenchHarness.PAGE_SIZE;
+
+ /**
+ * {@code totalPins} is {@link ThreadStats}'s pinned-page count -- every buffer-cache pin, including the
+ * ones {@code prepareColumns} issues internally. {@code leavesVisited} is {@code phase2PagePins}, which
+ * counts only the page0 pin per new leaf. The cluster's "64 pins per sample" figure is the former; an
+ * earlier version of this test reported the latter and the two are not comparable.
+ */
+ private record Row(String arm, int leafBytes, int target, long totalPins, long leavesVisited, int collected,
+ long phase2Nanos) {
+ double pinsPerSample() {
+ return collected > 0 ? (double) totalPins / collected : Double.NaN;
+ }
+
+ double samplesPerLeaf() {
+ return leavesVisited > 0 ? (double) collected / leavesVisited : Double.NaN;
+ }
+ }
+
+ private static Row run(String arm, int leafBytes, int target, Projection proj) throws Exception {
+ return run(arm, leafBytes, target, proj, -1);
+ }
+
+ private static Row run(String arm, int leafBytes, int target, Projection proj, int maxTuples) throws Exception {
+ ColumnSampleBenchHarness h =
+ new ColumnSampleBenchHarness(NUM_KEYS, SHADOW_PERCENT, NUM_SHADOW_COMPONENTS, proj);
+ if (leafBytes > 0) {
+ h.setMaxLeafNodeSize(leafBytes);
+ }
+ if (maxTuples > 0) {
+ h.setMaxTupleCount(maxTuples);
+ }
+ h.setUp();
+ try {
+ h.runSample(target, SEED, new SampleCursorStats()); // warm up JIT and page cache
+ SampleCursorStats st = new SampleCursorStats();
+ ThreadStats ts = new ThreadStats();
+ ((IThreadStatsCollector) h.getDiskBufferCache()).subscribe(ts);
+ ColumnSampleBenchHarness.SampleRun r;
+ try {
+ r = h.runSample(target, SEED, st);
+ } finally {
+ ((IThreadStatsCollector) h.getDiskBufferCache()).unsubscribe();
+ }
+ return new Row(arm, leafBytes > 0 ? leafBytes : PAGE * 8, target, ts.getPinnedPagesCount(),
+ st.phase2PagePins, r.emitted, st.phase2Nanos);
+ } finally {
+ h.tearDown();
+ }
+ }
+
+ @Test
+ public void pinsPerSampleTrackLeafSizeNotSampleCount() throws Exception {
+ List<Row> rows = new ArrayList<>();
+ // Leaf size swept at a fixed target: if phase 2's whole-leaf column load is the driver, this is the
+ // axis that moves pins-per-sample.
+ // Tuples per leaf is the real lever: fewer tuples per leaf means fewer pages per leaf, and at ~1
+ // sample per leaf (the regime a large component puts you in) pins/sample tracks pages/leaf directly.
+ for (int maxTuples : new int[] { 15000, 4000, 1000, 250, 60 }) {
+ rows.add(run("maxTuples=" + maxTuples, PAGE * 8, 1063, Projection.ALL_COLUMNS, maxTuples));
+ }
+ // Target swept at a fixed leaf size: pins should grow with the sample count but pins-per-sample stay
+ // roughly flat, since each sample lands on its own leaf either way.
+ for (int target : new int[] { 1063, 4252, 17008 }) {
+ rows.add(run("target=" + target, PAGE * 8, target, Projection.ALL_COLUMNS));
+ }
+ // PK-only projection: phase 2 loads no column data at all, which bounds how much of the cost is
+ // column materialization rather than the page0 pin itself.
+ rows.add(run("PK_ONLY", PAGE * 8, 1063, Projection.PK_ONLY));
+
+ System.out.println();
+ System.out.println("=== columnar sample page-pin attribution ===");
+ System.out.printf("%-14s %10s %8s %10s %11s %8s %12s %13s%n", "arm", "leafBytes", "target", "collected",
+ "totalPins", "leaves", "pins/sample", "samples/leaf");
+ for (Row r : rows) {
+ System.out.printf("%-14s %10d %8d %10d %11d %8d %12.1f %13.1f%n", r.arm(), r.leafBytes(), r.target(),
+ r.collected(), r.totalPins(), r.leavesVisited(), r.pinsPerSample(), r.samplesPerLeaf());
+ }
+ System.out.println();
+
+ Row leaf1 = rows.get(0);
+ Row leaf16 = rows.get(4);
+ Row pkOnly = rows.get(rows.size() - 1);
+ System.out.printf(
+ "NOTE harness regime: %.1f samples per leaf. The 128-partition cluster run sits near "
+ + "1.0, where phase 2 cannot amortize; above that this harness measures the amortized case.%n",
+ rows.get(3).samplesPerLeaf());
+
+ // Structural claim 1: a bigger leaf costs more pins per sample. If this fails, phase 2's whole-leaf
+ // column load is NOT what drives the cost and the reduction lever is somewhere else entirely.
+ Assert.assertTrue("pins/sample did not grow with leaf size: 1 page -> " + leaf1.pinsPerSample()
+ + ", 16 pages -> " + leaf16.pinsPerSample(), leaf16.pinsPerSample() > leaf1.pinsPerSample());
+
+ // Structural claim 2: pins/sample is a per-leaf property, so it must not scale with the sample target.
+ double smallTarget = rows.get(5).pinsPerSample();
+ double largeTarget = rows.get(7).pinsPerSample();
+ Assert.assertTrue(
+ "pins/sample changed more than 3x across a 16x target sweep (" + smallTarget + " -> " + largeTarget
+ + "), so it is not purely a per-leaf cost",
+ largeTarget < smallTarget * 3 && smallTarget < largeTarget * 3);
+
+ // Structural claim 3: dropping column materialization must reduce pins, or "load fewer columns" is
+ // not a viable lever.
+ Assert.assertTrue("PK_ONLY (" + pkOnly.pinsPerSample() + ") should pin no more per sample than "
+ + "ALL_COLUMNS (" + rows.get(3).pinsPerSample() + ")",
+ pkOnly.pinsPerSample() <= rows.get(3).pinsPerSample());
+ }
+}
diff --git a/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/sample/ColumnSampleResidencyBenchTest.java b/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/sample/ColumnSampleResidencyBenchTest.java
new file mode 100644
index 0000000..ad9579f
--- /dev/null
+++ b/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/sample/ColumnSampleResidencyBenchTest.java
@@ -0,0 +1,220 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.test.sample;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.asterix.column.test.sample.ColumnSampleBenchHarness.Projection;
+import org.apache.hyracks.storage.am.common.api.SampleCursorStats;
+import org.apache.hyracks.util.IThreadStatsCollector;
+import org.apache.hyracks.util.ThreadStats;
+import org.apache.hyracks.util.annotations.AiProvenance;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Measures the cost of a columnar sample scan as a function of how much of the index the disk buffer cache can
+ * hold, by shrinking the cache underneath a fixed index rather than growing the index past a fixed cache.
+ * <p>
+ * <b>Why shrink the cache instead of growing the data.</b> Both arms must read byte-identical component files, or
+ * the comparison silently becomes one of file size. Growing the dataset past a fixed budget changes the number of
+ * leaf pages, the tree height, the tuples per page and the sample's page-visit count all at once, so a difference
+ * in elapsed time cannot be attributed to residency. Holding the index fixed and moving only
+ * {@link ColumnSampleBenchHarness#setDiskPageBudget(int)} leaves the draw sequence, the liveness work and the
+ * column materialization identical between arms, so the delta is eviction and re-pinning and nothing else.
+ * <p>
+ * <b>What this does not measure.</b> The re-reads are served by the operating system's page cache, not by the
+ * device: a component small enough to build inside a test is small enough for the OS to hold whole. So the numbers
+ * here are the eviction, pin-miss and {@code pread}-plus-copy cost, and they are a <em>lower bound</em> on the cost
+ * of the same scan against a cold device. A genuinely device-bound figure needs a host where the page cache can be
+ * dropped and the data exceeds RAM; it cannot be obtained here, and this class does not claim to.
+ * <p>
+ * The reported hit ratio is computed from {@link ThreadStats}, subscribed to the buffer cache on the sampling
+ * thread — the same {@code pinnedPages} and {@code coldReads} counters that back the {@code bufferCacheHitRatio}
+ * the ANALYZE statement reports, so a figure measured here is directly comparable to one observed on a cluster.
+ */
+@AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_5, tool = AiProvenance.Tool.CLAUDE_CODE_CLI, contributionKind = AiProvenance.ContributionKind.TEST_GENERATED, notes = "Residency sweep for the sample cursor: fixed index, shrinking buffer cache, per-phase attribution")
+public class ColumnSampleResidencyBenchTest {
+
+ private static final int NUM_KEYS = 250000;
+ private static final int SHADOW_PERCENT = 90;
+ private static final int NUM_SHADOW_COMPONENTS = 6;
+ private static final int TARGET_CARDINALITY = 1063;
+ private static final long SEED = 12345L;
+ private static final int WARMUP_RUNS = 3;
+ private static final int MEASURED_RUNS = 7;
+ /**
+ * Samples run on a throwaway harness before the sweep starts. Without these the first arm absorbs the JIT and
+ * class-loading cost of the whole sample path and reads ~18 % slower than a later arm doing identical work at
+ * an identical 100 % hit ratio - which, since the first arm is the baseline every other arm is divided by,
+ * understates the effect being measured rather than merely adding noise.
+ */
+ private static final int GLOBAL_WARMUP_RUNS = 5;
+
+ /**
+ * Fractions of the measured index footprint to use as the cache budget. {@code 1.0} is a superset of the
+ * footprint and so reproduces the resident arm; the rest force eviction. The floor in
+ * {@link #budgetFor(int, double)} keeps the smallest arm above the page count a tree search needs pinned at
+ * once, below which the clock strategy would be unable to find a victim.
+ */
+ private static final double[] BUDGET_FRACTIONS = { 2.0, 1.0, 0.75, 0.5, 0.25, 0.10 };
+ /**
+ * The arm every other arm is reported relative to: the smallest budget that still holds the whole index, so
+ * the ratios isolate residency.
+ * <p>
+ * Deliberately not the {@code 2.0} arm, even though that one is the only arm that can satisfy the strict
+ * {@code allocated < budget} residency proof. An over-sized budget is measurably <em>slower</em> at an
+ * identical 100 % hit ratio doing identical work - ~11 %, reproducibly - because the budget also sizes the
+ * buffer cache's page map and its lazily grown allocation (349 pages against this index's 312). That is a
+ * cache-sizing artifact, not a residency effect, and baselining on it would fold an unrelated 11 % into every
+ * ratio below. The {@code 2.0} arm is retained only to prove non-eviction, which {@code 1.0} cannot.
+ */
+ private static final double BASELINE_FRACTION = 1.0;
+ private static final int MIN_WORKABLE_BUDGET_PAGES = 64;
+
+ private static int budgetFor(int indexPages, double fraction) {
+ return Math.max(MIN_WORKABLE_BUDGET_PAGES, (int) Math.ceil(indexPages * fraction));
+ }
+
+ private record Arm(double fraction, int budget, int indexPages, int allocated, long medianNanos, long pinnedPages,
+ long coldReads, long livenessNanos, long pkSeekNanos, long phase2Nanos, int collected) {
+
+ double hitRatio() {
+ return pinnedPages > 0 ? (double) (pinnedPages - coldReads) / pinnedPages : Double.NaN;
+ }
+ }
+
+ @Test
+ public void sampleCostVersusBufferCacheResidency() throws Exception {
+ // The footprint is a property of the data, not of the cache, so one build establishes it for every arm.
+ int indexPages;
+ ColumnSampleBenchHarness probe =
+ new ColumnSampleBenchHarness(NUM_KEYS, SHADOW_PERCENT, NUM_SHADOW_COMPONENTS, Projection.ALL_COLUMNS);
+ probe.setUp();
+ try {
+ indexPages = probe.getIndexPageCount();
+ for (int i = 0; i < GLOBAL_WARMUP_RUNS; i++) {
+ probe.runSample(TARGET_CARDINALITY, SEED, new SampleCursorStats());
+ }
+ } finally {
+ probe.tearDown();
+ }
+
+ List<Arm> arms = new ArrayList<>();
+ for (double fraction : BUDGET_FRACTIONS) {
+ arms.add(runArm(fraction, budgetFor(indexPages, fraction)));
+ }
+
+ report(indexPages, arms);
+
+ Arm overSized = arms.get(0);
+ Arm resident = arms.stream().filter(a -> a.fraction() == BASELINE_FRACTION).findFirst().orElseThrow();
+ Arm tightest = arms.get(arms.size() - 1);
+
+ // The premise of the whole sweep: the resident arm really is resident, and the tightest arm really did
+ // evict. Without both, the elapsed-time columns are comparing two identical configurations.
+ Assert.assertTrue(
+ "over-sized arm allocated " + overSized.allocated() + " of " + overSized.budget()
+ + " pages, so it evicted: nothing here is a proven non-evicting reference",
+ overSized.allocated() < overSized.budget());
+ Assert.assertEquals("baseline arm must hold the whole index, else its ratios are not residency ratios", 1.0,
+ resident.hitRatio(), 1e-6);
+ Assert.assertEquals("tightest arm must have filled its " + tightest.budget() + "-page budget, otherwise the "
+ + "index fit after all and no eviction was forced", tightest.budget(), tightest.allocated());
+ Assert.assertTrue("tightest arm hit ratio " + tightest.hitRatio() + " must be below the resident arm's "
+ + resident.hitRatio(), tightest.hitRatio() < resident.hitRatio());
+
+ // Every arm samples the same index with the same seed, so a differing sample size would mean the cache
+ // budget changed which tuples were collected - it must not.
+ for (Arm arm : arms) {
+ Assert.assertEquals(
+ "arm " + arm.fraction() + " collected a different sample size, so the cache budget "
+ + "changed the draw outcome and the arms are not comparable",
+ resident.collected(), arm.collected());
+ }
+ }
+
+ private Arm runArm(double fraction, int budget) throws Exception {
+ ColumnSampleBenchHarness harness =
+ new ColumnSampleBenchHarness(NUM_KEYS, SHADOW_PERCENT, NUM_SHADOW_COMPONENTS, Projection.ALL_COLUMNS);
+ harness.setDiskPageBudget(budget);
+ harness.setUp();
+ try {
+ // Warm up so class loading and JIT are not charged to the first measured run of the first arm.
+ for (int i = 0; i < WARMUP_RUNS; i++) {
+ harness.runSample(TARGET_CARDINALITY, SEED, new SampleCursorStats());
+ }
+
+ long[] elapsed = new long[MEASURED_RUNS];
+ SampleCursorStats totals = new SampleCursorStats();
+ ThreadStats threadStats = new ThreadStats();
+ int collected = -1;
+ ((IThreadStatsCollector) harness.getDiskBufferCache()).subscribe(threadStats);
+ try {
+ for (int i = 0; i < MEASURED_RUNS; i++) {
+ SampleCursorStats runStats = new SampleCursorStats();
+ ColumnSampleBenchHarness.SampleRun run = harness.runSample(TARGET_CARDINALITY, SEED, runStats);
+ elapsed[i] = run.wallNanos;
+ collected = run.emitted;
+ totals.livenessNanos += runStats.livenessNanos;
+ totals.pkSeekNanos += runStats.pkSeekNanos;
+ totals.phase2Nanos += runStats.phase2Nanos;
+ }
+ } finally {
+ ((IThreadStatsCollector) harness.getDiskBufferCache()).unsubscribe();
+ }
+
+ java.util.Arrays.sort(elapsed);
+ return new Arm(fraction, harness.getBufferCachePageBudget(), harness.getIndexPageCount(),
+ harness.getAllocatedBufferCachePages(), elapsed[MEASURED_RUNS / 2],
+ threadStats.getPinnedPagesCount() / MEASURED_RUNS, threadStats.getColdReadCount() / MEASURED_RUNS,
+ totals.livenessNanos / MEASURED_RUNS, totals.pkSeekNanos / MEASURED_RUNS,
+ totals.phase2Nanos / MEASURED_RUNS, collected);
+ } finally {
+ harness.tearDown();
+ }
+ }
+
+ private static void report(int indexPages, List<Arm> arms) {
+ double residentMedian =
+ arms.stream().filter(a -> a.fraction() == BASELINE_FRACTION).findFirst().orElseThrow().medianNanos();
+ System.out.println();
+ System.out.println("=== sample cost vs buffer-cache residency ===");
+ System.out.printf("index footprint: %d pages of %d KB = %.1f MB; target=%d tuples; median of %d runs%n",
+ indexPages, ColumnSampleBenchHarness.PAGE_SIZE / 1024,
+ indexPages * (ColumnSampleBenchHarness.PAGE_SIZE / 1024.0) / 1024.0, TARGET_CARDINALITY, MEASURED_RUNS);
+ System.out.println("NOTE: re-reads are served by the OS page cache, not the device - lower bound on cold cost");
+ System.out.println();
+ System.out.printf("%-9s %-8s %-10s %-9s %-9s %-8s %-9s %-9s %-11s%n", "budget", "alloc", "median", "vs 100%",
+ "hit%", "pinned", "liveness", "pkSeek", "materialize");
+ for (Arm a : arms) {
+ System.out.printf("%-9s %-8d %-10s %-9s %-9s %-8d %-9s %-9s %-11s%n",
+ a.budget() + "p/" + String.format("%.0f%%", a.fraction() * 100), a.allocated(),
+ millis(a.medianNanos()), String.format("%.2fx", a.medianNanos() / residentMedian),
+ Double.isNaN(a.hitRatio()) ? "n/a" : String.format("%.2f", a.hitRatio() * 100), a.pinnedPages(),
+ millis(a.livenessNanos()), millis(a.pkSeekNanos()), millis(a.phase2Nanos()));
+ }
+ System.out.println();
+ }
+
+ private static String millis(long nanos) {
+ return String.format("%.2fms", nanos / 1_000_000.0);
+ }
+}
diff --git a/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/sample/ColumnSampleUniformityTest.java b/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/sample/ColumnSampleUniformityTest.java
index 428cf85..369a98a 100644
--- a/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/sample/ColumnSampleUniformityTest.java
+++ b/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/sample/ColumnSampleUniformityTest.java
@@ -18,7 +18,7 @@
*/
package org.apache.asterix.column.test.sample;
-import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.SampleCursorStats;
+import org.apache.hyracks.storage.am.common.api.SampleCursorStats;
import org.apache.hyracks.util.annotations.AiProvenance;
import org.junit.After;
import org.junit.Assert;
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/DatasetStreamStatsOperatorDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/DatasetStreamStatsOperatorDescriptor.java
index 699fd3b..846cb41 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/DatasetStreamStatsOperatorDescriptor.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/DatasetStreamStatsOperatorDescriptor.java
@@ -43,7 +43,9 @@
import org.apache.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
import org.apache.hyracks.dataflow.std.base.AbstractUnaryInputUnaryOutputOperatorNodePushable;
import org.apache.hyracks.storage.am.common.api.IIndexDataflowHelper;
+import org.apache.hyracks.storage.am.common.api.SampleCursorStats;
import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
+import org.apache.hyracks.storage.am.lsm.btree.dataflow.BTreeSampleCollectorOperatorDescriptorNodePushable;
import org.apache.hyracks.storage.am.lsm.common.api.AbstractLSMWithBloomFilterDiskComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex;
@@ -127,6 +129,14 @@
if (statsCollector != null) {
IOperatorStats stats = statsCollector.getOperatorStats(operatorName);
Object totalNum = TaskUtil.get(ESTIMATE_CARDINALITY, ctx);
+ // Same route as the cardinality estimate; see SAMPLE_CURSOR_STATS.
+ SampleCursorStats cursorPhases =
+ TaskUtil.get(BTreeSampleCollectorOperatorDescriptorNodePushable.SAMPLE_CURSOR_PHASES, ctx);
+ if (cursorPhases != null) {
+ stats.sampleLivenessTimeCounter().update(cursorPhases.livenessNanos);
+ stats.samplePkSeekTimeCounter().update(cursorPhases.pkSeekNanos);
+ stats.sampleMaterializeTimeCounter().update(cursorPhases.phase2Nanos);
+ }
if (totalNum == null) {
totalNum = totalTupleCount;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/profiling/IOperatorStats.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/profiling/IOperatorStats.java
index 66d7dc0..3c440ff 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/profiling/IOperatorStats.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/profiling/IOperatorStats.java
@@ -105,6 +105,17 @@
ICounter getBytesWritten();
+ /**
+ * Nanoseconds probing newer components for liveness. Zero unless an instrumented sampling cursor contributed.
+ */
+ ICounter sampleLivenessTimeCounter();
+
+ /** @see #sampleLivenessTimeCounter() */
+ ICounter samplePkSeekTimeCounter();
+
+ /** @see #sampleLivenessTimeCounter() */
+ ICounter sampleMaterializeTimeCounter();
+
String getOperatorId();
void updateIndexesStats(Map<String, IndexStats> indexesStats);
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/profiling/NoOpOperatorStats.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/profiling/NoOpOperatorStats.java
index ce46f47..5e1418d 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/profiling/NoOpOperatorStats.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/profiling/NoOpOperatorStats.java
@@ -155,6 +155,21 @@
}
@Override
+ public ICounter sampleLivenessTimeCounter() {
+ return NOOP_COUNTER;
+ }
+
+ @Override
+ public ICounter samplePkSeekTimeCounter() {
+ return NOOP_COUNTER;
+ }
+
+ @Override
+ public ICounter sampleMaterializeTimeCounter() {
+ return NOOP_COUNTER;
+ }
+
+ @Override
public String getOperatorId() {
return INVALID_ODID;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/profiling/OperatorStats.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/profiling/OperatorStats.java
index 0ca44d1..f1c59f0 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/profiling/OperatorStats.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/profiling/OperatorStats.java
@@ -50,6 +50,9 @@
public final ICounter level;
public final ICounter bytesRead;
public final ICounter bytesWritten;
+ public final ICounter sampleLivenessTime;
+ public final ICounter samplePkSeekTime;
+ public final ICounter sampleMaterializeTime;
private final Map<String, IndexStats> indexesStats;
//TODO: this is quickly becoming gross it should just be a map where the value is obliged to be a Counter
@@ -75,6 +78,9 @@
level = new Counter("level");
bytesRead = new Counter("bytesRead");
bytesWritten = new Counter("bytesWritten");
+ sampleLivenessTime = new Counter("sampleLivenessTime");
+ samplePkSeekTime = new Counter("samplePkSeekTime");
+ sampleMaterializeTime = new Counter("sampleMaterializeTime");
level.set(-1);
indexesStats = new HashMap<>();
}
@@ -150,6 +156,21 @@
}
@Override
+ public ICounter sampleLivenessTimeCounter() {
+ return sampleLivenessTime;
+ }
+
+ @Override
+ public ICounter samplePkSeekTimeCounter() {
+ return samplePkSeekTime;
+ }
+
+ @Override
+ public ICounter sampleMaterializeTimeCounter() {
+ return sampleMaterializeTime;
+ }
+
+ @Override
public ICounter getBytesWritten() {
return bytesWritten;
}
@@ -194,7 +215,12 @@
pageReads.update(stats.getPageReadCounter().get());
tupleBytes.update(stats.getTupleBytes().get());
coldReadCounter.update(stats.coldReadCounter().get());
- cloudReadRequestCounter.update(stats.cloudReadPageCounter().get());
+ cloudReadRequestCounter.update(stats.cloudReadRequestCounter().get());
+ cloudReadPageCounter.update(stats.cloudReadPageCounter().get());
+ cloudPersistPageCounter.update(stats.cloudPersistPageCounter().get());
+ sampleLivenessTime.update(stats.sampleLivenessTimeCounter().get());
+ samplePkSeekTime.update(stats.samplePkSeekTimeCounter().get());
+ sampleMaterializeTime.update(stats.sampleMaterializeTimeCounter().get());
updateIndexesStats(stats.getIndexesStats());
}
@@ -217,6 +243,9 @@
output.writeLong(level.get());
output.writeLong(bytesRead.get());
output.writeLong(bytesWritten.get());
+ output.writeLong(sampleLivenessTime.get());
+ output.writeLong(samplePkSeekTime.get());
+ output.writeLong(sampleMaterializeTime.get());
writeIndexesStats(output);
}
@@ -237,6 +266,9 @@
level.set(input.readLong());
bytesRead.set(input.readLong());
bytesWritten.set(input.readLong());
+ sampleLivenessTime.set(input.readLong());
+ samplePkSeekTime.set(input.readLong());
+ sampleMaterializeTime.set(input.readLong());
readIndexesStats(input);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/DiskBTreeSampleCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/DiskBTreeSampleCursor.java
index f41f65b..6656328 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/DiskBTreeSampleCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/DiskBTreeSampleCursor.java
@@ -221,37 +221,21 @@
/**
* Advances to the next accepted sample.
* <p>
- * <b>Why the emitted sample is an exactly uniform random sample of the component's live tuples.</b> One draw
- * picks a leaf page uniformly ({@code 1/P}), accepts it with the Olken & Rotem fill correction
- * ({@code tupleCount/leafTupleCapacity}, see {@link #pinAndAcceptLeafPage}), then picks one of that page's
- * {@code n} slots uniformly. Any given tuple is therefore reached by a given draw with probability
- * {@code (1/P) × (n/C) × (1/n) = 1/(P × C)}, independent of the page's fill — cancelling {@code n} is precisely
- * what the fill correction is for. Antimatter and shadowing by a newer component are deterministic per-tuple
- * predicates and already-emitted tuples are rejected, so the draw sequence is i.i.d. draws of "nothing, or a
- * tuple chosen uniformly from the component", filtered to live tuples and de-duplicated: uniform sampling
- * without replacement.
+ * <b>Exactly uniform over the component's live tuples.</b> A draw picks a leaf page uniformly ({@code 1/P}),
+ * accepts it with the Olken & Rotem fill correction ({@code n/C}, see {@link #pinAndAcceptLeafPage}), then
+ * picks one of that page's {@code n} slots uniformly, so a tuple is reached with probability
+ * {@code (1/P) × (n/C) × (1/n) = 1/(P × C)} — independent of fill, which is what the correction is for.
+ * Antimatter and shadowing are deterministic per-tuple predicates and emitted tuples are rejected. Both stop
+ * conditions read counts only; the one rule that does not is truncating a pageId-sorted batch, which keeps
+ * exactly the lowest pageIds, so batches are sized to be consumed in full and both conditions are tested only
+ * at a boundary (see {@link #refillLeafDrawBatch()}). No {@code samplesPerPage} here, so a batch cannot
+ * overshoot and no downsampling is needed.
* <p>
- * That per-draw law is invariant under relabelling the live tuples, and both stop conditions — the target and
- * the attempts cap — are functions of counts only, so they commute with relabelling and every subset of the
- * collected size stays equally likely. The one rule that is <em>not</em> a function of counts is truncating a
- * pageId-sorted draw batch, which keeps exactly the lowest pageIds; batches are therefore sized so that they
- * are always consumed in full and both stop conditions are tested only at a batch boundary (see
- * {@link #refillLeafDrawBatch()}). Unlike the column cursor there is no {@code samplesPerPage} here: a draw
- * yields at most one tuple, so a batch cannot overshoot the target and no downsampling step is needed.
- * <p>
- * <b>Caller contract: the consumer must drain the cursor.</b> This cursor emits lazily — one sample per
- * {@code hasNext()} — so "the batch is consumed in full" holds only if the consumer keeps calling until
- * {@code hasNext()} returns {@code false}. A consumer that stops early keeps exactly the low-pageId prefix of
- * the batch still in flight, which is the same spatial filter described above, merely relocated to the caller.
- * The batch is at most the outstanding shortfall (formerly a fixed 32768 draws), so the exposure is far smaller
- * than it was, but it is not zero. The only production driver is
- * {@code org.apache.hyracks.storage.am.lsm.btree.impls.LSMIndexSampleCursor}, which drains fully; any new
- * consumer that wants a partial sample must ask for a smaller {@code componentSampleCardinality} rather than
- * abandoning a larger one mid-scan.
- * <p>
- * The attempts cap is now also tested only at a batch boundary, so it can be overshot by up to one batch of
- * draws. It is a count-based rule either way, so that costs a bounded amount of extra work and cannot affect
- * uniformity.
+ * <b>Caller contract: the consumer must drain the cursor.</b> Emission is lazy, so "consumed in full" holds
+ * only while the consumer keeps calling. Stopping early keeps the low-pageId prefix of the in-flight batch —
+ * the same spatial filter, relocated to the caller. The only production driver,
+ * {@code LSMIndexSampleCursor}, drains fully; a consumer wanting a partial sample must ask for a smaller
+ * {@code componentSampleCardinality} rather than abandon a larger one.
*/
@Override
@AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_4_8, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.REFACTORED, notes = "Consume each pageId-sorted draw batch in full and evaluate the stop conditions only at batch "
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ISampleCursorStatsReceiver.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ISampleCursorStatsReceiver.java
new file mode 100644
index 0000000..a8a31eb
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ISampleCursorStatsReceiver.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.common.api;
+
+import org.apache.hyracks.util.annotations.AiProvenance;
+
+/**
+ * A per-component sample cursor that records a phase cost breakdown into a shared {@link SampleCursorStats}.
+ * <p>
+ * Exists so the LSM-level sample cursor can hand its stats object down to whichever per-component cursors accept
+ * one without depending on their module: the columnar cursor implements this, the row cursor does not, and the
+ * LSM cursor lives in a module that cannot see either. Instrumenting a cursor is therefore opt-in — a cursor that
+ * does not implement this simply contributes nothing, and the phase totals stay at zero rather than being wrong.
+ */
+@AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_5, tool = AiProvenance.Tool.CLAUDE_CODE_CLI, contributionKind = AiProvenance.ContributionKind.GENERATED, notes = "Lets the LSM sample cursor push a stats object into instrumented per-component cursors across modules")
+public interface ISampleCursorStatsReceiver {
+
+ /**
+ * Directs this cursor's phase timings into {@code stats}. Called before the cursor is opened; the stats object
+ * is shared across every component cursor of one sampling pass, so the totals are the pass's, not one
+ * component's.
+ *
+ * @param stats the collector to accumulate into
+ */
+ void setStats(SampleCursorStats stats);
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/SampleCursorStats.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/SampleCursorStats.java
similarity index 90%
rename from hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/SampleCursorStats.java
rename to hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/SampleCursorStats.java
index 106f5c5..76f69b4 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/SampleCursorStats.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/SampleCursorStats.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.storage.am.lsm.btree.column.impls.btree;
+package org.apache.hyracks.storage.am.common.api;
import org.apache.hyracks.util.annotations.AiProvenance;
@@ -47,7 +47,6 @@
* <em>draws</em>: a batch is sorted by pageId, so the draws that landed on one page are consumed as a single
* group sharing one pass. {@code attempts / pageGroups} is therefore the amortization factor achieved.
*/
- @AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_4_8, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.GENERATED, notes = "Counter for phase-1 page groups, so the per-group PK-pass amortization is measurable")
public long pageGroups;
public void reset() {
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeExistencePointSearchCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeExistencePointSearchCursor.java
index 9775f3e..f73f039 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeExistencePointSearchCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBTreeExistencePointSearchCursor.java
@@ -28,51 +28,30 @@
/**
* A column point-search cursor that answers <b>existence only</b>: after {@code open}, {@code hasNext()} reports
- * whether the search key is physically present on the leaf page — including as an antimatter (delete) entry — and
- * nothing is positioned for reading. It exists for the column sample cursor's newer-component liveness probe
- * ({@code LSMBTreePointSearchCursor#keyExistsIncludingAntimatter}), which is the only consumer that asks the
- * existence question and never reads the tuple. It must never be handed to a query path; see
+ * whether the search key is physically present on the leaf page — antimatter included — and nothing is positioned
+ * for reading. Sole consumer is the column sample cursor's newer-component liveness probe
+ * ({@code LSMBTreePointSearchCursor#keyExistsIncludingAntimatter}). It must never reach a query path; see
* {@code LSMColumnBTree#createSampleLivenessSearchCursor}, the only place it is wired.
* <p>
- * <b>Why the answer is identical to {@link ColumnBTreePointSearchCursor}'s, key for key.</b> The base cursor
- * decides existence in {@code ColumnBTreeRangeSearchCursor#setCursorPosition()} as follows:
- * <ol>
- * <li>{@code start = getLowKeyIndex()}, which is
- * {@code frameTuple.findTupleIndex(lowKey, cmp, EXACT, NONE)} mapped to {@code frame.getTupleCount()} when that
- * returns a negative indicator. {@code FindTupleMode.EXACT} makes the result {@code >= 0} <em>iff</em> the key
- * matched exactly.</li>
- * <li>{@code end = getHighKeyIndex()}, and because the point cursor's {@code setSearchPredicate} leaves
- * {@code highKey} null, that is always {@code frame.getTupleCount() - 1}.</li>
- * <li>No match ⇒ {@code start = tupleCount > end} ⇒ {@code frameTuple.consume()} and {@code yieldFirstCall}
- * stays {@code false} ⇒ absent.</li>
- * <li>Match ⇒ {@code start <= end} ⇒ {@code frameTuple.reset(start, end)} (which pins and resets every projected
- * column, then {@code shouldYieldFirstCall()} re-compares the key against the tuple now positioned at
- * {@code start} — the very index the exact match was found at, so the comparison is {@code 0} by construction
- * and {@code yieldFirstCall} is unconditionally {@code true} ⇒ present.</li>
- * </ol>
- * So the verdict is a pure function of {@code getLowKeyIndex()}: the materialization in step 4 cannot change it,
- * it can only cost. This cursor computes the same index and derives the same verdict, skipping
- * {@code frameTuple.reset(...)} entirely. {@code findTupleIndex} random-accesses primary-key values via
- * {@code setKeyAt(mid)} → {@code reader.getValue(index)}, so it touches no definition levels and pins no column
- * page; combined with the key-only projection supplied by
+ * <b>The verdict is identical to {@link ColumnBTreePointSearchCursor}'s, key for key</b>, because the base cursor
+ * decides existence purely from {@code getLowKeyIndex()} — {@code findTupleIndex(lowKey, cmp, EXACT, NONE)}, which
+ * is {@code >= 0} iff the key matched exactly. On a match the base cursor then calls {@code frameTuple.reset(...)},
+ * pinning every projected column, and re-compares at the index the match was found at, so that comparison is
+ * {@code 0} by construction. The materialization cannot change the verdict, only its cost. This cursor computes the
+ * same index and skips the reset. {@code findTupleIndex} reaches PK values through {@code setKeyAt(mid)}, touching
+ * no definition levels, so with the key-only projection from
* {@code IColumnProjectionInfo#createExistenceOnlyProjectionInfo()} the probe reads page zero and nothing else.
* <p>
- * <b>Antimatter still counts as present.</b> Primary-key values are written for antimatter tuples too — that is
- * what lets the base cursor position on them — so {@code findTupleIndex} matches a delete entry exactly as it
- * matches an insert, and this cursor reports it present. That is required: a newer delete shadows the older live
- * tuple, and reporting it absent would leak logically-deleted rows into the sample.
+ * <b>Antimatter counts as present.</b> PK values are written for delete entries too, so {@code findTupleIndex}
+ * matches them exactly as it matches inserts. Required: a newer delete shadows the older live tuple, and reporting
+ * it absent would leak logically-deleted rows into the sample.
* <p>
- * <b>The stateful same-page fast path is disabled, deliberately.</b> {@link #getLastPageId()} always returns
- * {@link IBufferCache#INVALID_PAGEID}, so {@code DiskBTree#search} always descends from the root and never calls
- * {@code setCursorToNextKey}. This is not a behaviour change: the liveness probe closes the cursor after
- * <em>every</em> search ({@code keyExistsIncludingAntimatter} closes in a {@code finally}), and
- * {@link ColumnBTreePointSearchCursor#doClose()} already invalidates {@code pageId}, so the fast path was
- * unreachable on this path to begin with. Returning the invalid id makes that an enforced invariant rather than
- * an assumed one — necessary here, because without a positioned {@code frameTuple} the cursor cannot carry the
- * monotonic {@code tupleIndex} that the reused-page search relies on, and the sampler's probe keys are not
- * globally ascending anyway.
- */
-@AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_4_8, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.GENERATED, notes = "Existence-only column point-search cursor for the sample cursor's newer-component liveness probe")
+ * <b>{@link #getLastPageId()} always returns {@link IBufferCache#INVALID_PAGEID}</b>, disabling the same-page fast
+ * path. Not a behaviour change — the probe closes the cursor after every search and
+ * {@link ColumnBTreePointSearchCursor#doClose()} already invalidates {@code pageId} — but it makes that an enforced
+ * invariant, which matters because without a positioned {@code frameTuple} this cursor cannot carry the monotonic
+ * {@code tupleIndex} the reused-page search relies on.
+ */@AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_4_8, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.GENERATED, notes = "Existence-only column point-search cursor for the sample cursor's newer-component liveness probe")
public final class ColumnBTreeExistencePointSearchCursor extends ColumnBTreePointSearchCursor {
public ColumnBTreeExistencePointSearchCursor(ColumnBTreeReadLeafFrame frame, IIndexCursorStats stats, int index,
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBtreeSampleCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBtreeSampleCursor.java
index e87048a..96995ab 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBtreeSampleCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBtreeSampleCursor.java
@@ -30,7 +30,9 @@
import org.apache.hyracks.storage.am.btree.impls.BTreeOpContext;
import org.apache.hyracks.storage.am.btree.impls.BatchPredicateWithKeys;
import org.apache.hyracks.storage.am.common.api.ILSMIndexBatchPointCursor;
+import org.apache.hyracks.storage.am.common.api.ISampleCursorStatsReceiver;
import org.apache.hyracks.storage.am.common.api.ITreeIndexCursor;
+import org.apache.hyracks.storage.am.common.api.SampleCursorStats;
import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnReadMultiPageOp;
import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnTupleIterator;
import org.apache.hyracks.storage.am.lsm.btree.column.cloud.buffercache.IColumnReadContext;
@@ -51,32 +53,20 @@
/**
* Two-phase column sample cursor.
* <p>
- * <b>Phase 1 — Selection (page0-only):</b> picks random leaf pages via batched
- * sorted draws to minimize random I/O, pins only page0, uses
- * {@link AbstractColumnTupleReference#startSamplingPage} +
- * {@link AbstractColumnTupleReference#seekForwardPKOnly} to position PKs without
- * loading column mega-pages. Checks antimatter and newer-component existence from PK data
- * alone. Accepted tuples are recorded as packed {@code (pageId << 32) | tupleIndex} longs.
+ * <b>Phase 1 — Selection (page0-only):</b> picks random leaf pages via batched sorted draws, pins only page0, and
+ * positions PKs with {@link AbstractColumnTupleReference#startSamplingPage} +
+ * {@link AbstractColumnTupleReference#seekForwardPKOnly} — no column mega-pages. Antimatter and newer-component
+ * existence are decided from PK data alone. Accepted tuples are packed as {@code (pageId << 32) | tupleIndex}.
+ * Same-page draws are contiguous in the sorted batch and are consumed as one <b>page group</b>: one page0 pin, one
+ * rewind, one forward pass over the PK level decoders. See {@link #runPhase1Selection()} for the uniformity
+ * argument this cursor rests on.
* <p>
- * Because the batch is sorted by pageId, all draws that landed on the same page are
- * <em>contiguous</em> in consumption order. Phase 1 therefore consumes them as one
- * <b>page group</b>: it pins page0 once, applies the per-draw fill-rejection test, draws each
- * surviving draw's slots, and probes the group's whole candidate set in a <b>single</b>
- * {@code startSamplingPage()} rewind plus one forward pass over the sequential PK level
- * decoders. Previously each draw paid its own rewind and its own walk from slot 0, so a page
- * that a batch hit {@code m} times was scanned {@code m} times; grouping makes that one scan.
- * All three sorts — the batch's, the group's candidate sort, and phase 2's — are applied only
- * to sets that are then consumed in full, which is what keeps them permutations rather than
- * filters; see {@link #runPhase1Selection()} for the uniformity argument this cursor rests on.
- * <p>
- * <b>Phase 2 — Collection (sorted, full column load):</b> the collected
- * samples are sorted by pageId (then tupleIndex within a page). For each
- * distinct page we do one full column load via {@code reset()}, then
- * advance to additional tuples on the same page via the cheap forward-only
- * {@code setAt()}. This amortises mega-page I/O and gives cache-friendly
- * sequential access.
+ * <b>Phase 2 — Collection (sorted, full column load):</b> samples are sorted by pageId then tupleIndex, so each
+ * distinct page takes one full column load via {@code reset()} and reaches its remaining tuples through the
+ * forward-only {@code setAt()}. This amortises mega-page I/O and keeps access sequential.
*/
-public class ColumnBtreeSampleCursor extends EnforcedIndexCursor implements ITreeIndexCursor, IColumnReadMultiPageOp {
+public class ColumnBtreeSampleCursor extends EnforcedIndexCursor
+ implements ITreeIndexCursor, IColumnReadMultiPageOp, ISampleCursorStatsReceiver {
private static final Logger LOGGER = LogManager.getLogger();
@@ -152,11 +142,9 @@
private long totalTimeTakenToFindRandomTuples = 0;
private boolean endedPreemptively = false;
- // Optional cost breakdown. Null in production scans (zero overhead); attached by the perf harness.
- @AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_4_8, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.GENERATED, notes = "New field to hold an optional attached SampleCursorStats for cost-breakdown instrumentation")
private SampleCursorStats stats;
- @AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_4_8, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.GENERATED, notes = "Accessor to attach a SampleCursorStats for cost-breakdown instrumentation")
+ @Override
public void setStats(SampleCursorStats stats) {
this.stats = stats;
}
@@ -182,8 +170,6 @@
private IBufferCache bufferCache;
private int fileId = -1;
- @AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_4_8, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.REFACTORED, notes = "Derive the draw-batch capacity from the sample target instead of max(configured, target), size the "
- + "per-page candidate buffer to samplesPerPage, and drop the never-read tupleStartSeed draw array")
public ColumnBtreeSampleCursor(ColumnBTree columnBTree, ColumnBTreeReadLeafFrame leafFrame,
BTreeOpContext opContext, IColumnReadContext context, long componentSampleCardinality, long sampleSeed,
int index, ILSMIndexBatchPointCursor searchCursor, int maxLeafFindingAttempts, int leafDrawBatchSize,
@@ -307,102 +293,41 @@
/**
* Collects up to {@code componentSampleCardinality} distinct live tuples from the component.
* <p>
- * <b>The exactness claim below holds at {@code samplesPerPage == 1} — the only value production can reach, since
- * {@code STORAGE_COLUMN_SAMPLES_PER_PAGE} is clamped to 1 when it is read into a job. At
- * {@code samplesPerPage > 1} the sample is <em>not</em> exactly uniform; that case is quantified at the end, is
- * why the clamp exists, and remains reachable only from tests that construct this cursor directly.</b>
- * <p>
- * <b>Page grouping does not enter this argument.</b> Draws are consumed a page group at a time
- * ({@link #visitNextPageGroup()}) rather than one at a time, so a page a batch hit {@code m} times is
- * scanned once instead of {@code m} times. That is purely a change to the <em>order in which candidates are
- * probed</em>: each draw is still an independent uniform page choice, with its own independent
- * fill-rejection test, and its own slots drawn without replacement from that page — the RNG is consumed in
- * exactly the same order and quantity as a per-draw loop. What makes the reordering invisible is that the
- * collected result is <b>order-insensitive</b>: de-duplication is set-based ({@link #seenTupleIndexes}),
- * {@link #collectedSamples} is sorted before phase 2 reads it, and the loop's stopping rules read only
- * counts, which grouping does not change. The invariant that keeps this true is that a group never crosses a
- * batch boundary and every draw in it is consumed — see {@link #visitNextPageGroup()}. So the argument below
- * is the same one as before grouping, unchanged.
- * <p>
- * <b>Why the result is an exactly uniform random sample of the component's live tuples when {@code k = 1}.</b>
- * One draw picks a leaf page uniformly ({@code 1/P}), accepts it with the Olken & Rotem fill correction
- * ({@code tupleCount/leafTupleCapacity}), and then probes {@code samplesPerPage} <em>distinct</em> slots drawn
- * uniformly without replacement from that page's {@code tupleCount} slots. Any given tuple of the component is
- * therefore probed by a given draw with probability
+ * <b>Exactly uniform at {@code samplesPerPage == 1}</b>, the only value production can reach
+ * ({@code STORAGE_COLUMN_SAMPLES_PER_PAGE} is clamped to 1 when read into a job). A draw picks a leaf page
+ * uniformly ({@code 1/P}), accepts it with the Olken & Rotem fill correction ({@code n/C} for page tuple
+ * count {@code n} and capacity {@code C}), then probes {@code k} distinct slots without replacement, so a
+ * given tuple is probed with probability
*
* <pre> (1/P) × (n/C) × (k/n) = k / (P × C)</pre>
*
- * for {@code n = } the page's tuple count, {@code C = leafTupleCapacity}, {@code k = samplesPerPage} — a value
- * that does not depend on {@code n}. Cancelling {@code n} is exactly what the Olken step buys: without it,
- * tuples on partly-filled pages would be over-represented, so removing that step would trade this bias for a
- * different one. Liveness (antimatter, shadowed by a newer component) is a deterministic per-tuple predicate and
- * already-collected tuples are rejected. So at {@code k = 1} each draw yields either nothing or a
- * <em>single</em> tuple drawn uniformly from the whole component, filtered to live tuples and de-duplicated —
- * textbook uniform sampling without replacement.
+ * independent of {@code n}. Cancelling {@code n} is what the Olken step buys: without it, tuples on
+ * partly-filled pages would be over-represented. Liveness is a deterministic per-tuple predicate and
+ * already-collected tuples are rejected, so at {@code k = 1} each draw yields at most one tuple drawn
+ * uniformly from the component.
* <p>
- * With a single tuple per draw, that per-draw law is invariant under any relabelling of the live tuples, and
- * every rule deciding when to stop — the target check, the attempts cap, the batch sizing — is a function of
- * <em>counts</em> only, never of which tuples happened to be collected. Rules like that commute with
- * relabelling, so conditional on how many tuples were collected, every subset of the live tuples of that size is
- * equally likely: a simple random sample without replacement. Truncating a pageId-sorted batch is the one thing
- * that is <em>not</em> a function of counts — it keeps precisely the lowest pageIds — so batches are consumed
- * whole (see {@link #refillLeafDrawBatch()}). At {@code k = 1} a batch cannot overshoot the target, so
- * {@link #downsampleToTarget()} never runs and there is nothing left to correct.
+ * Page grouping ({@link #visitNextPageGroup()}) only reorders which candidates are probed when: the RNG is
+ * consumed identically, and the result is order-insensitive because de-duplication is set-based, phase 2 sorts
+ * {@link #collectedSamples}, and every stopping rule reads counts. Batches are therefore consumed whole (see
+ * {@link #refillLeafDrawBatch()}) -- truncating a pageId-sorted batch would keep precisely the lowest pageIds,
+ * the one rule that is not count-based.
* <p>
- * <b>Why this argument does not generalise to {@code samplesPerPage > 1}, and why raising the knob is not
- * worth it.</b> At {@code k > 1} a draw yields a page-constrained <em>set</em> of up to {@code k} tuples rather
- * than one tuple, so the process law is no longer invariant under relabelling live tuples across pages and
- * count-based stopping no longer suffices. Two distinct consequences, of very different sizes:
- * <ol>
- * <li><b>The design becomes a cluster sample, and this is the dominant defect.</b> Tuples taken within one
- * visit come from one page, so a whole page's worth of the sample is drawn from one contiguous stretch of the
- * key space. Raising {@code k} buys its speed precisely by reducing the number of page visits, and the number
- * of page visits <em>is</em> the number of clusters, so speed and clustering are the same knob. Measured on
- * {@code ColumnSampleUniformityTest}'s fixture (165 000 keys, 55 % shadowed, 51 leaf pages, target 1063, 40
- * seeds pooled into 20 bins over the live key range, critical value 43.82): chi-square rises 11.6 → 39.4 →
- * 110 → 337 → 1388 → 1962 for {@code k} = 1, 3, 10, 30, 100, 300, while the distinct leaf pages the sample is
- * built from collapse from 24 to under 4. A sample assembled from 4 pages is spatially clustered in
- * primary-key order, which is silently wrong for any column correlated with the key.</li>
- * <li><b>Individual inclusion probabilities also stop being exactly equal — but this part is a rounding error
- * next to (1).</b> A batch is sized in whole page visits, so its last visit can carry the collection past the
- * target, and {@link #downsampleToTarget()} then retains each collected tuple with probability
- * {@code target/m}. {@code m} correlates with the live density of the pages that batch happened to visit, so
- * the retention rate does not cancel: a tuple on a page with few live tuples (small {@code m}) survives more
- * often than one on a densely-live page. Concretely, with two fully-packed pages — page A holding 10 live
- * tuples, page B holding 1 live tuple plus 9 antimatter — {@code k = 3} and {@code target = 1}, the live tuple
- * on B is 3× as likely to be sampled as any tuple on A. The relative bias is {@code O((k-1)/target)}:
- * identically zero at {@code k = 1}, ≈0.2 % at {@code k = 3, target = 1063}. Removing it entirely was measured
- * and does <em>not</em> help: with the downsample disabled the chi-square figures above become 39.7 / 109 /
- * 342 / 1465 / 2251, i.e. unchanged. The non-uniformity at {@code k > 1} is (1), not (2).</li>
- * </ol>
+ * <b>At {@code k > 1} the sample becomes a cluster sample</b>, which is why the clamp exists. Tuples from one
+ * visit share a page, and raising {@code k} buys speed by reducing page visits -- so speed and clustering are
+ * the same knob. On {@code ColumnSampleUniformityTest}'s fixture, chi-square rises 11.6 / 39.4 / 110 / 337 /
+ * 1388 / 1962 for {@code k} = 1 / 3 / 10 / 30 / 100 / 300 against a critical value of 43.82, while the distinct
+ * leaf pages behind the sample collapse from 24 to under 4. Unequal inclusion probabilities from
+ * {@link #downsampleToTarget()} also appear, at {@code O((k-1)/target)}, but disabling the downsample leaves
+ * the chi-square figures unchanged -- clustering is the defect, not the retention rate.
* <p>
- * <b>Consequently {@code samplesPerPage} is pinned at 1 in production.</b> The per-page amortization it used to be
- * the only way to get — one rewind and one forward PK pass serving many candidates — is now unconditional, obtained
- * by grouping a batch's same-page draws (see the class javadoc). Raising {@code k} on top of that buys little speed
- * and costs the properties above, so {@code StorageProperties#getColumnSamplesPerPage()} now <em>clamps</em> the
- * configured {@code STORAGE_COLUMN_SAMPLES_PER_PAGE} to 1 (warning once if a larger value was configured). This
- * cursor still honours whatever it is handed: the clamp deliberately lives at the configuration boundary rather
- * than here, so that the {@code k > 1} code paths stay reachable — and stay tested and documented — from
- * {@code ColumnSampleUniformityTest}, which drives this cursor directly. Treat {@code k > 1} as a property of the
- * algorithm recorded for the record, not as a configuration a deployment can reach.
+ * The clamp lives at the configuration boundary rather than here so the {@code k > 1} paths stay reachable from
+ * {@code ColumnSampleUniformityTest}, which drives this cursor directly.
* <p>
- * Two further caveats, independent of {@code k}:
- * <ol>
- * <li>If {@code samplesPerPage} exceeded a page's tuple count, only that many slots could be probed and the
- * page's tuples would be under-weighted ({@code n/C} rather than {@code k/C}). Leaf tuple capacities are orders
- * of magnitude above any configured {@code samplesPerPage}, so this does not arise in practice.</li>
- * <li>If the component carries no {@code maxLeafTupleCount} metadata ({@code leafTupleCapacity == 0}) the fill
- * correction cannot be applied and every page is accepted, which re-weights tuples by {@code 1/n}. That is a
- * property of the missing metadata, not of the draw scheme.</li>
- * </ol>
- * <p>
- * Note that the attempts cap is now also tested only at a batch boundary, so it can be overshot by up to one
- * batch of draws. That is a count-based rule either way, so it costs a bounded amount of extra work and cannot
- * affect uniformity.
+ * Two cases independent of {@code k}: a {@code samplesPerPage} above a page's tuple count would under-weight
+ * that page ({@code n/C} rather than {@code k/C}), which capacities make unreachable in practice; and a
+ * component without {@code maxLeafTupleCount} metadata ({@code leafTupleCapacity == 0}) accepts every page,
+ * re-weighting by {@code 1/n}.
*/
- @AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_4_8, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.ASSISTED, notes = "Added attempts/pagesAccepted/pagesRejected counters (guarded by stats != null) for cost-breakdown instrumentation")
- @AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_4_8, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.REFACTORED, notes = "Consume each pageId-sorted draw batch in full and evaluate the stop conditions only at batch "
- + "boundaries, so the sort can no longer act as a low-pageId filter; downsample any overshoot uniformly")
private void runPhase1Selection() throws HyracksDataException {
while (true) {
if (pendingLeafDrawIndex >= currentBatchSize) {
@@ -439,33 +364,21 @@
}
/**
- * Consumes the whole run of draws that landed on one page — a <b>page group</b> — with a single page0 pin, a
- * single {@code startSamplingPage()} rewind and a single forward pass over the sequential PK level decoders.
+ * A group is the maximal run of equal pageIds starting at {@link #pendingLeafDrawIndex}; the sorted batch makes
+ * same-page draws contiguous.
* <p>
- * The batch is sorted by pageId, so same-page draws are contiguous and a group is simply the maximal run of
- * equal pageIds starting at {@link #pendingLeafDrawIndex}. Before grouping, each of those {@code m} draws
- * rewound the decoders and walked from slot 0 to its own slot, so a page hit {@code m} times in one batch was
- * scanned {@code m} times; now it is scanned once, to the group's highest candidate slot.
- * <p>
- * <b>Two invariants make this a pure reordering rather than a change of sampling design</b> (see
- * {@link #runPhase1Selection()} for why that is what matters):
+ * Two invariants keep this a pure reordering rather than a different sampling design:
* <ol>
- * <li><b>A group never crosses a batch boundary</b> — it is bounded by {@link #currentBatchSize} — and every
- * draw in it is consumed. So the set of legal stopping points is unchanged: still only batch boundaries.</li>
- * <li><b>The RNG is consumed in exactly the same order and quantity as the per-draw loop.</b> The batch's page
- * and acceptance-sample draws already happened in {@link #refillLeafDrawBatch()}; within a group the draws are
- * visited in ascending draw index — the order the sorted batch would have visited them anyway — and each
- * accepted draw pulls its slots at that point. Rejected draws pull none, exactly as before. This is what makes
- * the emitted sample byte-identical to the pre-grouping cursor at {@code samplesPerPage == 1}.</li>
+ * <li>A group never crosses a batch boundary (bounded by {@link #currentBatchSize}) and every draw in it is
+ * consumed, so the legal stopping points are unchanged.</li>
+ * <li>The RNG is consumed in the same order and quantity as a per-draw loop — page and acceptance draws
+ * already happened in {@link #refillLeafDrawBatch()}, and within a group draws are visited in ascending draw
+ * index, each accepted one pulling its slots there. The emitted sample is byte-identical to the pre-grouping
+ * cursor at {@code samplesPerPage == 1}.</li>
* </ol>
- * The per-draw give-up accounting ({@link #hasNextAttemptCount}) is likewise reproduced exactly rather than
- * approximated: the draw-to-slot mapping is one-to-many but known, so each draw's outcome is recovered after
- * the pass and replayed in draw order. Replaying at the end of the group is equivalent to interleaving,
- * because nothing between the group's first and last draw reads that counter — it is tested only at a batch
- * boundary.
+ * {@link #hasNextAttemptCount} is recovered after the pass and replayed in draw order, which is equivalent to
+ * interleaving because nothing within a group reads it.
*/
- @AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_4_8, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.REFACTORED, notes = "Consume a whole run of same-page draws as one page group: one pin, one startSamplingPage rewind "
- + "and one forward PK pass, replacing the per-draw rewind-and-walk-from-slot-0")
private void visitNextPageGroup() throws HyracksDataException {
int groupStart = pendingLeafDrawIndex;
// Sort key packs (pageId << 32 | drawIndex): high 32 bits ARE the pageId (no separate array),
@@ -516,28 +429,20 @@
* Draws the next batch of leaf-page visits and sorts it by pageId so the page0 pins that follow proceed in
* ascending pageId order.
* <p>
- * <b>Why the batch is sized to the outstanding shortfall.</b> Sorting a batch of random draws is only a
- * permutation of that batch, and a permutation cannot change which tuples the batch produces — <em>provided
- * every draw in it is consumed</em>. Truncate a batch that was sorted globally by pageId and the surviving
- * draws are, by construction, the ones with the lowest pageIds: the sort has become a spatial filter, and the
- * sample degenerates into a contiguous run of the key space. A fixed batch of {@code leafDrawBatchSize} draws
- * (32768 by default) guaranteed exactly that, because the target was met within a small prefix of it and the
- * rest was discarded.
+ * <b>Sized to the outstanding shortfall, because truncating a sorted batch is a spatial filter.</b> A sort is
+ * only a permutation, and cannot change which tuples a batch produces — provided every draw is consumed.
+ * Truncate a pageId-sorted batch and the survivors are precisely the lowest pageIds, degenerating the sample
+ * into a contiguous run of the key space. A fixed {@code leafDrawBatchSize} guaranteed that, since the target
+ * was met in a small prefix and the rest discarded.
* <p>
- * Sizing the batch to {@code ceil(remaining / samplesPerPage)} visits makes truncation unnecessary rather than
- * merely rare: one visit yields at most {@code samplesPerPage} tuples, so a batch cannot yield more than
- * {@code remaining + samplesPerPage - 1} and the target cannot be reached strictly before the batch's last
- * draw. The caller therefore consumes every draw and re-tests the target only here, at the boundary. Batches
- * are refilled — each smaller than the last — until the target is met or the attempts cap trips; the total
- * number of draws is unchanged, so this costs nothing beyond the shrinking sort windows.
+ * At {@code ceil(remaining / samplesPerPage)} visits a batch yields at most
+ * {@code remaining + samplesPerPage - 1}, so the target cannot be reached before its last draw and truncation
+ * never arises. Batches are refilled, each smaller than the last, until the target is met or the attempts cap
+ * trips; the total draw count is unchanged.
* <p>
- * A consequence worth knowing: because the size is {@code min(configured, ceil(remaining/k))} and the right-hand
- * term is the smaller one for any realistic target, the configured {@code leafDrawBatchSize}
- * ({@code STORAGE_SAMPLE_LEAF_DRAW_BATCH_SIZE}, default 32768) no longer binds in practice — it does so only if it
- * is set <em>below</em> the target. It is kept because it remains the ceiling on this cursor's draw-buffer
- * allocation ({@link #drawAcceptanceSamples}, {@link #drawSortKeys} and {@link #drawCollectedAny} are sized to it,
- * not to an arbitrarily large requested sample), and because a smaller batch is always still consumed in full and
- * so cannot change which tuples the sample contains — only the locality of the page-zero pins.
+ * Consequently {@code STORAGE_SAMPLE_LEAF_DRAW_BATCH_SIZE} (default 32768) no longer binds unless set below the
+ * target. It is kept as the ceiling on draw-buffer allocation ({@link #drawAcceptanceSamples},
+ * {@link #drawSortKeys}, {@link #drawCollectedAny}).
* <p>
* Sets {@link #currentBatchSize} to 0 when nothing more is needed (target met) or possible (no leaf pages).
*/
@@ -567,29 +472,15 @@
}
/**
- * Reduces an overshoot past the target to exactly the target by keeping a uniformly random subset, via a
- * partial Fisher-Yates shuffle of the collected keys.
+ * Keeps a uniformly random subset via a partial Fisher-Yates shuffle. <b>Dead at the production default
+ * {@code samplesPerPage == 1}</b>, where a batch is sized in single tuples and cannot overshoot.
* <p>
- * <b>This is dead code at the production default of {@code samplesPerPage == 1}</b>: a batch is then sized in
- * single tuples, so it cannot overshoot and this method returns immediately. It runs only when
- * {@code samplesPerPage > 1}, where a batch is sized in whole page visits and its last visit can add up to
- * {@code samplesPerPage - 1} tuples beyond the target.
- * <p>
- * Choosing the retained subset at random is strictly better than dropping the collected array's tail — that
- * array is filled page group by page group in ascending pageId order within each batch, so a tail-drop would
- * reintroduce exactly the spatial filter this cursor exists to avoid. But it is <b>not</b> a bias-free
- * correction, and it must not be read as one: the identity "a uniformly random {@code target}-subset of a
- * uniformly random {@code m}-subset is uniform" requires the collected set to be uniform over {@code m}-subsets
- * <em>conditional on {@code m}</em>, which holds at {@code k = 1} (where this method never runs) but not at
- * {@code k > 1}. At {@code k > 1} the retention rate {@code target/m} is correlated with the live density of the
- * pages the final batch visited, so it does not cancel and a residual bias of relative order
- * {@code O((k-1)/target)} remains.
- * <p>
- * That residual is, however, the <em>smaller</em> of the two things wrong with {@code k > 1}: removing it
- * entirely was measured and leaves the sample's clustering — and hence its measured non-uniformity over the key
- * space — essentially unchanged. See {@link #runPhase1Selection()} for the worked counterexample, the measured
- * figures, and why {@code samplesPerPage} should simply stay at 1 now that per-page amortization no longer
- * depends on it.
+ * Random retention beats dropping the array's tail, which is filled in ascending pageId order and would
+ * reintroduce the spatial filter this cursor exists to avoid. It is <b>not</b> bias-free: the "uniform subset
+ * of a uniform subset" identity needs the collected set to be uniform over {@code m}-subsets conditional on
+ * {@code m}, which holds only at {@code k = 1}, where this never runs. Residual bias is
+ * {@code O((k-1)/target)} and is the smaller of the two defects at {@code k > 1} — see
+ * {@link #runPhase1Selection()}.
*/
@AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_4_8, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.GENERATED, notes = "Uniform random downsample of the phase-1 overshoot that full-batch consumption can produce")
private void downsampleToTarget() {
@@ -648,25 +539,19 @@
* draw that survives it, draws that draw's {@code min(samplesPerPage, tupleCount)} candidate slots — packing
* them into {@link #groupCandidates} as {@code (tupleIndex << 32) | drawOrdinal}.
* <p>
- * <b>Each draw's slots are drawn without replacement from that draw's own set, and every candidate is later
- * probed.</b> Both properties are load bearing. Drawing without replacement makes a tuple's probability of
- * being probed by one draw exactly {@code samplesPerPage / tupleCount}, which is what cancels the page's fill
- * against the Olken acceptance probability in {@link #runPhase1Selection()}; drawing with replacement and
- * de-duplicating would instead give {@code 1 - (1 - 1/tupleCount)^samplesPerPage}, a fill-dependent value, and
- * so would re-introduce a cross-page bias. Note the rejection is deliberately against the <em>same draw's</em>
- * slots only, not against the whole group's: two distinct draws are independent trials and are allowed to
- * collide, exactly as they were when each was probed on its own page visit.
+ * Two load-bearing properties. Slots are drawn <b>without replacement within a draw</b>, making a tuple's
+ * per-draw probe probability exactly {@code samplesPerPage / tupleCount} — the term that cancels the page fill
+ * against the Olken acceptance probability in {@link #runPhase1Selection()}. With replacement plus
+ * de-duplication it would be {@code 1 - (1 - 1/tupleCount)^samplesPerPage}, fill-dependent, reintroducing
+ * cross-page bias. Rejection is against the same draw's slots only; distinct draws are independent trials and
+ * may collide.
* <p>
- * Probing all of them (in {@link #probeGroupCandidates(int, int)}) is the intra-page form of the batch rule:
- * the candidates are sorted ascending for the forward scan, so stopping at the first acceptable one — as an
- * oversampled "{@code 5 × samplesPerPage} candidates, keep the first {@code samplesPerPage}" scheme necessarily
- * does — would select the <em>minimum</em> of several uniform draws and pull every page's contribution towards
- * its low slots.
+ * And <b>every candidate is probed</b>. Candidates are sorted ascending for the forward scan, so stopping at
+ * the first acceptable one would select the minimum of several uniform draws and pull each page's contribution
+ * towards its low slots.
*
* @return the number of candidate slots written to {@link #groupCandidates}
*/
- @AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_4_8, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.REFACTORED, notes = "Split out of collectFromPage: draws the candidate slots for every draw of a page group in draw "
- + "order (preserving RNG consumption exactly) instead of one draw at a time")
private int drawGroupCandidates(int groupStart, int groupEnd, int tupleCount) {
int drawCount = Math.min(samplesPerPage, tupleCount);
ensureGroupCandidateCapacity((long) (groupEnd - groupStart) * drawCount);
@@ -718,27 +603,14 @@
}
/**
- * Probes the group's pre-drawn, slot-ascending candidates in a <b>single</b> forward-only pass over the PK
- * level decoders, collecting every one that turns out to be live.
+ * The PK definition-level decoder is sequential and cannot seek backwards, hence the ascending candidate order
+ * and the single {@link AbstractColumnTupleReference#startSamplingPage()} rewind.
* <p>
- * The PK definition-level decoder is sequential and cannot seek backwards, hence the ascending order and the
- * single {@link AbstractColumnTupleReference#startSamplingPage()} rewind at the top. This scans the page once
- * ({@code O(maxCandidateIndex)} level reads) for the whole group, where the pre-grouping code scanned it once
- * per draw. Antimatter, collision and newer-component rejections simply advance the forward scan.
- * <p>
- * <b>Repeated slots are not re-seeked.</b> Two draws of the group may pick the same slot, and
- * {@link AbstractColumnTupleReference#seekForwardPKOnly(int)} computes {@code startIndex - tupleIndex - 1}, so
- * feeding it the same index twice would pass a skip count of {@code -1} and desync the decoder. The reader is
- * already positioned there, so the repeat simply re-runs the liveness decision from the current position —
- * which reproduces the pre-grouping behaviour exactly, including that a repeat of an already-collected slot is
- * rejected by {@link #seenTupleIndexes} and that a repeat of a shadowed slot re-runs the newer-component
- * search.
+ * <b>Repeated slots are not re-seeked.</b> {@link AbstractColumnTupleReference#seekForwardPKOnly(int)} computes
+ * {@code startIndex - tupleIndex - 1}, so feeding the same index twice would pass a skip count of {@code -1}
+ * and desync the decoder. The reader is already positioned, so a repeat re-runs the liveness decision from
+ * there.
*/
- @AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_4_8, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.ASSISTED, notes = "Added pkSeekCalls/pkSeekNanos and livenessNanos/livenessCalls/livenessKeys timing "
- + "(guarded by stats != null) for cost-breakdown instrumentation; pkSeekNanos excludes "
- + "the liveness window so the two phases do not overlap")
- @AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_4_8, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.REFACTORED, notes = "Probe a whole page group's candidates in one rewind and one forward pass, recording each draw's "
- + "outcome so the per-draw give-up accounting is reproduced exactly")
private void probeGroupCandidates(int pageId, int candidateCount) throws HyracksDataException {
long nanos = (traceTimingEnabled || stats != null) ? System.nanoTime() : 0L;
// Liveness time (searchCursor.hasNextWithPredicate below) is measured separately into
@@ -812,7 +684,6 @@
* {@code groupSize × min(samplesPerPage, tupleCount)}, and {@code groupSize} cannot exceed the batch capacity,
* so this settles after the first few groups rather than growing without limit.
*/
- @AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_4_8, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.GENERATED, notes = "Amortized growth for the per-group candidate buffer")
private void ensureGroupCandidateCapacity(long needed) {
if (groupCandidates.length < needed) {
long grown = Math.max(needed, groupCandidates.length * 2L);
@@ -824,8 +695,6 @@
// Phase 2: Sorted column collection — one mega-page load per page
// ──────────────────────────────────────────────────────────────────────
- @AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_4_8, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.ASSISTED, notes = "Added phase2Nanos timing and phase2PagePins counter (guarded by stats != null) for "
- + "cost-breakdown instrumentation")
private boolean yieldNextFromPhase2() throws HyracksDataException {
if (yieldPos >= collectedCount) {
return false;
@@ -903,8 +772,6 @@
}
@Override
- @AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_4_8, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.ASSISTED, notes = "Reset the draw batch by emptying it (currentBatchSize = 0) rather than by parking the read "
- + "position past a fixed batch length, which no longer exists")
protected void doClose() throws HyracksDataException {
if (LOGGER.isTraceEnabled()) {
LOGGER.trace(
@@ -964,4 +831,4 @@
return bufferCache.getPageSize();
}
-}
\ No newline at end of file
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/BTreeSampleCollectorOperatorDescriptorNodePushable.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/BTreeSampleCollectorOperatorDescriptorNodePushable.java
index 2f7d3e6..0a56ee4 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/BTreeSampleCollectorOperatorDescriptorNodePushable.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/BTreeSampleCollectorOperatorDescriptorNodePushable.java
@@ -30,6 +30,7 @@
import org.apache.hyracks.storage.am.btree.dataflow.BTreeSearchOperatorNodePushable;
import org.apache.hyracks.storage.am.common.api.ISearchOperationCallbackFactory;
import org.apache.hyracks.storage.am.common.api.ITupleFilterFactory;
+import org.apache.hyracks.storage.am.common.api.SampleCursorStats;
import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTree;
import org.apache.hyracks.storage.am.lsm.btree.impls.LSMIndexSampleCursor;
@@ -43,6 +44,19 @@
public class BTreeSampleCollectorOperatorDescriptorNodePushable extends BTreeSearchOperatorNodePushable {
public static final String ESTIMATE_CARDINALITY = "ESTIMATE_CARDINALITY";
+
+ /**
+ * Same hand-off as {@link #ESTIMATE_CARDINALITY}: this operator's own {@code IOperatorStats} is a
+ * {@code NoOpOperatorStats} unless something sets it, and ANALYZE reads a different operator's stats.
+ */
+ public static final String SAMPLE_CURSOR_PHASES = "SAMPLE_CURSOR_PHASES";
+
+ /**
+ * Shared across every component cursor, so the totals describe the pass rather than one component. All-zero
+ * for cursor types that decline instrumentation -- today the row cursor -- rather than a partial figure that
+ * would read as complete.
+ */
+ private final SampleCursorStats sampleCursorStats = new SampleCursorStats();
private final int sampleCardinalityTargetPerPartition;
private final long sampleSeed;
private final int maxSampleLeafAttempts;
@@ -84,7 +98,9 @@
@Override
protected IIndexCursor createCursor(IIndex idx, IIndexAccessor idxAccessor) throws HyracksDataException {
ILSMIndexAccessor lsmAccessor = (ILSMIndexAccessor) idxAccessor;
- return ((LSMBTree) idx).createSampleCollectorCursor(lsmAccessor.getOpContext());
+ LSMIndexSampleCursor cursor = ((LSMBTree) idx).createSampleCollectorCursor(lsmAccessor.getOpContext());
+ cursor.setStats(sampleCursorStats);
+ return cursor;
}
@Override
@@ -128,6 +144,12 @@
((ILSMIndexAccessor) indexAccessors[pIdx]).scanDiskComponentsForSample(cursors[pIdx]);
writeSearchResults(i, cursors[pIdx]);
}
+ publishSampleCursorPhases();
+ }
+
+ /** Call once per sampling pass, after every component has been visited. */
+ private void publishSampleCursorPhases() {
+ TaskUtil.put(SAMPLE_CURSOR_PHASES, sampleCursorStats, ctx);
}
protected void searchAllPartitions(int tupleCount) throws Exception {
@@ -141,5 +163,6 @@
estimatedCardinality += ((LSMIndexSampleCursor) cursors[p]).getEstimatedCardinality();
}
TaskUtil.put(ESTIMATE_CARDINALITY, estimatedCardinality, ctx);
+ publishSampleCursorPhases();
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMIndexSampleCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMIndexSampleCursor.java
index aac5ba7..d687773 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMIndexSampleCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMIndexSampleCursor.java
@@ -29,7 +29,9 @@
import org.apache.hyracks.storage.am.btree.impls.DiskBTree;
import org.apache.hyracks.storage.am.common.api.ILSMIndexBatchPointCursor;
import org.apache.hyracks.storage.am.common.api.ILSMIndexCursor;
+import org.apache.hyracks.storage.am.common.api.ISampleCursorStatsReceiver;
import org.apache.hyracks.storage.am.common.api.ITreeIndexCursor;
+import org.apache.hyracks.storage.am.common.api.SampleCursorStats;
import org.apache.hyracks.storage.am.common.impls.NoOpIndexAccessParameters;
import org.apache.hyracks.storage.am.common.util.ResourceReleaseUtils;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
@@ -47,6 +49,8 @@
import org.apache.logging.log4j.Logger;
public class LSMIndexSampleCursor extends EnforcedIndexCursor implements ILSMIndexCursor {
+
+ private SampleCursorStats stats;
// IIndexAccessParameters keys for passing sampling knobs from the collector operator to this cursor.
// Sourced from StorageProperties at job build time (not persisted in the index resource).
public static final String SAMPLE_CARDINALITY = "SAMPLE_CARDINALITY";
@@ -163,9 +167,19 @@
private ITreeIndexCursor createCursor(DiskBTree.DiskBTreeAccessor bTreeAccessor, long componentSampleCardinality,
long sampleSeed, ILSMIndexBatchPointCursor searchCursor, int maxLeafTupleCount) {
- return bTreeAccessor.createSampleCursor(componentSampleCardinality, sampleSeed, searchCursor,
+ ITreeIndexCursor cursor = bTreeAccessor.createSampleCursor(componentSampleCardinality, sampleSeed, searchCursor,
maxSampleLeafAttempts, sampleLeafDrawBatchSize, maxLeafTupleCount, columnSamplesPerPage,
AntimatterAwareTupleAcceptor.INSTANCE);
+ if (stats != null && cursor instanceof ISampleCursorStatsReceiver) {
+ ((ISampleCursorStatsReceiver) cursor).setStats(stats);
+ }
+ return cursor;
+ }
+
+ /** Must be called before {@link #open}, which is where the component cursors are built. */
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_5, tool = AiProvenance.Tool.CLAUDE_CODE_CLI, contributionKind = AiProvenance.ContributionKind.GENERATED, notes = "Propagate a sampling phase-cost collector to instrumented per-component cursors")
+ public void setStats(SampleCursorStats stats) {
+ this.stats = stats;
}
@AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_4_8, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.REFACTORED, notes = "Split into logical units (collect stats, floor-allocate, distribute remainder, warn); "
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/pom.xml b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/pom.xml
index a7f6107..788ccf3 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/pom.xml
@@ -73,12 +73,6 @@
</dependency>
<dependency>
<groupId>org.apache.hyracks</groupId>
- <artifactId>hyracks-storage-am-lsm-btree-column</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hyracks</groupId>
<artifactId>hyracks-storage-am-common</artifactId>
<version>${project.version}</version>
<scope>test</scope>
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/SampleCursorStatsTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/SampleCursorStatsTest.java
index d8fb5d2..9a91d95 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/SampleCursorStatsTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/SampleCursorStatsTest.java
@@ -18,7 +18,7 @@
*/
package org.apache.hyracks.storage.am.lsm.btree;
-import org.apache.hyracks.storage.am.lsm.btree.column.impls.btree.SampleCursorStats;
+import org.apache.hyracks.storage.am.common.api.SampleCursorStats;
import org.apache.hyracks.util.annotations.AiProvenance;
import org.junit.Assert;
import org.junit.Test;
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21688?usp=email
To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings?usp=email
Gerrit-MessageType: newchange
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Change-Id: Ica90673813691c728d5c96b699b0363679af3813
Gerrit-Change-Number: 21688
Gerrit-PatchSet: 1
Gerrit-Owner: Ritik Raj <ri...@couchbase.com>