You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@asterixdb.apache.org by AsterixDB Code Review <do...@asterix-gerrit.ics.uci.edu> on 2026/07/15 15:12:49 UTC
[L] Change in asterixdb[master]: [ASTERIXDB-3702][STO] Propagate LSM sample-scan stats to query profile
From Ritik Raj <ri...@couchbase.com>:
Ritik Raj has uploaded this change for review. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21431?usp=email )
Change subject: [ASTERIXDB-3702][STO] Propagate LSM sample-scan stats to query profile
......................................................................
[ASTERIXDB-3702][STO] Propagate LSM sample-scan stats to query profile
The disk/column sample cursors tracked draw-attempt, pinned-page-reuse and
preemptive-end counters only in a debug log. Expose them via a new
ISampleCursorStats interface, aggregate them per component in
LSMIndexSampleCursor, and bridge them - like the existing ESTIMATE_CARDINALITY
value - through BTreeSampleCollectorOperatorDescriptorNodePushable into
DatasetStreamStats, so they surface as IOperatorStats counters in the query
profile and in the ANALYZED statement log. No change to the translator's
harvested-operator list.
Also drop the always-on isDebugEnabled per-batch unique-page counting scan in
DiskBTreeSampleCursor: it ran an O(batch) pass on every refill (debug is
enabled in some deployments) and was redundant with the pin/reuse counters.
Co-Authored-By: Claude Opus 4.8 (1M context) <no...@anthropic.com>
Change-Id: I1c5e0571b956f473525cd8e6cac2f210f39dea53
---
M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
M asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/DatasetStreamStats.java
M asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/DatasetStreamStatsOperatorDescriptor.java
M hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/profiling/IOperatorStats.java
M hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/profiling/NoOpOperatorStats.java
M hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/profiling/OperatorStats.java
M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/DiskBTreeSampleCursor.java
A hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ISampleCursorStats.java
M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBtreeSampleCursor.java
M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/BTreeSampleCollectorOperatorDescriptorNodePushable.java
M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMIndexSampleCursor.java
11 files changed, 224 insertions(+), 39 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/31/21431/1
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index 43b06d4..be90c1b 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -5278,8 +5278,11 @@
}
DatasetStreamStats stats = new DatasetStreamStats(opStats.get(0));
- LOGGER.info("ANALYZED statement stats: coldReads: {} -- pinnedPages: {} -- cloudPageReads:{}",
- stats.getColdReads(), stats.getPinnedPages(), stats.getCloudPageReads());
+ LOGGER.info(
+ "ANALYZED statement stats: coldReads: {} -- pinnedPages: {} -- cloudPageReads:{} -- "
+ + "sampleDraws: {} -- samplePinReuse: {} -- samplePreemptiveEnds: {}",
+ stats.getColdReads(), stats.getPinnedPages(), stats.getCloudPageReads(),
+ stats.getSampleAccessCount(), stats.getSamplePinReuseHits(), stats.getSamplePreemptiveEndCount());
Index.SampleIndexDetails newIndexDetailsFinal = new Index.SampleIndexDetails(dsDetails.getPrimaryKey(),
dsDetails.getKeySourceIndicator(), dsDetails.getPrimaryKeyType(), sampleCardinalityTarget,
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/DatasetStreamStats.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/DatasetStreamStats.java
index ac3cd9e..26ba8b3 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/DatasetStreamStats.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/DatasetStreamStats.java
@@ -37,6 +37,10 @@
private final long coldReads;
private final long cloudPageReads;
+ // Sample-scan profiling (ASTERIXDB-3702).
+ private final long sampleAccessCount;
+ private final long samplePinReuseHits;
+ private final long samplePreemptiveEndCount;
private final Map<String, IndexStats> indexesStats;
@@ -48,17 +52,24 @@
this.pinnedPages = opStats.getPageReadCounter().get();
this.coldReads = opStats.coldReadCounter().get();
this.cloudPageReads = opStats.cloudReadRequestCounter().get();
+ this.sampleAccessCount = opStats.getSampleAccessCounter().get();
+ this.samplePinReuseHits = opStats.getSamplePinReuseCounter().get();
+ this.samplePreemptiveEndCount = opStats.getSamplePreemptiveEndCounter().get();
this.indexesStats = opStats.getIndexesStats();
}
static void update(IOperatorStats opStats, long sampledTupleCount, long estimatedCardinality, long totalTupleLength,
- long pinnedPages, long coldReads, long cloudReadCount, Map<String, IndexStats> indexStats) {
+ long pinnedPages, long coldReads, long cloudReadCount, long sampleAccessCount, long samplePinReuseHits,
+ long samplePreemptiveEndCount, Map<String, IndexStats> indexStats) {
opStats.getTupleCounter().update(estimatedCardinality);
opStats.getInputTupleCounter().update(sampledTupleCount);
opStats.getTupleBytes().update(totalTupleLength);
opStats.cloudReadRequestCounter().update(cloudReadCount);
opStats.coldReadCounter().update(coldReads);
opStats.getPageReadCounter().update(pinnedPages);
+ opStats.getSampleAccessCounter().update(sampleAccessCount);
+ opStats.getSamplePinReuseCounter().update(samplePinReuseHits);
+ opStats.getSamplePreemptiveEndCounter().update(samplePreemptiveEndCount);
opStats.updateIndexesStats(indexStats);
}
@@ -82,6 +93,18 @@
return cloudPageReads;
}
+ public long getSampleAccessCount() {
+ return sampleAccessCount;
+ }
+
+ public long getSamplePinReuseHits() {
+ return samplePinReuseHits;
+ }
+
+ public long getSamplePreemptiveEndCount() {
+ return samplePreemptiveEndCount;
+ }
+
public Map<String, IndexStats> getIndexesStats() {
return indexesStats;
}
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/DatasetStreamStatsOperatorDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/DatasetStreamStatsOperatorDescriptor.java
index 699fd3b..7c4abb1 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/DatasetStreamStatsOperatorDescriptor.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/DatasetStreamStatsOperatorDescriptor.java
@@ -21,6 +21,9 @@
import static org.apache.hyracks.api.job.profiling.NoOpOperatorStats.INVALID_ODID;
import static org.apache.hyracks.storage.am.lsm.btree.dataflow.BTreeSampleCollectorOperatorDescriptorNodePushable.ESTIMATE_CARDINALITY;
+import static org.apache.hyracks.storage.am.lsm.btree.dataflow.BTreeSampleCollectorOperatorDescriptorNodePushable.SAMPLE_ACCESS_COUNT;
+import static org.apache.hyracks.storage.am.lsm.btree.dataflow.BTreeSampleCollectorOperatorDescriptorNodePushable.SAMPLE_PIN_REUSE_HITS;
+import static org.apache.hyracks.storage.am.lsm.btree.dataflow.BTreeSampleCollectorOperatorDescriptorNodePushable.SAMPLE_PREEMPTIVE_END_COUNT;
import java.nio.ByteBuffer;
import java.util.HashMap;
@@ -134,13 +137,22 @@
long cloudPageReadCount = ctx.getThreadStats().getCloudPageReadCount();
long coldReadCount = ctx.getThreadStats().getColdReadCount();
long pinnedPagesCount = ctx.getThreadStats().getPinnedPagesCount();
+ long sampleAccessCount = getSampleStat(SAMPLE_ACCESS_COUNT);
+ long samplePinReuseHits = getSampleStat(SAMPLE_PIN_REUSE_HITS);
+ long samplePreemptiveEndCount = getSampleStat(SAMPLE_PREEMPTIVE_END_COUNT);
DatasetStreamStats.update(stats, totalTupleCount, (long) totalNum, totalTupleLength,
- pinnedPagesCount, coldReadCount, cloudPageReadCount, indexesStats);
+ pinnedPagesCount, coldReadCount, cloudPageReadCount, sampleAccessCount,
+ samplePinReuseHits, samplePreemptiveEndCount, indexesStats);
}
}
writer.close();
}
+ private long getSampleStat(String key) {
+ Object v = TaskUtil.get(key, ctx);
+ return v == null ? 0L : (long) v;
+ }
+
@Override
public void flush() throws HyracksDataException {
writer.flush();
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/profiling/IOperatorStats.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/profiling/IOperatorStats.java
index 66d7dc0..0142a40 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/profiling/IOperatorStats.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/profiling/IOperatorStats.java
@@ -105,6 +105,15 @@
ICounter getBytesWritten();
+ /** Sample-scan draw attempts (ASTERIXDB-3702). */
+ ICounter getSampleAccessCounter();
+
+ /** Sample-scan batched-pin reuse hits (ASTERIXDB-3702). */
+ ICounter getSamplePinReuseCounter();
+
+ /** Sample-scan component cursors that ended before reaching their target (ASTERIXDB-3702). */
+ ICounter getSamplePreemptiveEndCounter();
+
String getOperatorId();
void updateIndexesStats(Map<String, IndexStats> indexesStats);
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/profiling/NoOpOperatorStats.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/profiling/NoOpOperatorStats.java
index ce46f47..8292745 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/profiling/NoOpOperatorStats.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/profiling/NoOpOperatorStats.java
@@ -155,6 +155,21 @@
}
@Override
+ public ICounter getSampleAccessCounter() {
+ return NOOP_COUNTER;
+ }
+
+ @Override
+ public ICounter getSamplePinReuseCounter() {
+ return NOOP_COUNTER;
+ }
+
+ @Override
+ public ICounter getSamplePreemptiveEndCounter() {
+ return NOOP_COUNTER;
+ }
+
+ @Override
public String getOperatorId() {
return INVALID_ODID;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/profiling/OperatorStats.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/profiling/OperatorStats.java
index 0ca44d1..42387fb 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/profiling/OperatorStats.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/profiling/OperatorStats.java
@@ -50,6 +50,9 @@
public final ICounter level;
public final ICounter bytesRead;
public final ICounter bytesWritten;
+ public final ICounter sampleAccessCounter;
+ public final ICounter samplePinReuseCounter;
+ public final ICounter samplePreemptiveEndCounter;
private final Map<String, IndexStats> indexesStats;
//TODO: this is quickly becoming gross it should just be a map where the value is obliged to be a Counter
@@ -75,6 +78,9 @@
level = new Counter("level");
bytesRead = new Counter("bytesRead");
bytesWritten = new Counter("bytesWritten");
+ sampleAccessCounter = new Counter("sampleAccessCounter");
+ samplePinReuseCounter = new Counter("samplePinReuseCounter");
+ samplePreemptiveEndCounter = new Counter("samplePreemptiveEndCounter");
level.set(-1);
indexesStats = new HashMap<>();
}
@@ -155,6 +161,21 @@
}
@Override
+ public ICounter getSampleAccessCounter() {
+ return sampleAccessCounter;
+ }
+
+ @Override
+ public ICounter getSamplePinReuseCounter() {
+ return samplePinReuseCounter;
+ }
+
+ @Override
+ public ICounter getSamplePreemptiveEndCounter() {
+ return samplePreemptiveEndCounter;
+ }
+
+ @Override
public ICounter getTotalFrameCount() {
return frameCounter;
}
@@ -195,6 +216,9 @@
tupleBytes.update(stats.getTupleBytes().get());
coldReadCounter.update(stats.coldReadCounter().get());
cloudReadRequestCounter.update(stats.cloudReadPageCounter().get());
+ sampleAccessCounter.update(stats.getSampleAccessCounter().get());
+ samplePinReuseCounter.update(stats.getSamplePinReuseCounter().get());
+ samplePreemptiveEndCounter.update(stats.getSamplePreemptiveEndCounter().get());
updateIndexesStats(stats.getIndexesStats());
}
@@ -217,6 +241,9 @@
output.writeLong(level.get());
output.writeLong(bytesRead.get());
output.writeLong(bytesWritten.get());
+ output.writeLong(sampleAccessCounter.get());
+ output.writeLong(samplePinReuseCounter.get());
+ output.writeLong(samplePreemptiveEndCounter.get());
writeIndexesStats(output);
}
@@ -237,6 +264,9 @@
level.set(input.readLong());
bytesRead.set(input.readLong());
bytesWritten.set(input.readLong());
+ sampleAccessCounter.set(input.readLong());
+ samplePinReuseCounter.set(input.readLong());
+ samplePreemptiveEndCounter.set(input.readLong());
readIndexesStats(input);
}
@@ -270,6 +300,9 @@
+ inputTupleCounter.getName() + "\": " + inputTupleCounter.get() + ", \"" + frameCounter.getName()
+ "\": " + frameCounter.get() + ", \"" + bytesRead.getName() + "\": " + bytesRead.get() + ", \""
+ bytesWritten.getName() + "\": " + bytesWritten.get() + ", \"" + level.getName() + "\": " + level.get()
+ + ", \"" + sampleAccessCounter.getName() + "\": " + sampleAccessCounter.get() + ", \""
+ + samplePinReuseCounter.getName() + "\": " + samplePinReuseCounter.get() + ", \""
+ + samplePreemptiveEndCounter.getName() + "\": " + samplePreemptiveEndCounter.get()
+ ", \"indexStats\": \"" + indexesStats + "\" }";
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/DiskBTreeSampleCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/DiskBTreeSampleCursor.java
index de19671..39d174d 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/DiskBTreeSampleCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/DiskBTreeSampleCursor.java
@@ -29,6 +29,7 @@
import org.apache.hyracks.storage.am.btree.api.IBTreeLeafFrame;
import org.apache.hyracks.storage.am.btree.api.ITupleAcceptor;
import org.apache.hyracks.storage.am.common.api.ILSMIndexBatchPointCursor;
+import org.apache.hyracks.storage.am.common.api.ISampleCursorStats;
import org.apache.hyracks.storage.am.common.api.ITreeIndexCursor;
import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleReference;
import org.apache.hyracks.storage.common.EnforcedIndexCursor;
@@ -53,7 +54,7 @@
* accepted samples. For row storage with typical low sampling density
* (scattered samples across many pages), this approach minimizes page pins.
*/
-public final class DiskBTreeSampleCursor extends EnforcedIndexCursor implements ITreeIndexCursor {
+public final class DiskBTreeSampleCursor extends EnforcedIndexCursor implements ITreeIndexCursor, ISampleCursorStats {
private static final Logger LOGGER = LogManager.getLogger();
private final DiskBTree bTree;
@@ -125,10 +126,7 @@
// to correct for bias from partially-filled pages
private final int leafTupleCapacity;
- // Debug and traceability
- private long totalLeafDrawBatches = 0;
- private long totalLeafDraws = 0;
- private long totalLeafDrawUniquePages = 0;
+ // Sampling counters, propagated to the query profile via ISampleCursorStats (ASTERIXDB-3702).
private long totalLeafPins = 0;
private long totalReusedPinnedPageHits = 0;
private boolean endedPreemptively = false;
@@ -277,21 +275,6 @@
// Primitive long[] sort: branch-predictable, comparator-free, and
// cache-friendly versus the previous boxed-object comparator sort.
Arrays.sort(drawSortKeys);
-
- if (LOGGER.isDebugEnabled()) {
- int uniquePages = 0;
- long prevPageId = Long.MIN_VALUE;
- for (int i = 0; i < leafDrawBatchSize; i++) {
- long currentPageId = drawSortKeys[i] >>> 32;
- if (currentPageId != prevPageId) {
- uniquePages++;
- prevPageId = currentPageId;
- }
- }
- totalLeafDrawBatches++;
- totalLeafDraws += leafDrawBatchSize;
- totalLeafDrawUniquePages += uniquePages;
- }
}
private long getPageTupleKey(int pageId, int tupleIndex) {
@@ -374,20 +357,13 @@
@Override
protected void doClose() throws HyracksDataException {
if (LOGGER.isDebugEnabled()) {
- double avgBatchDraws = totalLeafDrawBatches == 0 ? 0.0 : (double) totalLeafDraws / totalLeafDrawBatches;
- double avgBatchUniquePages =
- totalLeafDrawBatches == 0 ? 0.0 : (double) totalLeafDrawUniquePages / totalLeafDrawBatches;
- double uniquePerDraw = totalLeafDraws == 0 ? 0.0 : (double) totalLeafDrawUniquePages / totalLeafDraws;
double repinAvoidRate = (totalLeafPins + totalReusedPinnedPageHits) == 0 ? 0.0
: (double) totalReusedPinnedPageHits / (totalLeafPins + totalReusedPinnedPageHits);
LOGGER.debug(
- "{} stats - sampledCount: {}, totalAccessCount: {}, "
- + "leafDrawBatches: {}, totalLeafDraws: {}, totalLeafDrawUniquePages: {}, "
- + "avgBatchDraws: {}, avgBatchUniquePages: {}, uniquePerDraw: {}, totalLeafPins: {}, "
+ "{} stats - sampledCount: {}, totalAccessCount: {}, totalLeafPins: {}, "
+ "reusedPinnedPageHits: {}, repinAvoidRate: {}, endedPreemptively: {}",
- this.getClass().getName(), sampledCount, totalAccessCount, totalLeafDrawBatches, totalLeafDraws,
- totalLeafDrawUniquePages, avgBatchDraws, avgBatchUniquePages, uniquePerDraw, totalLeafPins,
- totalReusedPinnedPageHits, repinAvoidRate, endedPreemptively);
+ this.getClass().getName(), sampledCount, totalAccessCount, totalLeafPins, totalReusedPinnedPageHits,
+ repinAvoidRate, endedPreemptively);
}
if (page != null) {
releasePage();
@@ -401,9 +377,6 @@
pendingLeafDrawIndex = leafDrawBatchSize;
totalAccessCount = 0;
- totalLeafDrawBatches = 0;
- totalLeafDraws = 0;
- totalLeafDrawUniquePages = 0;
totalLeafPins = 0;
totalReusedPinnedPageHits = 0;
endedPreemptively = false;
@@ -415,6 +388,21 @@
}
@Override
+ public long getSampleAccessCount() {
+ return totalAccessCount;
+ }
+
+ @Override
+ public long getSamplePinReuseHits() {
+ return totalReusedPinnedPageHits;
+ }
+
+ @Override
+ public long getSamplePreemptiveEndCount() {
+ return endedPreemptively ? 1 : 0;
+ }
+
+ @Override
public void setBufferCache(IBufferCache bufferCache) {
this.bufferCache = bufferCache;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ISampleCursorStats.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ISampleCursorStats.java
new file mode 100644
index 0000000..2ad1fbe
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ISampleCursorStats.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.common.api;
+
+import org.apache.hyracks.util.annotations.AiProvenance;
+
+/**
+ * Read-only sampling counters exposed by the disk/column sample cursors (ASTERIXDB-3702) so the sample-scan
+ * operator can propagate them into the query profile (via {@code DatasetStreamStats}). Implemented by both the
+ * row and column sample cursors; values are meaningful only after a sample scan and before the cursor's next
+ * {@code close()} (which resets them). Counters not tracked by a given cursor return {@code 0}.
+ */
+@AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_4_8, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.GENERATED, notes = "Sample cursor profiling counters (ASTERIXDB-3702)")
+public interface ISampleCursorStats {
+
+ /** Total leaf/tuple draw attempts made to reach the sample target (draws-per-accept efficiency). */
+ long getSampleAccessCount();
+
+ /** Number of draws that reused an already-pinned leaf page (batched-pin I/O saving; 0 if not tracked). */
+ long getSamplePinReuseHits();
+
+ /** 1 if this cursor stopped before reaching its per-component target (sample fell short), else 0. */
+ long getSamplePreemptiveEndCount();
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBtreeSampleCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBtreeSampleCursor.java
index 4a73621..d4db12f 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBtreeSampleCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/impls/btree/ColumnBtreeSampleCursor.java
@@ -30,6 +30,7 @@
import org.apache.hyracks.storage.am.btree.impls.BTreeOpContext;
import org.apache.hyracks.storage.am.btree.impls.BatchPredicateWithKeys;
import org.apache.hyracks.storage.am.common.api.ILSMIndexBatchPointCursor;
+import org.apache.hyracks.storage.am.common.api.ISampleCursorStats;
import org.apache.hyracks.storage.am.common.api.ITreeIndexCursor;
import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnReadMultiPageOp;
import org.apache.hyracks.storage.am.lsm.btree.column.api.IColumnTupleIterator;
@@ -67,7 +68,8 @@
* {@code setAt()}. This amortises mega-page I/O and gives cache-friendly
* sequential access.
*/
-public class ColumnBtreeSampleCursor extends EnforcedIndexCursor implements ITreeIndexCursor, IColumnReadMultiPageOp {
+public class ColumnBtreeSampleCursor extends EnforcedIndexCursor
+ implements ITreeIndexCursor, IColumnReadMultiPageOp, ISampleCursorStats {
private static final Logger LOGGER = LogManager.getLogger();
@@ -496,6 +498,23 @@
}
@Override
+ public long getSampleAccessCount() {
+ return totalAccessCount;
+ }
+
+ @Override
+ public long getSamplePinReuseHits() {
+ // The column cursor pins page0 per selected page (Phase 1) and reloads once per page (Phase 2);
+ // it has no per-draw pinned-page reuse to report.
+ return 0;
+ }
+
+ @Override
+ public long getSamplePreemptiveEndCount() {
+ return endedPreemptively ? 1 : 0;
+ }
+
+ @Override
public ICachedPage pin(int pageId) throws HyracksDataException {
return bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), context);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/BTreeSampleCollectorOperatorDescriptorNodePushable.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/BTreeSampleCollectorOperatorDescriptorNodePushable.java
index 2f7d3e6..2565596 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/BTreeSampleCollectorOperatorDescriptorNodePushable.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/BTreeSampleCollectorOperatorDescriptorNodePushable.java
@@ -43,6 +43,10 @@
public class BTreeSampleCollectorOperatorDescriptorNodePushable extends BTreeSearchOperatorNodePushable {
public static final String ESTIMATE_CARDINALITY = "ESTIMATE_CARDINALITY";
+ // Sample profiling counters bridged to the downstream Sample.DatasetStats operator (ASTERIXDB-3702).
+ public static final String SAMPLE_ACCESS_COUNT = "SAMPLE_ACCESS_COUNT";
+ public static final String SAMPLE_PIN_REUSE_HITS = "SAMPLE_PIN_REUSE_HITS";
+ public static final String SAMPLE_PREEMPTIVE_END_COUNT = "SAMPLE_PREEMPTIVE_END_COUNT";
private final int sampleCardinalityTargetPerPartition;
private final long sampleSeed;
private final int maxSampleLeafAttempts;
@@ -132,14 +136,26 @@
protected void searchAllPartitions(int tupleCount) throws Exception {
long estimatedCardinality = 0;
+ long sampleAccessCount = 0;
+ long samplePinReuseHits = 0;
+ long samplePreemptiveEndCount = 0;
for (int p = 0; p < partitions.length; p++) {
for (int i = 0; i < tupleCount && !finished; i++) {
cursors[p].close();
((ILSMIndexAccessor) indexAccessors[p]).scanDiskComponentsForSample(cursors[p]);
writeSearchResults(i, cursors[p]);
}
- estimatedCardinality += ((LSMIndexSampleCursor) cursors[p]).getEstimatedCardinality();
+ // Read the per-partition sample counters here (post-drain, before the next close() resets them),
+ // mirroring the getEstimatedCardinality() harvest.
+ LSMIndexSampleCursor sampleCursor = (LSMIndexSampleCursor) cursors[p];
+ estimatedCardinality += sampleCursor.getEstimatedCardinality();
+ sampleAccessCount += sampleCursor.getSampleAccessCount();
+ samplePinReuseHits += sampleCursor.getSamplePinReuseHits();
+ samplePreemptiveEndCount += sampleCursor.getSamplePreemptiveEndCount();
}
TaskUtil.put(ESTIMATE_CARDINALITY, estimatedCardinality, ctx);
+ TaskUtil.put(SAMPLE_ACCESS_COUNT, sampleAccessCount, ctx);
+ TaskUtil.put(SAMPLE_PIN_REUSE_HITS, samplePinReuseHits, ctx);
+ TaskUtil.put(SAMPLE_PREEMPTIVE_END_COUNT, samplePreemptiveEndCount, ctx);
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMIndexSampleCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMIndexSampleCursor.java
index 6255272..40671ca 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMIndexSampleCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMIndexSampleCursor.java
@@ -29,6 +29,7 @@
import org.apache.hyracks.storage.am.btree.impls.DiskBTree;
import org.apache.hyracks.storage.am.common.api.ILSMIndexBatchPointCursor;
import org.apache.hyracks.storage.am.common.api.ILSMIndexCursor;
+import org.apache.hyracks.storage.am.common.api.ISampleCursorStats;
import org.apache.hyracks.storage.am.common.api.ITreeIndexCursor;
import org.apache.hyracks.storage.am.common.impls.NoOpIndexAccessParameters;
import org.apache.hyracks.storage.am.common.util.ResourceReleaseUtils;
@@ -155,6 +156,32 @@
return estimatedCardinality;
}
+ // Aggregate the per-component sample counters (ASTERIXDB-3702). Valid after a sample scan and before the
+ // next close() resets the sub-cursors; read alongside getEstimatedCardinality() by the collector operator.
+ public long getSampleAccessCount() {
+ return sumSampleStat(ISampleCursorStats::getSampleAccessCount);
+ }
+
+ public long getSamplePinReuseHits() {
+ return sumSampleStat(ISampleCursorStats::getSamplePinReuseHits);
+ }
+
+ public long getSamplePreemptiveEndCount() {
+ return sumSampleStat(ISampleCursorStats::getSamplePreemptiveEndCount);
+ }
+
+ private long sumSampleStat(java.util.function.ToLongFunction<ISampleCursorStats> selector) {
+ long total = 0;
+ if (btreeCursors != null) {
+ for (ITreeIndexCursor c : btreeCursors) {
+ if (c instanceof ISampleCursorStats) {
+ total += selector.applyAsLong((ISampleCursorStats) c);
+ }
+ }
+ }
+ return total;
+ }
+
private DiskBTree.DiskBTreeAccessor createAccessor(DiskBTree bTree, ILSMIndexOperationContext opCtx, int index)
throws HyracksDataException {
return (DiskBTree.DiskBTreeAccessor) bTree.createAccessor(NoOpIndexAccessParameters.INSTANCE, opCtx, index);
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21431?usp=email
To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings?usp=email
Gerrit-MessageType: newchange
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Change-Id: I1c5e0571b956f473525cd8e6cac2f210f39dea53
Gerrit-Change-Number: 21431
Gerrit-PatchSet: 1
Gerrit-Owner: Ritik Raj <ri...@couchbase.com>