You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by im...@apache.org on 2026/08/06 18:19:45 UTC
(asterixdb) 07/14: [ASTERIXDB-3754][HYR][STO] Storage layer of VTree index - patch 2
This is an automated email from the ASF dual-hosted git repository.
parshimers pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 683af52e5b059803c570d6802872792da164654a
Author: Le0shy <le...@gmail.com>
AuthorDate: Sun May 17 18:14:54 2026 -0700
[ASTERIXDB-3754][HYR][STO] Storage layer of VTree index - patch 2
- user model changes: no
- storage format changes: yes
- interface changes: yes
Details:
LSM VTree storage layer for hierarchical cluster-based approximate
nearest neighbor search.
Adds the hyracks-storage-am-lsm-vtree module:
- LSMVTree: LSM-based vector index coordinating memory/disk components
and index operations
- LSMVTreeLocalResource / LSMVTreeLocalResourceFactory: persisted index
resource
- Vector search operator, streaming and top-K search cursors, and
cluster-selection strategies
Ext-ref: MB-72975
Change-Id: I87c468c14cbeb3e698742429ee62df02c8e83efe
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21100
Contrib: Ian Maxon <im...@apache.org>
Reviewed-by: Ian Maxon <im...@apache.org>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
---
.../storage/am/common/build/IndexBuilder.java | 13 +
.../am/lsm/common/impls/AbstractLSMIndex.java | 2 +-
.../impls/LSMVTreeComponentFileReferences.java | 50 +
.../hyracks/hyracks-storage-am-lsm-vtree/pom.xml | 104 ++
.../lsm/vector/dataflow/LSMVTreeLocalResource.java | 406 ++++++++
.../dataflow/LSMVTreeLocalResourceFactory.java | 96 ++
.../lsm/vector/dataflow/QuantizedIndexBuilder.java | 74 ++
.../dataflow/QuantizedIndexBuilderFactory.java | 53 +
.../QuantizedIndexCreateOperatorDescriptor.java | 180 ++++
.../dataflow/VTreeSearchOperatorDescriptor.java | 134 +++
.../dataflow/VTreeSearchOperatorNodePushable.java | 334 ++++++
.../vector/impls/IClusterSelectionStrategy.java | 144 +++
.../am/lsm/vector/impls/IVectorSearchCursor.java | 50 +
.../storage/am/lsm/vector/impls/LSMVTree.java | 718 +++++++++++++
.../vector/impls/LSMVTreeCursorInitialState.java | 142 +++
.../am/lsm/vector/impls/LSMVTreeDiskComponent.java | 262 +++++
.../vector/impls/LSMVTreeDiskComponentFactory.java | 53 +
.../am/lsm/vector/impls/LSMVTreeFileManager.java | 213 ++++
.../lsm/vector/impls/LSMVTreeFlushOperation.java | 45 +
.../am/lsm/vector/impls/LSMVTreeIndexAccessor.java | 60 ++
.../lsm/vector/impls/LSMVTreeMemoryComponent.java | 85 ++
.../lsm/vector/impls/LSMVTreeMergeOperation.java | 47 +
.../am/lsm/vector/impls/LSMVTreeOpContext.java | 177 ++++
.../am/lsm/vector/impls/LSMVTreeSearchCursor.java | 1087 ++++++++++++++++++++
.../lsm/vector/impls/LSMVTreeTopKSearchCursor.java | 810 +++++++++++++++
.../impls/NprobeClusterSelectionStrategy.java | 297 ++++++
.../am/lsm/vector/impls/PKOnlyTupleProjector.java | 71 ++
.../vector/impls/PKOnlyTupleProjectorFactory.java | 49 +
.../impls/SequentialClusterSelectionStrategy.java | 150 +++
.../am/lsm/vector/impls/SpillableTopKBuffer.java | 433 ++++++++
.../vector/impls/SpillableTopKDrainIterator.java | 409 ++++++++
.../storage/am/lsm/vector/impls/VTreeFactory.java | 77 ++
.../am/lsm/vector/impls/VectorSearchHeapEntry.java | 53 +
.../vector/impls/VectorSearchHeapEntryFactory.java | 29 +
.../lsm/vector/tuples/LSMVTreeCopyTupleWriter.java | 59 ++
.../tuples/LSMVTreeCopyTupleWriterFactory.java | 52 +
.../vector/tuples/LSMVTreeDataTupleReference.java | 68 ++
.../lsm/vector/tuples/LSMVTreeDataTupleWriter.java | 97 ++
.../tuples/LSMVTreeDataTupleWriterFactory.java | 67 ++
.../storage/am/lsm/vector/utils/LSMVTreeUtils.java | 175 ++++
.../storage/am/vector/api/IQuantizedResource.java | 27 +
.../hyracks/storage/common/IResourceWrapper.java | 32 +
.../LSMVTreeFileManagerCrashRecoveryTest.java | 128 +++
.../LSMVTreeMultiVersionReconciliationTest.java | 240 +++++
hyracks-fullstack/hyracks/pom.xml | 1 +
45 files changed, 7852 insertions(+), 1 deletion(-)
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/build/IndexBuilder.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/build/IndexBuilder.java
index 8a67c71d09..7109aababf 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/build/IndexBuilder.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/build/IndexBuilder.java
@@ -65,6 +65,17 @@ public class IndexBuilder implements IIndexBuilder {
resourceRelPath = resourceRef.getRelativePath();
}
+ /**
+ * Hook for subclasses to configure the freshly created {@link IResource} before it is wrapped in a
+ * {@link LocalResource} and persisted. The base implementation does nothing; index types with extra
+ * per-resource state (e.g. the vector index injecting quantization constants) override it. Keeping this
+ * seam here lets index-type-specific configuration live in the subclass rather than leaking into the
+ * shared builder.
+ */
+ protected void configureResource(IResource resource) throws HyracksDataException {
+ // no-op by default
+ }
+
@Override
public void build() throws HyracksDataException {
IResourceLifecycleManager<IIndex> lcManager = storageManager.getLifecycleManager(ctx);
@@ -79,6 +90,8 @@ public class IndexBuilder implements IIndexBuilder {
}
resourceId = resourceIdFactory.createId();
IResource resource = localResourceFactory.createResource(resourceRef);
+ // Let subclasses inject index-type-specific state (e.g. vector-index quantization constants).
+ configureResource(resource);
lr = new LocalResource(resourceId, ITreeIndexFrame.Constants.VERSION, durable, resource);
IIndex index = lcManager.get(resourceRelPath);
if (index != null) {
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
index 93b9a373d7..0609ef2382 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
@@ -712,7 +712,7 @@ public abstract class AbstractLSMIndex implements ILSMIndex {
*
* @throws HyracksDataException
*/
- private void validateComponentIds() throws HyracksDataException {
+ protected void validateComponentIds() throws HyracksDataException {
for (int i = 0; i < diskComponents.size() - 1; i++) {
ILSMComponentId id1 = diskComponents.get(i).getId();
ILSMComponentId id2 = diskComponents.get(i + 1).getId();
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMVTreeComponentFileReferences.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMVTreeComponentFileReferences.java
new file mode 100644
index 0000000000..07334e802b
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMVTreeComponentFileReferences.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.lsm.common.impls;
+
+import org.apache.hyracks.api.io.FileReference;
+
+public class LSMVTreeComponentFileReferences extends LSMComponentFileReferences {
+ private final FileReference staticStructureFileReference;
+
+ public LSMVTreeComponentFileReferences(FileReference insertIndexFileReference,
+ FileReference deleteIndexFileReference, FileReference bloomFilterFileReference,
+ FileReference staticStructureFileReference) {
+ super(insertIndexFileReference, deleteIndexFileReference, bloomFilterFileReference);
+ this.staticStructureFileReference = staticStructureFileReference;
+ }
+
+ public FileReference getStaticStructureFileReference() {
+ return staticStructureFileReference;
+ }
+
+ @Override
+ public FileReference[] getFileReferences() {
+ return new FileReference[] { getInsertIndexFileReference(), getDeleteIndexFileReference(),
+ getBloomFilterFileReference(), staticStructureFileReference };
+ }
+
+ @Override
+ public String toString() {
+ return "{ \"insert\" : \"" + getInsertIndexFileReference() + "\", \"delete\" : \""
+ + getDeleteIndexFileReference() + "\", \"bloom\" : \"" + getBloomFilterFileReference()
+ + "\", \"static\" : \"" + staticStructureFileReference + "\"}";
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/pom.xml b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/pom.xml
new file mode 100644
index 0000000000..5203cd959d
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/pom.xml
@@ -0,0 +1,104 @@
+<!--
+ ! Licensed to the Apache Software Foundation (ASF) under one
+ ! or more contributor license agreements. See the NOTICE file
+ ! distributed with this work for additional information
+ ! regarding copyright ownership. The ASF licenses this file
+ ! to you under the Apache License, Version 2.0 (the
+ ! "License"); you may not use this file except in compliance
+ ! with the License. You may obtain a copy of the License at
+ !
+ ! http://www.apache.org/licenses/LICENSE-2.0
+ !
+ ! Unless required by applicable law or agreed to in writing,
+ ! software distributed under the License is distributed on an
+ ! "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ! KIND, either express or implied. See the License for the
+ ! specific language governing permissions and limitations
+ ! under the License.
+ !-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>hyracks-storage-am-lsm-vtree</artifactId>
+ <parent>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks</artifactId>
+ <version>0.3.10-SNAPSHOT</version>
+ </parent>
+ <licenses>
+ <license>
+ <name>Apache License, Version 2.0</name>
+ <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+ <distribution>repo</distribution>
+ <comments>A business-friendly OSS license</comments>
+ </license>
+ </licenses>
+ <properties>
+ <root.dir>${basedir}/../..</root.dir>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-storage-am-vtree</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-storage-am-lsm-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-storage-am-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-dataflow-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-dataflow-std</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-api</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-storage-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-data-std</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-util</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-control-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-control-nc</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-api</artifactId>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/dataflow/LSMVTreeLocalResource.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/dataflow/LSMVTreeLocalResource.java
new file mode 100644
index 0000000000..e6f928702a
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/dataflow/LSMVTreeLocalResource.java
@@ -0,0 +1,406 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.vector.dataflow;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hyracks.api.application.INCServiceContext;
+import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import org.apache.hyracks.api.dataflow.value.ITypeTraits;
+import org.apache.hyracks.api.exceptions.ErrorCode;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.IIOManager;
+import org.apache.hyracks.api.io.IJsonSerializable;
+import org.apache.hyracks.api.io.IPersistedResourceRegistry;
+import org.apache.hyracks.control.common.controllers.NCConfig;
+import org.apache.hyracks.control.nc.NodeControllerService;
+import org.apache.hyracks.storage.am.common.api.IMetadataPageManagerFactory;
+import org.apache.hyracks.storage.am.common.api.INullIntrospector;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackFactory;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationSchedulerProvider;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTrackerFactory;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMPageWriteCallbackFactory;
+import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
+import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCacheProvider;
+import org.apache.hyracks.storage.am.lsm.common.dataflow.LsmResource;
+import org.apache.hyracks.storage.am.lsm.vector.utils.LSMVTreeUtils;
+import org.apache.hyracks.storage.am.vector.api.IQuantizedResource;
+import org.apache.hyracks.storage.am.vector.api.IVTreeBinaryAccessorFactory;
+import org.apache.hyracks.storage.am.vector.api.IVTreeDataTupleBuilderFactory;
+import org.apache.hyracks.storage.am.vector.api.IVTreeDistanceFunctionFactory;
+import org.apache.hyracks.storage.am.vector.api.VTreeQuantizationParams;
+import org.apache.hyracks.storage.am.vector.impls.VTreeDataTupleBuilderFactory;
+import org.apache.hyracks.storage.am.vector.utils.CrossPollinationConfig;
+import org.apache.hyracks.storage.common.IIndex;
+import org.apache.hyracks.storage.common.IStorageManager;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
+public class LSMVTreeLocalResource extends LsmResource implements IQuantizedResource {
+
+ private static final long serialVersionUID = 2L;
+
+ // Persisted/wire key names for quantization parameters. Shared with QuantizedIndexCreate so
+ // the producer and the persisted resource agree on the vocabulary.
+ public static final String KEY_MIN_QUANTILE = "minQuantile";
+ public static final String KEY_MAX_QUANTILE = "maxQuantile";
+ public static final String KEY_ALPHA = "alpha";
+ public static final String KEY_BITS = "bits";
+ public static final String KEY_CONFIDENCE_INTERVAL = "confidenceInterval";
+ public static final String KEY_SAMPLE_COUNT = "sampleCount";
+
+ private static final double BLOOM_FILTER_FALSE_POSITIVE_RATE = 0.01;
+ /** JSON key under which the vector accessor factory is persisted. */
+ private static final String KEY_VECTOR_ACCESSOR_FACTORY = "vectorAccessorFactory";
+ /** JSON key under which the distance-function factory is persisted. */
+ private static final String KEY_DISTANCE_FUNCTION_FACTORY = "distanceFunctionFactory";
+ /** JSON keys for the cross-pollination placement parameters (see {@link CrossPollinationConfig}). */
+ private static final String KEY_CROSS_POLLINATION_M = "crossPollinationM";
+ private static final String KEY_RNG_FACTOR = "rngFactor";
+ private static final String KEY_EPSILON = "epsilon";
+
+ protected final int vectorDimensions;
+ protected final int[] vectorFields;
+ protected final int[] filterFields;
+ protected final boolean atomic;
+ protected final IVTreeBinaryAccessorFactory vectorAccessorFactory;
+ protected final int numPrimaryKeyFields;
+ protected final int numIncludeFields;
+ protected final IVTreeDataTupleBuilderFactory dataTupleBuilderFactory;
+
+ /**
+ * Distance-function factory supplied at DDL time. Persisted via the resource registry so a
+ * restarted index reconstructs the same distance implementation.
+ */
+ protected final IVTreeDistanceFunctionFactory distanceFunctionFactory;
+
+ /**
+ * Cross-pollination placement config supplied at DDL time. Persisted to JSON so that incremental
+ * insert/delete on a restarted index replicate into the same leaf clusters bulk-load used (M=1 =
+ * legacy single-closest).
+ */
+ protected final CrossPollinationConfig crossPollination;
+
+ // Quantization parameters (optional, set by QuantizedIndexCreate during index creation)
+ protected Float confidenceInterval;
+ protected Float minQuantile;
+ protected Float maxQuantile;
+ protected Float alpha;
+ protected Integer bits;
+ protected Integer sampleCount;
+
+ public LSMVTreeLocalResource(String path, IStorageManager storageManager, ITypeTraits[] typeTraits,
+ IBinaryComparatorFactory[] cmpFactories, ITypeTraits[] filterTypeTraits,
+ IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields,
+ ILSMOperationTrackerFactory opTrackerProvider, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
+ ILSMPageWriteCallbackFactory pageWriteCallbackFactory,
+ IMetadataPageManagerFactory metadataPageManagerFactory, IVirtualBufferCacheProvider vbcProvider,
+ ILSMIOOperationSchedulerProvider ioSchedulerProvider, ILSMMergePolicyFactory mergePolicyFactory,
+ Map<String, String> mergePolicyProperties, boolean durable, int vectorDimensions, int[] vectorFields,
+ ITypeTraits nullTypeTraits, INullIntrospector nullIntrospector, boolean atomic,
+ IVTreeBinaryAccessorFactory vectorAccessorFactory, int numPrimaryKeyFields, int numIncludeFields,
+ IVTreeDataTupleBuilderFactory dataTupleBuilderFactory,
+ IVTreeDistanceFunctionFactory distanceFunctionFactory, Float confidenceInterval, Float minQuantile,
+ Float maxQuantile, Float alpha, Integer bits, Integer sampleCount,
+ CrossPollinationConfig crossPollination) {
+ super(path, storageManager, typeTraits, cmpFactories, filterTypeTraits, filterCmpFactories, filterFields,
+ opTrackerProvider, ioOpCallbackFactory, pageWriteCallbackFactory, metadataPageManagerFactory,
+ vbcProvider, ioSchedulerProvider, mergePolicyFactory, mergePolicyProperties, durable, nullTypeTraits,
+ nullIntrospector);
+ this.vectorDimensions = vectorDimensions;
+ this.vectorFields = vectorFields;
+ this.filterFields = filterFields;
+ this.atomic = atomic;
+ this.distanceFunctionFactory = distanceFunctionFactory;
+ this.crossPollination = crossPollination != null ? crossPollination : CrossPollinationConfig.LEGACY;
+ this.confidenceInterval = confidenceInterval;
+ this.minQuantile = minQuantile;
+ this.maxQuantile = maxQuantile;
+ this.alpha = alpha;
+ this.bits = bits;
+ this.sampleCount = sampleCount;
+ this.vectorAccessorFactory = vectorAccessorFactory;
+ this.numPrimaryKeyFields = numPrimaryKeyFields;
+ this.numIncludeFields = numIncludeFields;
+ this.dataTupleBuilderFactory = dataTupleBuilderFactory;
+ }
+
+ protected LSMVTreeLocalResource(IPersistedResourceRegistry registry, JsonNode json, int vectorDimensions,
+ int[] vectorFields, int[] filterFields, boolean atomic, Float confidenceInterval, Float minQuantile,
+ Float maxQuantile, Float alpha, Integer bits, Integer sampleCount,
+ IVTreeBinaryAccessorFactory vectorAccessorFactory, int numPrimaryKeyFields, int numIncludeFields,
+ IVTreeDataTupleBuilderFactory dataTupleBuilderFactory,
+ IVTreeDistanceFunctionFactory distanceFunctionFactory, CrossPollinationConfig crossPollination)
+ throws HyracksDataException {
+ super(registry, json);
+ this.vectorDimensions = vectorDimensions;
+ this.vectorFields = vectorFields;
+ this.filterFields = filterFields;
+ this.atomic = atomic;
+ this.distanceFunctionFactory = distanceFunctionFactory;
+ this.crossPollination = crossPollination != null ? crossPollination : CrossPollinationConfig.LEGACY;
+ this.confidenceInterval = confidenceInterval;
+ this.minQuantile = minQuantile;
+ this.maxQuantile = maxQuantile;
+ this.alpha = alpha;
+ this.bits = bits;
+ this.sampleCount = sampleCount;
+ this.vectorAccessorFactory = vectorAccessorFactory;
+ this.numPrimaryKeyFields = numPrimaryKeyFields;
+ this.numIncludeFields = numIncludeFields;
+ this.dataTupleBuilderFactory = dataTupleBuilderFactory;
+ }
+
+ @Override
+ public IIndex createInstance(INCServiceContext ncServiceCtx) throws HyracksDataException {
+ IIOManager ioManager = storageManager.getIoManager(ncServiceCtx);
+ NCConfig storageConfig = ((NodeControllerService) ncServiceCtx.getControllerService()).getConfiguration();
+ FileReference fileRef = ioManager.resolve(path);
+
+ List<IVirtualBufferCache> virtualBufferCaches = vbcProvider.getVirtualBufferCaches(ncServiceCtx, fileRef);
+ ioOpCallbackFactory.initialize(ncServiceCtx, this);
+ pageWriteCallbackFactory.initialize(ncServiceCtx, this);
+
+ // Construction-time invariant: vectorAccessorFactory is always supplied — by
+ // VTreeResourceFactoryProvider in production and by the test harness in tests, and round-
+ // tripped through the resource registry on JSON read. A null here indicates a corrupted
+ // resource (e.g., a JSON file with the {@link #KEY_VECTOR_ACCESSOR_FACTORY} key missing).
+ if (vectorAccessorFactory == null) {
+ throw new HyracksDataException("Vector accessor factory missing from LSMVTreeLocalResource (path=" + path
+ + "); resource is corrupted or was written by an unsupported codebase version.");
+ }
+ // Distance-function factory is always supplied at DDL and round-tripped through the registry;
+ // a null indicates a corrupted or unsupported-version resource.
+ if (distanceFunctionFactory == null) {
+ throw new HyracksDataException("Distance-function factory missing from LSMVTreeLocalResource (path=" + path
+ + "); resource is corrupted or was written by an unsupported codebase version.");
+ }
+
+ // Quantization params for lazy quantizer creation at query time
+ VTreeQuantizationParams quantizationParams = null;
+ if (hasQuantizationParams()) {
+ quantizationParams =
+ new VTreeQuantizationParams(minQuantile, maxQuantile, alpha, confidenceInterval, bits, sampleCount);
+ }
+
+ return LSMVTreeUtils.createLSMTree(storageConfig, ioManager, virtualBufferCaches, fileRef,
+ storageManager.getBufferCache(ncServiceCtx), typeTraits, cmpFactories, BLOOM_FILTER_FALSE_POSITIVE_RATE,
+ mergePolicyFactory.createMergePolicy(mergePolicyProperties, ncServiceCtx),
+ opTrackerProvider.getOperationTracker(ncServiceCtx, this),
+ ioSchedulerProvider.getIoScheduler(ncServiceCtx), ioOpCallbackFactory, pageWriteCallbackFactory,
+ vectorDimensions, vectorFields, filterFields, null, // filterFrameFactory
+ null, // filterManager
+ null, // filterHelper
+ durable, metadataPageManagerFactory, atomic, null, vectorAccessorFactory, numPrimaryKeyFields,
+ numIncludeFields, dataTupleBuilderFactory, quantizationParams, distanceFunctionFactory,
+ crossPollination);
+ }
+
+ @Override
+ public JsonNode toJson(IPersistedResourceRegistry registry) throws HyracksDataException {
+ ObjectNode jsonObject = registry.getClassIdentifier(getClass(), serialVersionUID);
+ appendToJson(jsonObject, registry); // Call this.appendToJson() to include quantization params
+ return jsonObject;
+ }
+
+ @Override
+ protected void appendToJson(final ObjectNode json, IPersistedResourceRegistry registry)
+ throws HyracksDataException {
+ super.appendToJson(json, registry);
+ json.put("vectorDimensions", vectorDimensions);
+ json.putPOJO("vectorFields", vectorFields);
+ json.putPOJO("filterFields", filterFields);
+ json.put("atomic", atomic);
+ // Cross-pollination placement params — only when active (M>1); absence reads back as legacy M=1.
+ if (crossPollination != null && crossPollination.enabled()) {
+ json.put(KEY_CROSS_POLLINATION_M, crossPollination.m());
+ json.put(KEY_RNG_FACTOR, crossPollination.rngFactor());
+ json.put(KEY_EPSILON, crossPollination.epsilon());
+ }
+ // Vector accessor factory — round-tripped via IPersistedResourceRegistry. The factory
+ // implementation (AOrderedListVectorBinaryAccessorFactory) must be registered with the
+ // registry; see PersistedResourceRegistry#registerClasses.
+ if (vectorAccessorFactory != null) {
+ json.set(KEY_VECTOR_ACCESSOR_FACTORY, vectorAccessorFactory.toJson(registry));
+ }
+ // Distance-function factory — round-tripped via the registry (implementation must be
+ // registered; see PersistedResourceRegistry#registerClasses).
+ if (distanceFunctionFactory != null) {
+ json.set(KEY_DISTANCE_FUNCTION_FACTORY, distanceFunctionFactory.toJson(registry));
+ }
+ // Write quantization parameters only when set (a non-quantized index has none).
+ putIfNotNull(json, KEY_CONFIDENCE_INTERVAL, confidenceInterval);
+ putIfNotNull(json, KEY_MIN_QUANTILE, minQuantile);
+ putIfNotNull(json, KEY_MAX_QUANTILE, maxQuantile);
+ putIfNotNull(json, KEY_ALPHA, alpha);
+ putIfNotNull(json, KEY_BITS, bits);
+ putIfNotNull(json, KEY_SAMPLE_COUNT, sampleCount);
+ json.put("numPrimaryKeyFields", numPrimaryKeyFields);
+ json.put("numIncludeFields", numIncludeFields);
+ }
+
+ public static IJsonSerializable fromJson(IPersistedResourceRegistry registry, JsonNode json)
+ throws HyracksDataException {
+ // appendToJson always writes vectorDimensions, so its absence (or a non-positive value) means the
+ // resource is corrupt/foreign. Fail fast with a clear message instead of carrying -1 into deeper,
+ // less obvious failures. (numPrimaryKeyFields/numIncludeFields keep plausible back-compat defaults.)
+ if (!json.has("vectorDimensions") || json.get("vectorDimensions").asInt() <= 0) {
+ throw HyracksDataException.create(ErrorCode.ILLEGAL_STATE,
+ "LSMVTreeLocalResource is missing or has a non-positive vectorDimensions; resource is corrupt");
+ }
+ int vectorDimensions = json.get("vectorDimensions").asInt();
+ int numPrimaryKeyFields = json.has("numPrimaryKeyFields") ? json.get("numPrimaryKeyFields").asInt() : 1;
+ int numIncludeFields = json.has("numIncludeFields") ? json.get("numIncludeFields").asInt() : 0;
+ int[] vectorFields =
+ json.has("vectorFields") ? OBJECT_MAPPER.convertValue(json.get("vectorFields"), int[].class) : null;
+ int[] filterFields =
+ json.has("filterFields") ? OBJECT_MAPPER.convertValue(json.get("filterFields"), int[].class) : null;
+ boolean atomic = json.has("atomic") && json.get("atomic").asBoolean();
+
+ // Vector accessor factory — required on read; createInstance() throws if absent. New
+ // resources always carry it (see appendToJson). A missing key indicates a corrupted or
+ // unsupported-version JSON file.
+ IVTreeBinaryAccessorFactory vectorAccessorFactory = json.has(KEY_VECTOR_ACCESSOR_FACTORY)
+ ? (IVTreeBinaryAccessorFactory) registry.deserialize(json.get(KEY_VECTOR_ACCESSOR_FACTORY)) : null;
+
+ // Distance-function factory — required on read; createInstance() throws if absent. New resources
+ // always carry it. Missing key => corrupted or unsupported-version JSON.
+ IVTreeDistanceFunctionFactory distanceFunctionFactory = json.has(KEY_DISTANCE_FUNCTION_FACTORY)
+ ? (IVTreeDistanceFunctionFactory) registry.deserialize(json.get(KEY_DISTANCE_FUNCTION_FACTORY)) : null;
+
+ // Read quantization parameters with backward compatibility (missing → null).
+ Float confidenceInterval = readOptionalFloat(json, KEY_CONFIDENCE_INTERVAL);
+ Float minQuantile = readOptionalFloat(json, KEY_MIN_QUANTILE);
+ Float maxQuantile = readOptionalFloat(json, KEY_MAX_QUANTILE);
+ Float alpha = readOptionalFloat(json, KEY_ALPHA);
+ Integer bits = readOptionalInt(json, KEY_BITS);
+ Integer sampleCount = readOptionalInt(json, KEY_SAMPLE_COUNT);
+ // Determine quantized vs non-quantized based on presence of quantization parameters
+ boolean isQuantized = (minQuantile != null);
+ IVTreeDataTupleBuilderFactory dataTupleBuilderFactory =
+ new VTreeDataTupleBuilderFactory(numIncludeFields, isQuantized);
+
+ // Cross-pollination params — back-compat: absent (or M<=1) reads back as legacy single-closest.
+ CrossPollinationConfig crossPollination = CrossPollinationConfig.LEGACY;
+ if (json.has(KEY_CROSS_POLLINATION_M)) {
+ int m = json.get(KEY_CROSS_POLLINATION_M).asInt(1);
+ double rngFactor = json.has(KEY_RNG_FACTOR) ? json.get(KEY_RNG_FACTOR).asDouble(1.0) : 1.0;
+ double epsilon = json.has(KEY_EPSILON) ? json.get(KEY_EPSILON).asDouble(0.3) : 0.3;
+ crossPollination = new CrossPollinationConfig(m, rngFactor, epsilon);
+ }
+
+ return new LSMVTreeLocalResource(registry, json, vectorDimensions, vectorFields, filterFields, atomic,
+ confidenceInterval, minQuantile, maxQuantile, alpha, bits, sampleCount, vectorAccessorFactory,
+ numPrimaryKeyFields, numIncludeFields, dataTupleBuilderFactory, distanceFunctionFactory,
+ crossPollination);
+ }
+
+ /** Read an optional float field from JSON; returns {@code null} if the field is missing or null. */
+ private static Float readOptionalFloat(JsonNode json, String fieldName) {
+ if (!json.has(fieldName)) {
+ return null;
+ }
+ JsonNode node = json.get(fieldName);
+ return node.isNull() ? null : (float) node.asDouble();
+ }
+
+ /** Read an optional int field from JSON; returns {@code null} if the field is missing or null. */
+ private static Integer readOptionalInt(JsonNode json, String fieldName) {
+ if (!json.has(fieldName)) {
+ return null;
+ }
+ JsonNode node = json.get(fieldName);
+ return node.isNull() ? null : node.asInt();
+ }
+
+ private static void putIfNotNull(ObjectNode json, String fieldName, Float value) {
+ if (value != null) {
+ json.put(fieldName, value);
+ }
+ }
+
+ private static void putIfNotNull(ObjectNode json, String fieldName, Integer value) {
+ if (value != null) {
+ json.put(fieldName, value);
+ }
+ }
+
+ public Float getConfidenceInterval() {
+ return confidenceInterval;
+ }
+
+ public Float getMinQuantile() {
+ return minQuantile;
+ }
+
+ public Float getMaxQuantile() {
+ return maxQuantile;
+ }
+
+ public Float getAlpha() {
+ return alpha;
+ }
+
+ public Integer getBits() {
+ return bits;
+ }
+
+ public Integer getSampleCount() {
+ return sampleCount;
+ }
+
+ public int getVectorDimensions() {
+ return vectorDimensions;
+ }
+
+ public int[] getVectorFields() {
+ return vectorFields;
+ }
+
+ public int[] getFilterFields() {
+ return filterFields;
+ }
+
+ public boolean isAtomic() {
+ return atomic;
+ }
+
+ /** @return true iff all required quantization parameters are present. */
+ public boolean hasQuantizationParams() {
+ return bits != null && confidenceInterval != null && minQuantile != null && maxQuantile != null
+ && alpha != null;
+ }
+
+ @Override
+ public void setQuantizationParameters(VTreeQuantizationParams parameters) {
+ if (parameters == null) {
+ return;
+ }
+ this.minQuantile = parameters.minQuantile();
+ this.maxQuantile = parameters.maxQuantile();
+ this.alpha = parameters.alpha();
+ this.bits = parameters.bits();
+ this.confidenceInterval = parameters.confidenceInterval();
+ this.sampleCount = parameters.sampleCount();
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/dataflow/LSMVTreeLocalResourceFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/dataflow/LSMVTreeLocalResourceFactory.java
new file mode 100644
index 0000000000..6fc9680ead
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/dataflow/LSMVTreeLocalResourceFactory.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.vector.dataflow;
+
+import java.util.Map;
+
+import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import org.apache.hyracks.api.dataflow.value.ITypeTraits;
+import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.storage.am.common.api.IMetadataPageManagerFactory;
+import org.apache.hyracks.storage.am.common.api.INullIntrospector;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackFactory;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationSchedulerProvider;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTrackerFactory;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMPageWriteCallbackFactory;
+import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCacheProvider;
+import org.apache.hyracks.storage.am.lsm.common.dataflow.LsmResourceFactory;
+import org.apache.hyracks.storage.am.vector.api.IVTreeBinaryAccessorFactory;
+import org.apache.hyracks.storage.am.vector.api.IVTreeDataTupleBuilderFactory;
+import org.apache.hyracks.storage.am.vector.api.IVTreeDistanceFunctionFactory;
+import org.apache.hyracks.storage.am.vector.utils.CrossPollinationConfig;
+import org.apache.hyracks.storage.common.IResource;
+import org.apache.hyracks.storage.common.IStorageManager;
+
+public class LSMVTreeLocalResourceFactory extends LsmResourceFactory {
+
+ private static final long serialVersionUID = 2L;
+
+ protected final int vectorDimensions;
+ protected final int[] vectorFields;
+ protected final boolean atomic;
+ protected final IVTreeBinaryAccessorFactory vectorAccessorFactory;
+ protected final int numPrimaryKeyFields;
+ protected final int numIncludeFields;
+ protected final IVTreeDataTupleBuilderFactory dataTupleBuilderFactory;
+
+ /** Distance-function factory supplied at DDL time; threaded onto {@link LSMVTreeLocalResource} for persistence. */
+ protected final IVTreeDistanceFunctionFactory distanceFunctionFactory;
+
+ /** Cross-pollination placement config supplied at DDL time; threaded onto {@link LSMVTreeLocalResource}. */
+ protected final CrossPollinationConfig crossPollination;
+
+ public LSMVTreeLocalResourceFactory(IStorageManager storageManager, ITypeTraits[] typeTraits,
+ IBinaryComparatorFactory[] cmpFactories, ITypeTraits[] filterTypeTraits,
+ IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields,
+ ILSMOperationTrackerFactory opTrackerFactory, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
+ ILSMPageWriteCallbackFactory pageWriteCallbackFactory,
+ IMetadataPageManagerFactory metadataPageManagerFactory, IVirtualBufferCacheProvider vbcProvider,
+ ILSMIOOperationSchedulerProvider ioSchedulerProvider, ILSMMergePolicyFactory mergePolicyFactory,
+ Map<String, String> mergePolicyProperties, boolean durable, int vectorDimensions, int[] vectorFields,
+ ITypeTraits nullTypeTraits, INullIntrospector nullIntrospector, boolean atomic,
+ IVTreeBinaryAccessorFactory vectorAccessorFactory, int numPrimaryKeyFields, int numIncludeFields,
+ IVTreeDataTupleBuilderFactory dataTupleBuilderFactory,
+ IVTreeDistanceFunctionFactory distanceFunctionFactory, CrossPollinationConfig crossPollination) {
+ super(storageManager, typeTraits, cmpFactories, filterTypeTraits, filterCmpFactories, filterFields,
+ opTrackerFactory, ioOpCallbackFactory, pageWriteCallbackFactory, metadataPageManagerFactory,
+ vbcProvider, ioSchedulerProvider, mergePolicyFactory, mergePolicyProperties, durable, nullTypeTraits,
+ nullIntrospector);
+ this.vectorDimensions = vectorDimensions;
+ this.vectorFields = vectorFields;
+ this.atomic = atomic;
+ this.distanceFunctionFactory = distanceFunctionFactory;
+ this.crossPollination = crossPollination != null ? crossPollination : CrossPollinationConfig.LEGACY;
+ this.vectorAccessorFactory = vectorAccessorFactory;
+ this.numPrimaryKeyFields = numPrimaryKeyFields;
+ this.numIncludeFields = numIncludeFields;
+ this.dataTupleBuilderFactory = dataTupleBuilderFactory;
+ }
+
+ @Override
+ public IResource createResource(FileReference fileRef) {
+ return new LSMVTreeLocalResource(fileRef.getRelativePath(), storageManager, typeTraits, cmpFactories,
+ filterTypeTraits, filterCmpFactories, filterFields, opTrackerProvider, ioOpCallbackFactory,
+ pageWriteCallbackFactory, metadataPageManagerFactory, vbcProvider, ioSchedulerProvider,
+ mergePolicyFactory, mergePolicyProperties, durable, vectorDimensions, vectorFields, nullTypeTraits,
+ nullIntrospector, atomic, vectorAccessorFactory, numPrimaryKeyFields, numIncludeFields,
+ dataTupleBuilderFactory, distanceFunctionFactory, null, null, null, null, null, null, crossPollination);
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/dataflow/QuantizedIndexBuilder.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/dataflow/QuantizedIndexBuilder.java
new file mode 100644
index 0000000000..c923208d7b
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/dataflow/QuantizedIndexBuilder.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.vector.dataflow;
+
+import org.apache.hyracks.api.application.INCServiceContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.storage.am.common.build.IndexBuilder;
+import org.apache.hyracks.storage.am.vector.api.IQuantizedResource;
+import org.apache.hyracks.storage.am.vector.api.VTreeQuantizationParams;
+import org.apache.hyracks.storage.common.IResource;
+import org.apache.hyracks.storage.common.IResourceFactory;
+import org.apache.hyracks.storage.common.IResourceWrapper;
+import org.apache.hyracks.storage.common.IStorageManager;
+import org.apache.hyracks.storage.common.file.IResourceIdFactory;
+import org.apache.hyracks.util.annotations.AiProvenance;
+
+/**
+ * Vector-index-specific {@link IndexBuilder} that carries scalar-quantization constants and injects them
+ * into the freshly created resource at build time. Quantization is a vector concept, so the params and the
+ * {@link IQuantizedResource} injection live here rather than on the shared {@link IndexBuilder}, which builds
+ * every index type.
+ * <p>
+ * The constants are not known when the builder is constructed: they are sampled upstream and delivered to
+ * {@link #setQuantizationParameters(VTreeQuantizationParams)} by
+ * {@link QuantizedIndexCreateOperatorDescriptor} before
+ * {@link #build()} runs.
+ */
+@AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_4_8, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.GENERATED)
+public class QuantizedIndexBuilder extends IndexBuilder {
+
+ private VTreeQuantizationParams quantizationParams;
+
+ public QuantizedIndexBuilder(INCServiceContext ctx, IStorageManager storageManager,
+ IResourceIdFactory resourceIdFactory, FileReference resourceRef, IResourceFactory localResourceFactory,
+ boolean durable) throws HyracksDataException {
+ super(ctx, storageManager, resourceIdFactory, resourceRef, localResourceFactory, durable);
+ }
+
+ public void setQuantizationParameters(VTreeQuantizationParams quantizationParams) {
+ this.quantizationParams = quantizationParams;
+ }
+
+ @Override
+ protected void configureResource(IResource resource) throws HyracksDataException {
+ if (quantizationParams == null) {
+ return;
+ }
+ // The builder receives the DatasetLocalResource wrapper; the quantizable resource is the one it
+ // wraps. Unwrap generically (via IResourceWrapper) so the vector-only quantization concern stays
+ // out of the shared DatasetLocalResource.
+ IResource target =
+ (resource instanceof IResourceWrapper) ? ((IResourceWrapper) resource).getResource() : resource;
+ if (target instanceof IQuantizedResource) {
+ ((IQuantizedResource) target).setQuantizationParameters(quantizationParams);
+ }
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/dataflow/QuantizedIndexBuilderFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/dataflow/QuantizedIndexBuilderFactory.java
new file mode 100644
index 0000000000..412f55a2fa
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/dataflow/QuantizedIndexBuilderFactory.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.vector.dataflow;
+
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
+import org.apache.hyracks.storage.am.common.api.IIndexBuilder;
+import org.apache.hyracks.storage.am.common.build.IndexBuilderFactory;
+import org.apache.hyracks.storage.common.IResourceFactory;
+import org.apache.hyracks.storage.common.IStorageManager;
+import org.apache.hyracks.util.annotations.AiProvenance;
+
+/**
+ * Produces {@link QuantizedIndexBuilder}s for the quantized vector-index create path. The build ingredients
+ * are identical to {@link IndexBuilderFactory}; only the concrete builder type differs, so the quantization
+ * constants sampled upstream can be injected into the resource at build time.
+ */
+@AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_4_8, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.GENERATED)
+public class QuantizedIndexBuilderFactory extends IndexBuilderFactory {
+
+ private static final long serialVersionUID = 1L;
+
+ public QuantizedIndexBuilderFactory(IStorageManager storageManager, IFileSplitProvider fileSplitProvider,
+ IResourceFactory localResourceFactory, boolean durable) {
+ super(storageManager, fileSplitProvider, localResourceFactory, durable);
+ }
+
+ @Override
+ public IIndexBuilder create(IHyracksTaskContext ctx, int partition) throws HyracksDataException {
+ FileReference resourceRef = fileSplitProvider.getFileSplits()[partition].getFileReference(ctx.getIoManager());
+ return new QuantizedIndexBuilder(ctx.getJobletContext().getServiceContext(), storageManager,
+ storageManager.getResourceIdFactory(ctx.getJobletContext().getServiceContext()), resourceRef,
+ localResourceFactory, durable);
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/dataflow/QuantizedIndexCreateOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/dataflow/QuantizedIndexCreateOperatorDescriptor.java
new file mode 100644
index 0000000000..c7788e32bb
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/dataflow/QuantizedIndexCreateOperatorDescriptor.java
@@ -0,0 +1,180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.vector.dataflow;
+
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.dataflow.IOperatorNodePushable;
+import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
+import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
+import org.apache.hyracks.api.exceptions.ErrorCode;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
+import org.apache.hyracks.data.std.primitive.ByteArrayPointable;
+import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
+import org.apache.hyracks.dataflow.common.data.accessors.FrameTupleReference;
+import org.apache.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
+import org.apache.hyracks.dataflow.std.base.AbstractUnaryInputSinkOperatorNodePushable;
+import org.apache.hyracks.storage.am.common.api.IIndexBuilder;
+import org.apache.hyracks.storage.am.common.api.IIndexBuilderFactory;
+import org.apache.hyracks.storage.am.vector.api.VTreeQuantizationParams;
+
+/**
+ * Operator descriptor for creating a quantized vector index.
+ *
+ * Input: a single tuple containing the serialized quantization parameters
+ * (minQuantile, maxQuantile, alpha, bits, confidenceInterval, sampleCount).
+ * Output: none (sink). On close() the underlying {@link IIndexBuilder}s are built,
+ * propagating the extracted quantization parameters to each storage partition.
+ */
+public class QuantizedIndexCreateOperatorDescriptor extends AbstractSingleActivityOperatorDescriptor {
+
+ private static final long serialVersionUID = 1L;
+ private final IIndexBuilderFactory[][] indexBuilderFactories;
+ private final int[][] partitionsMap;
+
+ public QuantizedIndexCreateOperatorDescriptor(IOperatorDescriptorRegistry spec,
+ IIndexBuilderFactory[][] indexBuilderFactories, int[][] partitionsMap, RecordDescriptor recordDesc) {
+ super(spec, 1, 0);
+ this.indexBuilderFactories = indexBuilderFactories;
+ this.partitionsMap = partitionsMap;
+ }
+
+ @Override
+ public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
+ IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
+ int[] storagePartitions = partitionsMap[partition];
+ IIndexBuilderFactory[] partitionIndexBuilderFactories = indexBuilderFactories[partition];
+ IIndexBuilder[] indexBuilders = new IIndexBuilder[storagePartitions.length];
+ for (int i = 0; i < storagePartitions.length; i++) {
+ indexBuilders[i] = partitionIndexBuilderFactories[i].create(ctx, storagePartitions[i]);
+ }
+ RecordDescriptor inputRecordDesc = recordDescProvider.getInputRecordDescriptor(getActivityId(), 0);
+ return new QuantizedIndexCreateOperatorNodePushable(ctx, indexBuilders, inputRecordDesc);
+ }
+
+ private static class QuantizedIndexCreateOperatorNodePushable extends AbstractUnaryInputSinkOperatorNodePushable {
+ private final IIndexBuilder[] indexBuilders;
+ private final FrameTupleAccessor tupleAccessor;
+ private final FrameTupleReference tupleHelper;
+ private VTreeQuantizationParams quantizationParams;
+ private boolean failed;
+
+ public QuantizedIndexCreateOperatorNodePushable(IHyracksTaskContext ctx, IIndexBuilder[] indexBuilders,
+ RecordDescriptor inputRecordDesc) {
+ this.indexBuilders = indexBuilders;
+ this.tupleAccessor = new FrameTupleAccessor(inputRecordDesc);
+ this.tupleHelper = new FrameTupleReference();
+ }
+
+ @Override
+ public void open() throws HyracksDataException {
+ // No-op: index builders are constructed eagerly; build() is deferred until close().
+ }
+
+ @Override
+ public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ tupleAccessor.reset(buffer);
+ int tupleCount = tupleAccessor.getTupleCount();
+ if (tupleCount == 0) {
+ return;
+ }
+ // Exactly one quantization-params tuple is expected. Fail loudly on a second tuple/frame or a
+ // multi-tuple frame rather than silently dropping the extras (a wrong-input hazard).
+ if (quantizationParams != null) {
+ throw HyracksDataException.create(ErrorCode.ILLEGAL_STATE,
+ "QuantizedIndexCreateOperator received more than one quantization-params tuple");
+ }
+ if (tupleCount != 1) {
+ throw HyracksDataException.create(ErrorCode.ILLEGAL_STATE,
+ "QuantizedIndexCreateOperator expected a single quantization-params tuple but got "
+ + tupleCount);
+ }
+ tupleHelper.reset(tupleAccessor, 0);
+ quantizationParams = extractQuantizationParams(tupleHelper);
+ for (IIndexBuilder indexBuilder : indexBuilders) {
+ if (indexBuilder instanceof QuantizedIndexBuilder) {
+ ((QuantizedIndexBuilder) indexBuilder).setQuantizationParameters(quantizationParams);
+ }
+ }
+ }
+
+ // Six big-endian 4-byte values: minQuantile, maxQuantile, alpha (floats), bits (int),
+ // confidenceInterval (float), sampleCount (int).
+ private static final int PARAMS_PAYLOAD_BYTES = 6 * Integer.BYTES;
+
+ private VTreeQuantizationParams extractQuantizationParams(FrameTupleReference tuple)
+ throws HyracksDataException {
+ byte[] data = tuple.getFieldData(0);
+ int start = tuple.getFieldStart(0);
+ int length = tuple.getFieldLength(0);
+
+ // Skip the 1-byte ADM type tag, then the ByteArrayPointable's varlen length prefix.
+ ByteArrayPointable ptr = new ByteArrayPointable();
+ ptr.set(data, start + 1, length - 1);
+ int contentLength = ptr.getContentLength();
+ if (contentLength < PARAMS_PAYLOAD_BYTES) {
+ throw HyracksDataException.create(ErrorCode.ILLEGAL_STATE, "quantization-params payload is too short: "
+ + contentLength + " bytes, need at least " + PARAMS_PAYLOAD_BYTES);
+ }
+
+ // Big-endian; DataInputStream is the exact inverse of the DataOutput/ByteBuffer writer and
+ // bounds-checks (EOFException) instead of reading past the field.
+ try (DataInputStream in = new DataInputStream(
+ new ByteArrayInputStream(ptr.getByteArray(), ptr.getContentStartOffset(), contentLength))) {
+ float minQuantile = in.readFloat();
+ float maxQuantile = in.readFloat();
+ float alpha = in.readFloat();
+ int bits = in.readInt();
+ float confidenceInterval = in.readFloat();
+ int sampleCount = in.readInt();
+ return new VTreeQuantizationParams(minQuantile, maxQuantile, alpha, confidenceInterval, bits,
+ sampleCount);
+ } catch (IOException e) {
+ throw HyracksDataException.create(e);
+ }
+ }
+
+ @Override
+ public void fail() throws HyracksDataException {
+ // A failure upstream (e.g. the sampler producing the quantization constants) must not leave a
+ // half-built index: mark failed so close() skips build() rather than building from a failed
+ // or partial pipeline.
+ failed = true;
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ if (failed) {
+ return; // do not build from a failed pipeline; let the original failure propagate
+ }
+ if (quantizationParams != null) {
+ for (IIndexBuilder indexBuilder : indexBuilders) {
+ indexBuilder.build();
+ }
+ } else {
+ throw new HyracksDataException("QuantizedIndexCreateOperator: No quantization constants received.");
+ }
+ }
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/dataflow/VTreeSearchOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/dataflow/VTreeSearchOperatorDescriptor.java
new file mode 100644
index 0000000000..6377d04a3e
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/dataflow/VTreeSearchOperatorDescriptor.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.vector.dataflow;
+
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.dataflow.IOperatorNodePushable;
+import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
+import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
+import org.apache.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
+import org.apache.hyracks.storage.am.common.api.ISearchOperationCallbackFactory;
+import org.apache.hyracks.storage.am.common.api.ITupleFilterFactory;
+import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
+import org.apache.hyracks.storage.am.lsm.vector.impls.PKOnlyTupleProjectorFactory;
+import org.apache.hyracks.storage.am.vector.api.IVTreeBinaryAccessorFactory;
+import org.apache.hyracks.storage.am.vector.api.IVTreeDistanceFunctionFactory;
+import org.apache.hyracks.storage.am.vector.api.IVTreeQuantizerFactory;
+import org.apache.hyracks.storage.common.projection.ITupleProjectorFactory;
+
+/**
+ * Operator descriptor for vector index search (ANN search).
+ * This creates the runtime operator (VTreeSearchOperatorNodePushable) that performs
+ * the actual search on each node controller.
+ */
+public class VTreeSearchOperatorDescriptor extends AbstractSingleActivityOperatorDescriptor {
+
+ private static final long serialVersionUID = 1L;
+
+ // Field indexes in input tuple: [query_vector_field, k_field, metric_field]
+ protected final int[] queryFields;
+
+ // Factory to open LSMVTree index
+ protected final IIndexDataflowHelperFactory indexHelperFactory;
+
+ // Whether to retain input tuples in output
+ protected final boolean retainInput;
+
+ // Transaction callback factory
+ protected final ISearchOperationCallbackFactory searchCallbackFactory;
+
+ // Partition mapping (compute nodes to storage nodes)
+ protected final int[][] partitionsMap;
+
+ // Number of primary and secondary keys for tuple projection
+ protected final int numPrimaryKeys;
+ protected final int numSecondaryKeys;
+
+ // Tuple projector factory (extracts only PKs from index results)
+ protected final ITupleProjectorFactory tupleProjectorFactory;
+
+ // Factory for creating vector binary accessors (for extracting AOrderedList<ADouble>)
+ protected final IVTreeBinaryAccessorFactory vectorAccessorFactory;
+
+ // Factory for creating distance functions, injected from the AsterixDB layer via a Hyracks-side
+ // interface to keep this module free of AsterixDB type dependencies.
+ protected final IVTreeDistanceFunctionFactory distanceFunctionFactory;
+
+ // Factory for creating per-query quantizers from the float[6] params persisted on the tree.
+ // Provided by AsterixDB (OptimizedScalarQuantizerFactory). Nullable for non-quantized indexes.
+ protected final IVTreeQuantizerFactory quantizerFactory;
+
+ // Factory for creating tuple filters for INCLUDE field predicates (e.g., year > 2000)
+ // When set, the cursor will only return tuples that pass this filter
+ protected final ITupleFilterFactory tupleFilterFactory;
+
+ // Multiplier for candidate limit: K * kMultiplier candidates sent to PK for reranking
+ protected final int kMultiplier;
+
+ /** Epsilon from vector index WITH metadata (ANN / cluster search). */
+ protected final double indexEpsilon;
+
+ /**
+ * Index-only ANN plan flag. When {@code true}, the runtime emits one extra ADOUBLE field per
+ * tuple carrying {@code D(q,x)} (read from the cursor via {@code IVectorSearchCursor}), the
+ * downstream sort orders on that field, and the primary BTree lookup + rerank ASSIGN are skipped
+ * by the optimizer. When {@code false} the legacy lookup-and-rerank path is used.
+ */
+ protected final boolean indexOnly;
+
+ public VTreeSearchOperatorDescriptor(IOperatorDescriptorRegistry spec, RecordDescriptor outRecDesc,
+ int[] queryFields, IIndexDataflowHelperFactory indexHelperFactory, boolean retainInput,
+ ISearchOperationCallbackFactory searchCallbackFactory, IVTreeBinaryAccessorFactory vectorAccessorFactory,
+ IVTreeDistanceFunctionFactory distanceFunctionFactory, IVTreeQuantizerFactory quantizerFactory,
+ int[][] partitionsMap, int numPrimaryKeys, int numSecondaryKeys, ITupleFilterFactory tupleFilterFactory,
+ int kMultiplier, double indexEpsilon, boolean indexOnly) {
+ super(spec, 1, 1); // 1 input, 1 output
+ this.queryFields = queryFields;
+ this.indexHelperFactory = indexHelperFactory;
+ this.retainInput = retainInput;
+ this.searchCallbackFactory = searchCallbackFactory;
+ this.vectorAccessorFactory = vectorAccessorFactory;
+ this.distanceFunctionFactory = distanceFunctionFactory;
+ this.quantizerFactory = quantizerFactory;
+ this.partitionsMap = partitionsMap;
+ this.numPrimaryKeys = numPrimaryKeys;
+ this.numSecondaryKeys = numSecondaryKeys;
+ this.tupleFilterFactory = tupleFilterFactory;
+ this.kMultiplier = kMultiplier;
+ this.indexEpsilon = indexEpsilon;
+ this.indexOnly = indexOnly;
+ this.outRecDescs[0] = outRecDesc;
+
+ // Create tuple projector factory that extracts only PK fields
+ // This avoids writing large embedding vectors (4KB-16KB) to output frames
+ this.tupleProjectorFactory = new PKOnlyTupleProjectorFactory(numSecondaryKeys, numPrimaryKeys);
+ }
+
+ @Override
+ public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
+ IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
+ return new VTreeSearchOperatorNodePushable(ctx, partition,
+ recordDescProvider.getInputRecordDescriptor(getActivityId(), 0), queryFields, indexHelperFactory,
+ retainInput, searchCallbackFactory, tupleProjectorFactory, vectorAccessorFactory,
+ distanceFunctionFactory, quantizerFactory, partitionsMap, tupleFilterFactory, kMultiplier, indexEpsilon,
+ numPrimaryKeys, indexOnly);
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/dataflow/VTreeSearchOperatorNodePushable.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/dataflow/VTreeSearchOperatorNodePushable.java
new file mode 100644
index 0000000000..6e0d5f7091
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/dataflow/VTreeSearchOperatorNodePushable.java
@@ -0,0 +1,334 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.vector.dataflow;
+
+import java.io.IOException;
+
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.util.HyracksConstants;
+import org.apache.hyracks.data.std.primitive.DoublePointable;
+import org.apache.hyracks.data.std.primitive.IntegerPointable;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.dataflow.common.data.accessors.PermutingFrameTupleReference;
+import org.apache.hyracks.storage.am.common.api.ISearchOperationCallbackFactory;
+import org.apache.hyracks.storage.am.common.api.ITupleFilter;
+import org.apache.hyracks.storage.am.common.api.ITupleFilterFactory;
+import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
+import org.apache.hyracks.storage.am.common.dataflow.IndexSearchOperatorNodePushable;
+import org.apache.hyracks.storage.am.lsm.vector.impls.IVectorSearchCursor;
+import org.apache.hyracks.storage.am.lsm.vector.impls.LSMVTreeTopKSearchCursor;
+import org.apache.hyracks.storage.am.vector.api.IVTreeBinaryAccessorFactory;
+import org.apache.hyracks.storage.am.vector.api.IVTreeDistanceFunctionFactory;
+import org.apache.hyracks.storage.am.vector.api.IVTreeQuantizerFactory;
+import org.apache.hyracks.storage.am.vector.impls.VTreeSearchPredicate;
+import org.apache.hyracks.storage.common.IIndex;
+import org.apache.hyracks.storage.common.IIndexAccessParameters;
+import org.apache.hyracks.storage.common.IIndexCursor;
+import org.apache.hyracks.storage.common.ISearchPredicate;
+import org.apache.hyracks.storage.common.projection.ITupleProjectorFactory;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+/**
+ * Runtime operator for vector index search (ANN search).
+ * Extends IndexSearchOperatorNodePushable which handles the heavy lifting:
+ * - Opening/closing indexes
+ * - Frame/tuple iteration
+ * - Output buffering
+ * - Transaction callbacks
+ *
+ * This class implements the schema-agnostic pattern using IVTreeBinaryAccessor
+ * to abstract over different vector serialization formats (e.g., AOrderedList).
+ *
+ * This class only needs to implement:
+ * 1. createSearchPredicate() - Create a VTreeSearchPredicate (query passed via IAP, not deserialized here)
+ * 2. resetSearchPredicate() - Point the predicate at the current input tuple
+ * 3. getFieldCount() - Return number of output fields
+ * 4. addAdditionalIndexAccessorParams() - Add vector-specific params (accessor/distance/quantizer factories)
+ */
+public class VTreeSearchOperatorNodePushable extends IndexSearchOperatorNodePushable {
+
+ private static final Logger LOGGER = LogManager.getLogger();
+
+ // Field indexes in input tuple: [query_vector_field, k_field, metric_field]
+ protected final int[] queryFields;
+
+ // Factory for creating vector accessors (passed from AsterixDB layer)
+ protected final IVTreeBinaryAccessorFactory vectorAccessorFactory;
+
+ // Factory for creating distance functions (injected from the AsterixDB layer)
+ protected final IVTreeDistanceFunctionFactory distanceFunctionFactory;
+
+ // Factory for creating per-query quantizers (passed from AsterixDB layer). Nullable for
+ // non-quantized indexes and for test contexts that pre-inject a pre-built IVTreeQuantizer.
+ protected final IVTreeQuantizerFactory quantizerFactory;
+
+ // Tuple reference for extracting query parameters
+ protected PermutingFrameTupleReference queryParamsTuple;
+
+ // Factory for creating tuple filters for INCLUDE field predicates (e.g., year > 2000)
+ // When set, the filter is pushed down to the cursor level for proper K counting
+ protected final ITupleFilterFactory tupleFilterFactory;
+
+ // The actual tuple filter, created from the factory
+ protected ITupleFilter tupleFilter;
+
+ // Multiplier for candidate limit: K * kMultiplier candidates sent to PK for reranking
+ protected final int kMultiplier;
+
+ /** Epsilon from vector index metadata (default 0.3 when absent in catalog). */
+ protected final double indexEpsilon;
+
+ /**
+ * Number of primary-key fields in the dataset. Needed by {@link #getFieldCount} and by the
+ * index-only write path to know where the projector's PK bytes end before appending the
+ * distance field.
+ */
+ protected final int numPrimaryKeys;
+
+ /**
+ * Index-only ANN flag. When true the pushable emits {@code [pk..., D(q,x)]} per candidate by
+ * stashing the active cursor and reaching into {@link IVectorSearchCursor} for the per-tuple
+ * distance, then appending it as an ADOUBLE after the PK bytes the existing PKOnlyTupleProjector
+ * writes. When false the pushable behaves exactly as before.
+ */
+ protected final boolean indexOnly;
+
+ /**
+ * Stashed by {@link #writeSearchResults} so the overridden {@link #writeTupleToOutput} can read
+ * the per-tuple distance from the cursor that produced the current tuple.
+ */
+ private IIndexCursor activeCursor;
+
+ public VTreeSearchOperatorNodePushable(IHyracksTaskContext ctx, int partition, RecordDescriptor inputRecDesc,
+ int[] queryFields, IIndexDataflowHelperFactory indexHelperFactory, boolean retainInput,
+ ISearchOperationCallbackFactory searchCallbackFactory, ITupleProjectorFactory projectorFactory,
+ IVTreeBinaryAccessorFactory vectorAccessorFactory, IVTreeDistanceFunctionFactory distanceFunctionFactory,
+ IVTreeQuantizerFactory quantizerFactory, int[][] partitionsMap, ITupleFilterFactory tupleFilterFactory,
+ int kMultiplier, double indexEpsilon, int numPrimaryKeys, boolean indexOnly) throws HyracksDataException {
+ // Vector search does its filtering in the cursor, so the operator passes no filter fields,
+ // tuple filter, output limit, or search-callback proceed result (see the args below).
+ super(ctx, inputRecDesc, partition, null, // minFilterFieldIndexes
+ null, // maxFilterFieldIndexes
+ indexHelperFactory, retainInput, false, // retainMissing
+ null, // nonMatchWriterFactory
+ searchCallbackFactory, false, // appendIndexFilter
+ null, // nonFilterWriterFactory
+ null, // tupleFilterFactory - we handle this at cursor level, not operator level
+ -1, // outputLimit
+ false, // appendOpCallbackProceedResult
+ null, // searchCallbackProceedResultFalseValue
+ null, // searchCallbackProceedResultTrueValue
+ projectorFactory, // ← PKOnlyTupleProjectorFactory (extracts only PK fields)
+ null, // tuplePartitionerFactory
+ partitionsMap);
+
+ this.queryFields = queryFields;
+ this.vectorAccessorFactory = vectorAccessorFactory;
+ this.distanceFunctionFactory = distanceFunctionFactory;
+ this.quantizerFactory = quantizerFactory;
+ this.tupleFilterFactory = tupleFilterFactory;
+ this.kMultiplier = kMultiplier;
+ this.indexEpsilon = indexEpsilon;
+ this.numPrimaryKeys = numPrimaryKeys;
+ this.indexOnly = indexOnly;
+
+ // Setup permuting tuple reference to extract query parameters
+ if (queryFields != null && queryFields.length > 0) {
+ queryParamsTuple = new PermutingFrameTupleReference();
+ queryParamsTuple.setFieldPermutation(queryFields);
+ }
+ }
+
+ @Override
+ public void open() throws HyracksDataException {
+ super.open();
+
+ // Create tuple filter from factory if available
+ // This filter is pushed down to the cursor level for proper K counting
+ if (tupleFilterFactory != null) {
+ tupleFilter = tupleFilterFactory.createTupleFilter(ctx);
+ }
+ }
+
+ @Override
+ protected ISearchPredicate createSearchPredicate(IIndex index) {
+ // Create simple marker predicate
+ // The actual query vector is passed via IIndexAccessParameters in addAdditionalIndexAccessorParams()
+ return new VTreeSearchPredicate();
+ }
+
+ // Field layout of the query-parameters tuple, positionally packed by the Asterix optimizer side:
+ // [0]=query vector, [1]=K, [2]=metric, [3]=min_probe_fraction, [4]=k_multiplier. The metric (position 2)
+ // is resolved during compilation and is not read at runtime here, so this method reads 0,1,3,4 and skips 2.
+ private static final int QP_FIELD_QUERY_VECTOR = 0;
+ private static final int QP_FIELD_K = 1;
+ private static final int QP_FIELD_MIN_PROBE_FRACTION = 3;
+ private static final int QP_FIELD_K_MULTIPLIER = 4;
+
+ @Override
+ protected void resetSearchPredicate(int tupleIndex) {
+ // Update queryParamsTuple to point to current input tuple
+ if (queryParamsTuple != null) {
+ queryParamsTuple.reset(accessor, tupleIndex);
+
+ // Update predicate with current tuple reference
+ // Following RTree pattern: predicate holds reference, updated per-tuple
+ VTreeSearchPredicate vectorPred = (VTreeSearchPredicate) searchPred;
+ vectorPred.setQueryTuple(queryParamsTuple);
+ vectorPred.setQueryFieldIndex(QP_FIELD_QUERY_VECTOR);
+
+ // Extract K value if available (+1 to skip the ADM type tag)
+ if (queryFields.length > QP_FIELD_K) {
+ int k = IntegerPointable.getInteger(queryParamsTuple.getFieldData(QP_FIELD_K),
+ queryParamsTuple.getFieldStart(QP_FIELD_K) + 1);
+ vectorPred.setK(k);
+ }
+
+ // Extract min_probe_fraction (double, +1 to skip type tag).
+ // Fraction of leaf clusters to probe (0.0-1.0). 0 means use default (0.1).
+ if (queryFields.length > QP_FIELD_MIN_PROBE_FRACTION) {
+ double minProbeFraction =
+ DoublePointable.getDouble(queryParamsTuple.getFieldData(QP_FIELD_MIN_PROBE_FRACTION),
+ queryParamsTuple.getFieldStart(QP_FIELD_MIN_PROBE_FRACTION) + 1);
+ if (minProbeFraction > 0.0) {
+ vectorPred.setMinProbeFraction(minProbeFraction);
+ }
+ }
+
+ // Extract k_multiplier (int, +1 to skip type tag)
+ if (queryFields.length > QP_FIELD_K_MULTIPLIER) {
+ int queryKMult = IntegerPointable.getInteger(queryParamsTuple.getFieldData(QP_FIELD_K_MULTIPLIER),
+ queryParamsTuple.getFieldStart(QP_FIELD_K_MULTIPLIER) + 1);
+ vectorPred.setKMultiplier(Math.max(1, queryKMult));
+ }
+
+ // Set tuple filter for INCLUDE field predicates (e.g., year > 2000)
+ // This filter is applied at cursor level for proper K counting
+ if (tupleFilter != null) {
+ vectorPred.setTupleFilter(tupleFilter);
+ }
+
+ // Session config compiler.vector.kmultiplier overrides query arg if set (kMultiplier > 1 from constructor)
+ if (kMultiplier > 1) {
+ vectorPred.setKMultiplier(kMultiplier);
+ }
+
+ vectorPred.setEpsilon(indexEpsilon);
+ }
+ }
+
+ /**
+ * Serialized ADOUBLE type tag (value 12), matching {@code ATypeTag.DOUBLE.serialize()} in
+ * {@code asterix-om}. Hardcoded here because Hyracks cannot depend on asterix-om; the Asterix
+ * optimizer side that builds this operator chain also declares the corresponding algebra
+ * variable as BuiltinType.ADOUBLE, so the on-wire convention stays in sync.
+ */
+ private static final byte ADOUBLE_TYPE_TAG = 12;
+
+ @Override
+ protected int getFieldCount(IIndex index) {
+ // numPrimaryKeys is supplied by the descriptor from dataset metadata (no more hardcoded 1).
+ // Index-only mode appends one extra ADOUBLE field per emitted tuple (the per-candidate
+ // D(q,x) read from the cursor in writeTupleToOutput).
+ return numPrimaryKeys + (indexOnly ? 1 : 0);
+ }
+
+ /**
+ * Stash the cursor that produced the tuples for this input row so {@link #writeTupleToOutput}
+ * can reach into it for the per-tuple {@code D(q,x)} in index-only mode. Cleared on exit so the
+ * stash never outlives the call that owns the cursor.
+ */
+ @Override
+ protected void writeSearchResults(int tupleIndex, IIndexCursor cursor) throws Exception {
+ this.activeCursor = cursor;
+ try {
+ super.writeSearchResults(tupleIndex, cursor);
+ } finally {
+ this.activeCursor = null;
+ }
+ }
+
+ /**
+ * Two responsibilities:
+ * <ol>
+ * <li>Always run the configured projector (the {@code PKOnlyTupleProjector}) so the PK bytes
+ * from the cursor tuple land in {@code tb} exactly as in the legacy path.</li>
+ * <li>In index-only mode, append the distance field after the PK bytes by reading
+ * {@code D(q,x)} from {@link #activeCursor} via {@link IVectorSearchCursor}. The distance
+ * is written as {@code [ADOUBLE type tag (1 byte), IEEE-754 double (8 bytes)]} to match the
+ * algebra-side variable type ({@code BuiltinType.ADOUBLE}). Search always uses the pruned top-K
+ * cursor (the only {@link IVectorSearchCursor}), so the cast is safe.</li>
+ * </ol>
+ *
+ * <p>A genuine {@link Double#NaN} distance (e.g. a zero-magnitude vector under cosine) is a real value
+ * and flows through; it is not treated as an error. {@code getCurrentDistance()} is valid for the current
+ * tuple both before and after projection (projection does not advance the cursor).
+ */
+ @Override
+ protected ITupleReference writeTupleToOutput(ITupleReference tuple) throws IOException {
+ double dqx = Double.NaN;
+ if (indexOnly) {
+ // Search always uses the pruned top-K cursor (addAdditionalIndexAccessorParams sets
+ // USE_TOPK_SEARCH), which is the only IVectorSearchCursor; the streaming LSMVTreeSearchCursor
+ // serves merges, full scans, and tests and never backs an index-only plan. A genuine NaN here
+ // (e.g. the cosine distance of a zero-magnitude vector) is a real value and flows through.
+ dqx = ((IVectorSearchCursor) activeCursor).getCurrentDistance();
+ }
+ ITupleReference projected = tupleProjector.project(tuple, dos, tb);
+ if (projected == null) {
+ return null;
+ }
+ if (indexOnly) {
+ dos.writeByte(ADOUBLE_TYPE_TAG);
+ dos.writeDouble(dqx);
+ tb.addFieldEndOffset();
+ }
+ return projected;
+ }
+
+ @Override
+ protected void addAdditionalIndexAccessorParams(IIndexAccessParameters iap) {
+ // Vector accessor factory: storage layer uses this to extract the query vector from the
+ // search predicate's tuple, keeping the extraction in the storage layer (no AsterixDB types
+ // leak down).
+ iap.getParameters().put(IVTreeBinaryAccessorFactory.IAP_KEY, vectorAccessorFactory);
+
+ // Distance-function factory injected from AsterixDB so the VTree can build an
+ // IVTreeDistanceFunction without depending on asterix-runtime types.
+ iap.getParameters().put(IVTreeDistanceFunctionFactory.IAP_KEY, distanceFunctionFactory);
+
+ // Quantizer factory (nullable). The VTree builds a per-query IVTreeQuantizer from the
+ // float[6] params persisted on the index. Null for non-quantized indexes and for test
+ // contexts that inject a pre-built IVTreeQuantizer under IVTreeQuantizer.IAP_KEY.
+ if (quantizerFactory != null) {
+ iap.getParameters().put(IVTreeQuantizerFactory.IAP_KEY, quantizerFactory);
+ }
+
+ // Task context for the spillable top-K buffer (follows inverted-index pattern).
+ iap.getParameters().put(HyracksConstants.HYRACKS_TASK_CONTEXT, ctx);
+
+ // Production ANN search always uses the quantized top-K cursor. Without this flag,
+ // LSMVTreeIndexAccessor defaults to the streaming LSMVTreeSearchCursor (used by component
+ // merges and by test fixtures that verify through full-scan iteration).
+ iap.getParameters().put(LSMVTreeTopKSearchCursor.IAP_KEY, Boolean.TRUE);
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/IClusterSelectionStrategy.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/IClusterSelectionStrategy.java
new file mode 100644
index 0000000000..afff9de79e
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/IClusterSelectionStrategy.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.lsm.vector.impls;
+
+import java.util.Set;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.vector.api.IVTreeDistanceFunction;
+import org.apache.hyracks.storage.am.vector.api.IVTreeQuantizer;
+import org.apache.hyracks.storage.am.vector.impls.ClusterSearchResult;
+import org.apache.hyracks.storage.am.vector.impls.VTree;
+import org.apache.hyracks.storage.am.vector.impls.VTreeSearchCursor;
+
+/**
+ * Strategy interface for cluster selection in multi-cluster vector search.
+ *
+ * Encapsulates the logic for determining which clusters to explore and when to stop.
+ * This allows different search strategies (nprobe-based, beam search, adaptive) to be
+ * plugged into the LSM cursor without modifying the cursor itself.
+ */
+public interface IClusterSelectionStrategy {
+
+ /**
+ * Initialize the strategy with search context.
+ * Called once at the beginning of a search.
+ *
+ * @param vTree The VTree (first component's tree for level-wise computation)
+ * @param queryVector The query vector
+ * @param distFunc Distance function
+ * @param k Target number of results
+ * @throws HyracksDataException if initialization fails
+ */
+ void initialize(VTree vTree, double[] queryVector, IVTreeDistanceFunction distFunc, int k)
+ throws HyracksDataException;
+
+ /**
+ * Get the next cluster to explore.
+ * Returns clusters based on the strategy (e.g., level-wise first, then DFS fallback).
+ *
+ * @return ClusterSearchResult with directoryPageId for O(1) access, or null if no more clusters
+ * @throws HyracksDataException if an error occurs
+ */
+ ClusterSearchResult getNextCluster() throws HyracksDataException;
+
+ /**
+ * Check if there are more clusters available to explore.
+ *
+ * @return true if more clusters can be retrieved via getNextCluster()
+ */
+ boolean hasMoreClusters();
+
+ /**
+ * Check if we should stop advancing to next cluster based on results collected.
+ *
+ * @param minClustersExplored Minimum clusters explored across all components
+ * @param resultsCollected Number of reconciled results collected so far
+ * @return true if we should stop advancing (e.g., nprobe satisfied AND K reached)
+ */
+ boolean shouldStopAdvancing(int minClustersExplored, int resultsCollected);
+
+ /**
+ * Get the shared visited set for cross-component deduplication.
+ * This set is passed to all component cursors to avoid revisiting clusters.
+ *
+ * @return Set of visited centroid IDs
+ */
+ Set<Integer> getVisitedCentroidIds();
+
+ /**
+ * Set the first cursor for DFS fallback.
+ * Called after all component cursors are opened.
+ * The first cursor maintains NavigationState for DFS traversal.
+ *
+ * @param firstCursor First component's VTreeSearchCursor
+ */
+ void setFirstCursorForDFS(VTreeSearchCursor firstCursor);
+
+ /**
+ * Get the first cluster result (computed during initialization).
+ * Used to synchronize all component cursors to the same initial cluster.
+ *
+ * @return The first cluster to explore, or null if not yet computed
+ */
+ ClusterSearchResult getFirstCluster();
+
+ /**
+ * Set quantizer state for computing quantized D(q,C) in ClusterSearchResult.
+ * Call before initialize() so that level-wise navigation can populate
+ * ClusterSearchResult.quantizedDistance for each leaf centroid found.
+ *
+ * @param quantizedQueryVector Dequantized form of query vector (nullable — pass null to skip)
+ * @param quantizer Quantizer for dequantizing leaf centroid bytes (nullable — pass null to skip)
+ */
+ void setQuantizer(double[] quantizedQueryVector, IVTreeQuantizer quantizer);
+
+ /**
+ * Hand over the greedy-DFS seed cluster that the level-wise re-open displaced, so the strategy can
+ * probe it once after the level-wise phase. {@code initializeClusterIterator} already consumed it
+ * from the DFS iterator and marked it visited, so otherwise it is never scanned and its records are
+ * dropped (with {@code cross_pollination_m=1} there is no replica to cover them).
+ *
+ * @param seed the displaced seed cluster, or {@code null} to clear
+ */
+ void deferSeedCluster(ClusterSearchResult seed);
+
+ /**
+ * Reset the strategy for a new search.
+ * Called when the cursor is reopened.
+ */
+ void reset();
+
+ /**
+ * Get the number of level-wise clusters computed.
+ * Used for logging/debugging.
+ *
+ * @return Number of level-wise clusters, or 0 if not computed
+ */
+ int getLevelWiseClusterCount();
+
+ /**
+ * Check if level-wise phase is complete.
+ * Used for logging/debugging.
+ *
+ * @return true if level-wise exploration is done
+ */
+ boolean isLevelWisePhaseComplete();
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/IVectorSearchCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/IVectorSearchCursor.java
new file mode 100644
index 0000000000..322de6836a
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/IVectorSearchCursor.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.vector.impls;
+
+/**
+ * Capability interface for vector ANN search cursors that expose the per-candidate distance
+ * to the query vector (the {@code D(q,x)} value the cursor used during its top-K search).
+ *
+ * <p>Required by the index-only ANN plan, where the operator emits {@code [pk..., dist]} to its
+ * output frame and the downstream {@code ORDER BY $$dist → LIMIT k} avoids the primary BTree
+ * lookup and the rerank ASSIGN. The cursor's emitted tuple still carries only the index's
+ * on-disk fields (where field 0 is the distance-to-centroid, not the query distance), so the
+ * operator obtains the per-candidate {@code D(q,x)} via {@link #getCurrentDistance()} and
+ * appends it as a separate ADOUBLE field.
+ *
+ * <p>{@link #getCurrentDistance()} returns the distance for the tuple currently held by the
+ * cursor (the one that {@code cursor.getTuple()} would return). It must be called between
+ * {@code cursor.next()} and the next {@code cursor.next()} or {@code cursor.close()}.
+ *
+ * <p>Implementing this interface <em>is</em> the capability signal: a cursor implements it iff it computes
+ * a per-candidate query distance. Cursors that only order by distance-to-centroid (the streaming non-pruned
+ * {@code LSMVTreeSearchCursor}, used for merges/full scans/tests) simply do not implement it, so the
+ * index-only ANN plan detects the unsupported case with a plain {@code instanceof} check. A NaN from
+ * {@link #getCurrentDistance()} is therefore never a "distance unavailable" signal; it is a genuine value
+ * (e.g. the cosine distance of a zero-magnitude vector).
+ */
+public interface IVectorSearchCursor {
+
+ /**
+ * @return the distance from the query vector to the candidate vector for the current tuple. This may be
+ * {@link Double#NaN} as a genuine value (e.g. the cosine distance of a zero-magnitude vector).
+ */
+ double getCurrentDistance();
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/LSMVTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/LSMVTree.java
new file mode 100644
index 0000000000..4abf63d3ac
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/LSMVTree.java
@@ -0,0 +1,718 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.lsm.vector.impls;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import org.apache.hyracks.api.exceptions.ErrorCode;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.IIOManager;
+import org.apache.hyracks.api.util.CleanupUtils;
+import org.apache.hyracks.control.common.controllers.NCConfig;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.storage.am.common.api.IExtendedModificationOperationCallback;
+import org.apache.hyracks.storage.am.common.api.IIndexOperationContext;
+import org.apache.hyracks.storage.am.common.api.IPageManager;
+import org.apache.hyracks.storage.am.common.api.ITreeIndex;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexMetadataFrame;
+import org.apache.hyracks.storage.am.common.impls.NoOpIndexAccessParameters;
+import org.apache.hyracks.storage.am.lsm.common.api.IComponentFilterHelper;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFrameFactory;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentBulkLoader;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentFactory;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackFactory;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMMemoryComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMPageWriteCallbackFactory;
+import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
+import org.apache.hyracks.storage.am.lsm.common.freepage.VirtualFreePageManager;
+import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndex;
+import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexOperationContext;
+import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
+import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFilterManager;
+import org.apache.hyracks.storage.am.lsm.common.impls.LSMIndexDiskComponentBulkLoader;
+import org.apache.hyracks.storage.am.lsm.common.impls.LSMTreeIndexAccessor.ICursorFactory;
+import org.apache.hyracks.storage.am.lsm.common.impls.LSMVTreeComponentFileReferences;
+import org.apache.hyracks.storage.am.lsm.common.impls.LoadOperation;
+import org.apache.hyracks.storage.am.vector.api.IVTreeBinaryAccessorFactory;
+import org.apache.hyracks.storage.am.vector.api.IVTreeDataTupleBuilderFactory;
+import org.apache.hyracks.storage.am.vector.api.IVTreeDistanceFunctionFactory;
+import org.apache.hyracks.storage.am.vector.api.VTreeQuantizationParams;
+import org.apache.hyracks.storage.am.vector.impls.VTree;
+import org.apache.hyracks.storage.am.vector.impls.VTreeFlushLoader;
+import org.apache.hyracks.storage.am.vector.impls.VTreeSearchPredicate;
+import org.apache.hyracks.storage.am.vector.utils.CrossPollinationConfig;
+import org.apache.hyracks.storage.common.IIndexAccessParameters;
+import org.apache.hyracks.storage.common.IIndexBulkLoader;
+import org.apache.hyracks.storage.common.IIndexCursor;
+import org.apache.hyracks.storage.common.IIndexCursorStats;
+import org.apache.hyracks.storage.common.ISearchPredicate;
+import org.apache.hyracks.storage.common.NoOpIndexCursorStats;
+import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.buffercache.ICachedPage;
+import org.apache.hyracks.storage.common.buffercache.IPageWriteCallback;
+import org.apache.hyracks.util.annotations.AiProvenance;
+import org.apache.hyracks.util.trace.ITracer;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+/**
+ * LSM Vector Clustering Tree. Uses {@link VTree} as both in-memory components and disk components.
+ * The static structure (root + interior + leaf navigation tree) is built once at CREATE INDEX time
+ * and stored in its own {@link LSMVTreeDiskComponent}; memory components navigate via a shared
+ * accessor onto that static-structure component, while each flushed/bulk-loaded disk component
+ * carries its own copy of the navigation pages. The per-cluster metadata ("directory") pages and
+ * the data pages are always component-local.
+ */
+public class LSMVTree extends AbstractLSMIndex implements ITreeIndex {
+ private static final Logger LOGGER = LogManager.getLogger();
+
+ private static final ICursorFactory cursorFactory = LSMVTreeSearchCursor::new;
+
+ protected final ITreeIndexFrameFactory interiorFrameFactory;
+ protected final ITreeIndexFrameFactory leafFrameFactory;
+ protected final ITreeIndexFrameFactory metadataFrameFactory;
+ protected final ITreeIndexFrameFactory insertDataFrameFactory;
+ protected final ITreeIndexFrameFactory deleteDataFrameFactory;
+
+ protected final IBinaryComparatorFactory[] cmpFactories;
+ protected final int vectorDimensions;
+ protected final IVTreeBinaryAccessorFactory vectorAccessorFactory;
+
+ // Data tuple format depends on quantization (see VTreeDataTupleBuilder):
+ // non-quantized: [distance, centroidId, primary_keys..., include_fields...] (pkStartField=2)
+ // quantized: [distance, centroidId, quantized_distance, quantized_embedding,
+ // primary_keys..., include_fields...] (pkStartField=4)
+ // Quantization is enforced at index creation in this build, so the quantized form is the default.
+ protected final int numPrimaryKeyFields;
+ protected final int numIncludeFields;
+ protected final IVTreeDataTupleBuilderFactory dataTupleBuilderFactory;
+
+ // Raw quantization params for lazy quantizer creation at query time (null = non-quantized path)
+ protected final VTreeQuantizationParams quantizationParams;
+ protected final IVTreeDistanceFunctionFactory distanceFunctionFactory;
+ protected final CrossPollinationConfig crossPollination;
+
+ // volatile: written under synchronized setStaticStructure/activation but read unsynchronized on the
+ // search/flush path; the volatile publishes the reference safely to those readers.
+ protected volatile LSMVTreeDiskComponent staticStructure;
+
+ public LSMVTree(NCConfig storageConfig, IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches,
+ ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory leafFrameFactory,
+ ITreeIndexFrameFactory metadataFrameFactory, ITreeIndexFrameFactory insertDataFrameFactory,
+ ITreeIndexFrameFactory deleteDataFrameFactory, IBufferCache diskBufferCache,
+ ILSMIndexFileManager fileManager, ILSMDiskComponentFactory componentFactory,
+ ILSMDiskComponentFactory bulkLoadComponentFactory, IComponentFilterHelper filterHelper,
+ ILSMComponentFilterFrameFactory filterFrameFactory, LSMComponentFilterManager filterManager,
+ double bloomFilterFalsePositiveRate, IBinaryComparatorFactory[] cmpFactories, ILSMMergePolicy mergePolicy,
+ ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
+ ILSMIOOperationCallbackFactory ioOpCallbackFactory, ILSMPageWriteCallbackFactory pageWriteCallbackFactory,
+ int vectorDimensions, int[] vectorFields, int[] filterFields, boolean durable, boolean atomic,
+ IVTreeBinaryAccessorFactory vectorAccessorFactory, int numPrimaryKeyFields, int numIncludeFields,
+ IVTreeDataTupleBuilderFactory dataTupleBuilderFactory, VTreeQuantizationParams quantizationParams,
+ IVTreeDistanceFunctionFactory distanceFunctionFactory, CrossPollinationConfig crossPollination)
+ throws HyracksDataException {
+
+ super(storageConfig, ioManager, virtualBufferCaches, diskBufferCache, fileManager, bloomFilterFalsePositiveRate,
+ mergePolicy, opTracker, ioScheduler, ioOpCallbackFactory, pageWriteCallbackFactory, componentFactory,
+ bulkLoadComponentFactory, filterFrameFactory, filterManager, filterFields, durable, filterHelper,
+ vectorFields, ITracer.NONE, atomic);
+ this.interiorFrameFactory = interiorFrameFactory;
+ this.leafFrameFactory = leafFrameFactory;
+ this.metadataFrameFactory = metadataFrameFactory;
+ this.insertDataFrameFactory = insertDataFrameFactory;
+ this.deleteDataFrameFactory = deleteDataFrameFactory;
+ this.cmpFactories = cmpFactories;
+ this.vectorDimensions = vectorDimensions;
+ this.vectorAccessorFactory = vectorAccessorFactory;
+ this.numPrimaryKeyFields = numPrimaryKeyFields;
+ this.numIncludeFields = numIncludeFields;
+ this.dataTupleBuilderFactory = dataTupleBuilderFactory;
+ this.quantizationParams = quantizationParams;
+ this.distanceFunctionFactory = distanceFunctionFactory;
+ this.crossPollination = crossPollination != null ? crossPollination : CrossPollinationConfig.LEGACY;
+
+ int i = 0;
+ for (IVirtualBufferCache virtualBufferCache : virtualBufferCaches) {
+ String baseDirPath = fileManager.getBaseDir() + "_virtual_" + i;
+ FileReference virtualFileRef = ioManager.resolve(baseDirPath);
+ // Memory components use insertDataFrameFactory for normal inserts
+ VTree vTree = new VTree(virtualBufferCache, new VirtualFreePageManager(virtualBufferCache),
+ interiorFrameFactory, leafFrameFactory, metadataFrameFactory, insertDataFrameFactory, cmpFactories,
+ 1, vectorDimensions, virtualFileRef, vectorAccessorFactory, dataTupleBuilderFactory,
+ quantizationParams, this.distanceFunctionFactory, this.crossPollination);
+ LSMVTreeMemoryComponent mutableComponent = new LSMVTreeMemoryComponent(this, vTree, virtualBufferCache,
+ filterHelper == null ? null : filterHelper.createFilter());
+ memoryComponents.add(mutableComponent);
+ i++;
+ }
+ }
+
+ public void setStaticStructure(LSMVTreeDiskComponent staticStructure) throws HyracksDataException {
+ this.staticStructure = staticStructure;
+ staticStructure.setInitialized();
+ if (memoryComponentsAllocated) {
+ initializeMemoryComponentsStaticStructure();
+ }
+ }
+
+ @Override
+ public synchronized void allocateMemoryComponents() throws HyracksDataException {
+ super.allocateMemoryComponents();
+ initializeMemoryComponentsStaticStructure();
+ }
+
+ private void initializeMemoryComponentsStaticStructure() throws HyracksDataException {
+ if (staticStructure == null) {
+ return; // Not yet loaded (during initial index creation)
+ }
+ for (ILSMMemoryComponent memComponent : memoryComponents) {
+ reinitializeMemoryComponent((LSMVTreeMemoryComponent) memComponent);
+ }
+ }
+
+ /**
+ * Re-initialize a single memory component's static structure directory pages.
+ * Called during initial setup and after a memory component is recycled post-flush.
+ */
+ void reinitializeMemoryComponent(LSMVTreeMemoryComponent memComponent) throws HyracksDataException {
+ if (staticStructure == null) {
+ return;
+ }
+ VTree vTree = memComponent.getIndex();
+ if (!vTree.isInitialized()) {
+ VTree.VTreeAccessor staticAccessor =
+ (VTree.VTreeAccessor) staticStructure.getIndex().createAccessor(NoOpIndexAccessParameters.INSTANCE);
+ try {
+ vTree.setStaticStructure(staticAccessor);
+ } finally {
+ // setStaticStructure only reads from the accessor; destroy it so its op-context (frames +
+ // any pinned static metadata page) is not leaked on every memory-component recycle.
+ staticAccessor.destroy();
+ }
+ }
+ }
+
+ protected ILSMDiskComponent createStaticStructure(ILSMDiskComponentFactory factory,
+ FileReference staticStructureFileReference, boolean createComponent) throws HyracksDataException {
+ ILSMDiskComponent component =
+ factory.createComponent(this, new LSMComponentFileReferences(staticStructureFileReference, null, null));
+ try {
+ ((LSMVTreeDiskComponent) component).setStaticStructure(true);
+ component.activate(createComponent);
+ } catch (HyracksDataException e) {
+ component.returnPages();
+ throw e;
+ }
+ return component;
+ }
+
+ @Override
+ public void addBulkLoadedDiskComponent(ILSMDiskComponent c) throws HyracksDataException {
+ LSMVTreeDiskComponent vTreeDiskComponent = (LSMVTreeDiskComponent) c;
+ if (vTreeDiskComponent.isStaticStructure()) {
+ setStaticStructure(vTreeDiskComponent);
+ return;
+ }
+ vTreeDiskComponent.setInitialized();
+ diskComponents.addFirst(c);
+ validateComponentIds();
+ }
+
+ public LSMVTreeDiskComponent getStaticStructure() {
+ if (staticStructure == null) {
+ throw new IllegalStateException("Static structure must be built before loading records");
+ }
+ return staticStructure;
+ }
+
+ /**
+ * Creates a bulk loader for either static-structure creation or data loading, depending on
+ * whether structure-building parameters are present.
+ */
+ @Override
+ public IIndexBulkLoader createBulkLoader(float fillFactor, boolean verifyInput, long numElementsHint,
+ Map<String, Object> parameters) throws HyracksDataException {
+ if (parameters == null) {
+ // Both paths dereference parameters (data load also writes the static-structure component into
+ // it); fail with a clear message rather than NPEing below.
+ throw HyracksDataException.create(ErrorCode.ILLEGAL_STATE,
+ "createBulkLoader requires a non-null parameters map (static-structure build params, or "
+ + "a map to receive the static-structure component for data load)");
+ }
+ AbstractLSMIndexOperationContext opCtx = createOpContext(NoOpIndexAccessParameters.INSTANCE);
+ boolean isStaticStructureLoad = LSMVTreeDiskComponent.isStaticStructureLoad(parameters);
+ if (!isStaticStructureLoad) {
+ // For data loading, ensure static structure is already built
+ parameters.put(LSMVTreeDiskComponent.PARAM_STATIC_STRUCTURE_COMPONENT, getStaticStructure());
+ }
+ opCtx.setParameters(parameters);
+ LSMVTreeComponentFileReferences componentFileRefs =
+ (LSMVTreeComponentFileReferences) fileManager.getRelFlushFileReference();
+ LoadOperation loadOp = new LoadOperation(componentFileRefs, ioOpCallback, getIndexIdentifier(), parameters);
+
+ ILSMDiskComponent component;
+ if (isStaticStructureLoad) {
+ component = createStaticStructure(bulkLoadComponentFactory,
+ componentFileRefs.getStaticStructureFileReference(), true);
+ } else {
+ component = createDiskComponent(bulkLoadComponentFactory, componentFileRefs.getInsertIndexFileReference(),
+ null, null, true);
+ }
+
+ loadOp.setNewComponent(component);
+ ioOpCallback.scheduled(loadOp);
+ opCtx.setIoOperation(loadOp);
+ return new LSMIndexDiskComponentBulkLoader(storageConfig, this, opCtx, fillFactor, verifyInput,
+ numElementsHint);
+ }
+
+ @Override
+ public boolean isPrimaryIndex() {
+ // VTree is always a secondary index (there is no vector primary index), matching
+ // LSMRTree/LSMInvertedIndex.
+ return false;
+ }
+
+ @Override
+ public IBinaryComparatorFactory[] getComparatorFactories() {
+ return cmpFactories;
+ }
+
+ public int getVectorDimensions() {
+ return vectorDimensions;
+ }
+
+ /**
+ * Whether this index stores data tuples in the quantized layout (pkStartField=4). Production
+ * quantized indexes carry non-null {@code quantizationParams}; test fixtures may select the
+ * quantized layout purely through the data-tuple-creator factory.
+ */
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_FABLE_5, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.ASSISTED)
+ public boolean isQuantized() {
+ return quantizationParams != null || dataTupleBuilderFactory.isQuantized();
+ }
+
+ public int getNumPrimaryKeyFields() {
+ return numPrimaryKeyFields;
+ }
+
+ @Override
+ public void modify(IIndexOperationContext ictx, ITupleReference tuple) throws HyracksDataException {
+ LSMVTreeOpContext ctx = (LSMVTreeOpContext) ictx;
+
+ ITupleReference indexTuple;
+ if (ctx.getIndexTuple() != null) {
+ ctx.getIndexTuple().reset(tuple);
+ indexTuple = ctx.getIndexTuple();
+ } else {
+ indexTuple = tuple;
+ }
+
+ switch (ctx.getOperation()) {
+ case PHYSICALDELETE:
+ ctx.getCurrentMutableVTreeAccessor().delete(indexTuple);
+ break;
+ case INSERT:
+ insert(indexTuple, ctx);
+ break;
+ case DELETE:
+ delete(indexTuple, ctx);
+ break;
+ default:
+ ctx.getCurrentMutableVTreeAccessor().upsert(indexTuple);
+ break;
+ }
+ updateFilter(ctx, tuple);
+ }
+
+ private void insert(ITupleReference tuple, LSMVTreeOpContext ctx) throws HyracksDataException {
+ ctx.getCurrentMutableVTreeAccessor().insert(tuple);
+ }
+
+ private void delete(ITupleReference tuple, LSMVTreeOpContext ctx) throws HyracksDataException {
+ ctx.getCurrentMutableVTreeAccessor().delete(tuple);
+ }
+
+ @Override
+ public void search(ILSMIndexOperationContext ictx, IIndexCursor cursor, ISearchPredicate pred)
+ throws HyracksDataException {
+ LSMVTreeOpContext ctx = (LSMVTreeOpContext) ictx;
+ List<ILSMComponent> operationalComponents = ctx.getComponentHolder();
+ ctx.getSearchInitialState().reset(pred, operationalComponents);
+ cursor.open(ctx.getSearchInitialState(), pred);
+ }
+
+ @Override
+ public void scanDiskComponents(ILSMIndexOperationContext ictx, IIndexCursor cursor) {
+ // Vector clustering trees don't support disk-component scanning the way BTrees do.
+ throw new UnsupportedOperationException("Disk component scanning not supported for vector clustering trees");
+ }
+
+ @Override
+ public ILSMDiskComponent doFlush(ILSMIOOperation operation) throws HyracksDataException {
+ LSMVTreeFlushOperation flushOp = (LSMVTreeFlushOperation) operation;
+ LSMVTreeMemoryComponent flushingComponent = (LSMVTreeMemoryComponent) flushOp.getFlushingComponent();
+ VTree memTree = flushingComponent.getIndex();
+
+ ILSMDiskComponent component;
+ VTreeFlushLoader flushLoader = null;
+ // Transient accessors used only for this flush; each holds an op-context (frames + possibly a
+ // pinned static metadata page) and MUST be destroyed, else every flush leaks one.
+ VTree.VTreeAccessor vTreeAccessor = null;
+ VTree.VTreeAccessor staticAccessor = null;
+
+ try {
+ component = createDiskComponent(componentFactory, flushOp.getTarget(), null, null, true);
+ VTree diskTree = ((LSMVTreeDiskComponent) component).getIndex();
+
+ // Create flush loader (new signature: callback, diskTree, sourceMemoryTree)
+ IPageWriteCallback callback = pageWriteCallbackFactory.createPageWriteCallback();
+ flushLoader = new VTreeFlushLoader(callback, diskTree, memTree);
+ callback.initialize(flushLoader);
+
+ // Copy ALL VBC pages (identity mapping: VBC page N -> disk page N)
+ vTreeAccessor = (VTree.VTreeAccessor) memTree.createAccessor(NoOpIndexAccessParameters.INSTANCE);
+ ITreeIndexMetadataFrame componentMetaFrame = vTreeAccessor.getOpContext().getMetaFrame();
+ int maxPageId = memTree.getPageManager().getMaxPageId(componentMetaFrame);
+
+ for (int pageId = 0; pageId <= maxPageId; pageId++) {
+ ICachedPage sourcePage = vTreeAccessor.getCachedPage(pageId);
+ try {
+ flushLoader.copyPage(sourcePage);
+ } finally {
+ // Release the pin even if copyPage throws (I/O, compression, disk-full); otherwise a
+ // failed flush permanently leaks a buffer-cache pin and progressively starves the cache.
+ vTreeAccessor.releasePage(sourcePage);
+ }
+ }
+
+ // Append static structure pages at end of file. Route through getStaticStructure() so a flush
+ // scheduled before the static structure is built fails with a clear message, not an NPE.
+ staticAccessor = (VTree.VTreeAccessor) getStaticStructure().getIndex()
+ .createAccessor(NoOpIndexAccessParameters.INSTANCE);
+ int rootPageId = flushLoader.copyStaticStructure(staticAccessor);
+
+ // Finalize with correct metadata
+ flushLoader.end(memTree.getNumLeafCentroidMem(), memTree.getFirstLeafCentroidIdMem(), rootPageId);
+
+ } catch (Throwable e) {
+ try {
+ if (flushLoader != null) {
+ flushLoader.abort();
+ }
+ } catch (Throwable th) {
+ e.addSuppressed(th);
+ }
+ throw e;
+ } finally {
+ // Best-effort: destroy both transient accessors without masking a primary failure.
+ Throwable failure = CleanupUtils.destroy(null, vTreeAccessor, staticAccessor);
+ if (failure != null) {
+ LOGGER.warn("Failed to destroy transient flush accessors", failure);
+ }
+ }
+ return component;
+ }
+
+ @Override
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_FABLE_5, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.ASSISTED)
+ public ILSMDiskComponent doMerge(ILSMIOOperation operation) throws HyracksDataException {
+ LSMVTreeMergeOperation mergeOp = (LSMVTreeMergeOperation) operation;
+ IIndexCursor cursor = mergeOp.getCursor();
+ ILSMDiskComponent mergedComponent;
+ ILSMDiskComponentBulkLoader componentBulkLoader = null;
+
+ try {
+ try {
+ // Full-scan predicate: epsilon=0 disables level-wise selection. The cursor was
+ // created in full-scan mode by createMergeOperation(), so it uses
+ // SequentialClusterSelectionStrategy to iterate clusters in order.
+ VTreeSearchPredicate mergePred = new VTreeSearchPredicate();
+ mergePred.setEpsilon(0.0);
+ // The merge cursor (LSMVTreeSearchCursor) derives pkStartField from the index's
+ // isQuantized flag, so its reconciliation key is <distance (field 0), PK...> — quantized
+ // layouts correctly skip the quantized_distance/quantized_embedding fields. (Getting this
+ // wrong would pull field 2, whose write semantics differ between bulk load and DML, into
+ // the key and break matter/antimatter cancellation during COMPACT.)
+
+ // Cursor was already created in full-scan mode by createMergeOperation()
+ search(mergeOp.getAccessor().getOpContext(), cursor, mergePred);
+
+ try {
+ mergedComponent = createDiskComponent(componentFactory, mergeOp.getTarget(), null, null, true);
+
+ // Static structure reference is required by VTreeBulkLoader
+ Map<String, Object> parameters = new HashMap<>();
+ parameters.put(LSMVTreeDiskComponent.PARAM_STATIC_STRUCTURE_COMPONENT, getStaticStructure());
+ mergeOp.getAccessor().getOpContext().setParameters(parameters);
+
+ IPageWriteCallback pageWriteCallback = pageWriteCallbackFactory.createPageWriteCallback();
+ // numElementsHint = 0 (unknown): the VTree merge loader does not consume it, and the
+ // former value (sum of component byte sizes) was the wrong unit — a tuple count, not bytes.
+ componentBulkLoader = mergedComponent.createBulkLoader(storageConfig, operation, 1.0f, false, 0L,
+ false, false, false, pageWriteCallback);
+
+ // Cursor delivers tuples cluster-by-cluster, already antimatter-reconciled
+ while (cursor.hasNext()) {
+ cursor.next();
+ ITupleReference frameTuple = cursor.getTuple();
+ componentBulkLoader.add(frameTuple);
+ }
+ } finally {
+ cursor.close();
+ }
+ } finally {
+ cursor.destroy();
+ }
+
+ componentBulkLoader.end();
+ } catch (Throwable e) { // NOSONAR.. As per the contract, we should either abort or end
+ try {
+ if (componentBulkLoader != null) {
+ componentBulkLoader.abort();
+ }
+ } catch (Throwable th) { // NOSONAR Don't lose the root failure
+ e.addSuppressed(th);
+ }
+ throw e;
+ }
+
+ return mergedComponent;
+ }
+
+ @Override
+ protected ILSMIOOperation createFlushOperation(AbstractLSMIndexOperationContext opCtx,
+ LSMComponentFileReferences componentFileRefs, ILSMIOOperationCallback callback) {
+ return new LSMVTreeFlushOperation(createAccessor(opCtx), componentFileRefs.getInsertIndexFileReference(),
+ callback, getIndexIdentifier());
+ }
+
+ @Override
+ public LSMVTreeOpContext createOpContext(IIndexAccessParameters iap) {
+ return new LSMVTreeOpContext(this, memoryComponents, getTreeFields(), getFilterFields(),
+ getFilterCmpFactories(), (IExtendedModificationOperationCallback) iap.getModificationCallback(),
+ iap.getSearchOperationCallback(), tracer, iap);
+ }
+
+ @Override
+ public ILSMIndexAccessor createAccessor(IIndexAccessParameters iap) {
+ return createAccessor(createOpContext(iap));
+ }
+
+ public ILSMIndexAccessor createAccessor(AbstractLSMIndexOperationContext opCtx) {
+ return new LSMVTreeIndexAccessor(getHarness(), opCtx, getCursorFactory(), this);
+ }
+
+ @Override
+ public ITreeIndexFrameFactory getInteriorFrameFactory() {
+ LSMVTreeMemoryComponent mutableComponent =
+ (LSMVTreeMemoryComponent) memoryComponents.get(currentMutableComponentId.get());
+ return mutableComponent.getIndex().getInteriorFrameFactory();
+ }
+
+ public ITreeIndexFrameFactory getMetadataFrameFactory() {
+ return metadataFrameFactory;
+ }
+
+ public ITreeIndexFrameFactory getInsertDataFrameFactory() {
+ return insertDataFrameFactory;
+ }
+
+ public ITreeIndexFrameFactory getDeleteDataFrameFactory() {
+ return deleteDataFrameFactory;
+ }
+
+ public IBinaryComparatorFactory[] getCmpFactories() {
+ return cmpFactories;
+ }
+
+ @Override
+ public int getFieldCount() {
+ LSMVTreeMemoryComponent mutableComponent =
+ (LSMVTreeMemoryComponent) memoryComponents.get(currentMutableComponentId.get());
+ return mutableComponent.getIndex().getFieldCount();
+ }
+
+ @Override
+ public int getFileId() {
+ LSMVTreeMemoryComponent mutableComponent =
+ (LSMVTreeMemoryComponent) memoryComponents.get(currentMutableComponentId.get());
+ return mutableComponent.getIndex().getFileId();
+ }
+
+ @Override
+ public IPageManager getPageManager() {
+ LSMVTreeMemoryComponent mutableComponent =
+ (LSMVTreeMemoryComponent) memoryComponents.get(currentMutableComponentId.get());
+ return mutableComponent.getIndex().getPageManager();
+ }
+
+ @Override
+ public ITreeIndexFrameFactory getLeafFrameFactory() {
+ LSMVTreeMemoryComponent mutableComponent =
+ (LSMVTreeMemoryComponent) memoryComponents.get(currentMutableComponentId.get());
+ return mutableComponent.getIndex().getLeafFrameFactory();
+ }
+
+ @Override
+ public int getRootPageId() {
+ LSMVTreeMemoryComponent mutableComponent =
+ (LSMVTreeMemoryComponent) memoryComponents.get(currentMutableComponentId.get());
+ return mutableComponent.getIndex().getRootPageId();
+ }
+
+ @Override
+ protected LSMComponentFileReferences getMergeFileReferences(ILSMDiskComponent firstComponent,
+ ILSMDiskComponent lastComponent) throws HyracksDataException {
+ // Extract file names from components (e.g., "0_0" from file "0_0_vct")
+ LSMVTreeDiskComponent first = (LSMVTreeDiskComponent) firstComponent;
+ LSMVTreeDiskComponent last = (LSMVTreeDiskComponent) lastComponent;
+
+ String firstName = first.getIndex().getFileReference().getFile().getName();
+ String lastName = last.getIndex().getFileReference().getFile().getName();
+
+ return fileManager.getRelMergeFileReference(firstName, lastName);
+ }
+
+ @Override
+ protected ILSMIOOperation createMergeOperation(AbstractLSMIndexOperationContext opCtx,
+ LSMComponentFileReferences mergeFileRefs, ILSMIOOperationCallback callback) {
+
+ // If we're merging the oldest disk component (i.e. there's nothing older that could still
+ // need anti-matter tuples for reconciliation), antimatter tuples can be discarded during the
+ // merge; otherwise they must be preserved.
+ List<ILSMComponent> mergingComponents = opCtx.getComponentHolder();
+ boolean returnDeletedTuples = true;
+ if (!diskComponents.isEmpty() && !mergingComponents.isEmpty()
+ && mergingComponents.getLast() == diskComponents.getLast()) {
+ returnDeletedTuples = false;
+ }
+
+ // Create cursor with full-scan mode enabled for merge
+ IIndexCursorStats stats = NoOpIndexCursorStats.INSTANCE;
+ ILSMIndexAccessor accessor = createAccessor(opCtx);
+
+ // Create LSMVTreeSearchCursor in full-scan mode for merge operations
+ // fullScanMode=true enables sequential cluster iteration (0→1→2→...)
+ // returnDeletedTuples=true ensures antimatter tuples are visible for reconciliation
+ IIndexCursor cursor = new LSMVTreeSearchCursor(opCtx, returnDeletedTuples, true, stats);
+
+ return new LSMVTreeMergeOperation(accessor, cursor, stats, mergeFileRefs.getInsertIndexFileReference(),
+ callback, getIndexIdentifier());
+ }
+
+ protected ICursorFactory getCursorFactory() {
+ return cursorFactory;
+ }
+
+ /**
+ * Creates the top-K search cursor: all search work happens in {@code open()}, results
+ * are collected into a quantized-distance-keyed top-K window, and the cursor drains that
+ * window on subsequent {@code hasNext()/next()/getTuple()} calls. This is the canonical
+ * search cursor for the VTree index; the streaming {@link LSMVTreeSearchCursor} is reserved
+ * for component merges.
+ */
+ public IIndexCursor createTopKSearchCursor(ILSMIndexOperationContext opCtx) {
+ return new LSMVTreeTopKSearchCursor(opCtx);
+ }
+
+ @Override
+ public synchronized void deactivate(boolean flush) throws HyracksDataException {
+ // Flush must happen before static structure cleanup because doFlush() needs staticStructure
+ super.deactivate(flush);
+ // Clean up static structure component after parent deactivation (including flush)
+ if (staticStructure != null) {
+ try {
+ staticStructure.deactivateAndPurge();
+ } catch (Exception e) {
+ // Best-effort cleanup; do not fail deactivation if static structure cleanup fails. Log at
+ // WARN with the throwable — a failed purge leaks a file handle / buffer-cache file slot,
+ // which is worth surfacing rather than hiding at TRACE.
+ LOGGER.warn("Failed to deactivate static structure component", e);
+ }
+ staticStructure = null;
+ }
+ }
+
+ @Override
+ public synchronized void destroy() throws HyracksDataException {
+ // Clean up static structure component before parent destruction
+ if (staticStructure != null) {
+ try {
+ staticStructure.destroy();
+ } catch (Exception e) {
+ // Best-effort cleanup; do not fail destruction if static structure cleanup fails. Log at
+ // WARN with the throwable — a failed destroy leaves an orphaned static-structure file.
+ LOGGER.warn("Failed to destroy static structure component", e);
+ }
+ staticStructure = null;
+ }
+ super.destroy();
+ }
+
+ @Override
+ public synchronized void activate() throws HyracksDataException {
+ if (isActive) {
+ throw HyracksDataException.create(ErrorCode.CANNOT_ACTIVATE_ACTIVE_INDEX);
+ }
+ loadDiskComponents();
+ completeActivation();
+ isActive = true;
+ }
+
+ private void loadStaticStructure() throws HyracksDataException {
+ LSMVTreeFileManager vTreeFileManager = (LSMVTreeFileManager) fileManager;
+ LSMVTreeComponentFileReferences ssFileRef = vTreeFileManager.getStaticStructureFileReference();
+ if (ssFileRef == null) {
+ return;
+ }
+ ILSMDiskComponent ssComponent =
+ createStaticStructure(componentFactory, ssFileRef.getStaticStructureFileReference(), false);
+ setStaticStructure((LSMVTreeDiskComponent) ssComponent);
+ }
+
+ private void loadDiskComponents() throws HyracksDataException {
+ diskComponents.clear();
+ List<LSMComponentFileReferences> validFileReferences = fileManager.cleanupAndGetValidFiles();
+ for (LSMComponentFileReferences lsmComponentFileReferences : validFileReferences) {
+ ILSMDiskComponent component =
+ createDiskComponent(componentFactory, lsmComponentFileReferences.getInsertIndexFileReference(),
+ lsmComponentFileReferences.getDeleteIndexFileReference(),
+ lsmComponentFileReferences.getBloomFilterFileReference(), false);
+ diskComponents.add(component);
+ }
+ loadStaticStructure();
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/LSMVTreeCursorInitialState.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/LSMVTreeCursorInitialState.java
new file mode 100644
index 0000000000..9ed810e86f
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/LSMVTreeCursorInitialState.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.lsm.vector.impls;
+
+import java.util.List;
+
+import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMHarness;
+import org.apache.hyracks.storage.common.ICursorInitialState;
+import org.apache.hyracks.storage.common.ISearchOperationCallback;
+import org.apache.hyracks.storage.common.ISearchPredicate;
+import org.apache.hyracks.storage.common.MultiComparator;
+import org.apache.hyracks.storage.common.buffercache.ICachedPage;
+
+/**
+ * Cursor initial state for LSM Vector Clustering Tree operations.
+ *
+ * This class maintains the initial state and context required for cursors operating on
+ * LSM Vector Clustering Trees, including frame factories for different node types,
+ * comparators, and operational components.
+ */
+public class LSMVTreeCursorInitialState implements ICursorInitialState {
+
+ private final ITreeIndexFrameFactory interiorFrameFactory;
+ private final ITreeIndexFrameFactory leafFrameFactory;
+ private final ITreeIndexFrameFactory metadataFrameFactory;
+ private final ITreeIndexFrameFactory dataFrameFactory;
+ private final MultiComparator cmp;
+ private final ILSMHarness lsmHarness;
+ private ISearchPredicate predicate;
+ private ISearchOperationCallback searchCallback;
+ private List<ILSMComponent> operationalComponents;
+ private boolean isDiskComponentScan;
+
+ public LSMVTreeCursorInitialState(ITreeIndexFrameFactory interiorFrameFactory,
+ ITreeIndexFrameFactory leafFrameFactory, ITreeIndexFrameFactory metadataFrameFactory,
+ ITreeIndexFrameFactory dataFrameFactory, MultiComparator cmp, ILSMHarness lsmHarness,
+ ISearchPredicate predicate, ISearchOperationCallback searchCallback,
+ List<ILSMComponent> operationalComponents) {
+ this.interiorFrameFactory = interiorFrameFactory;
+ this.leafFrameFactory = leafFrameFactory;
+ this.metadataFrameFactory = metadataFrameFactory;
+ this.dataFrameFactory = dataFrameFactory;
+ this.cmp = cmp;
+ this.lsmHarness = lsmHarness;
+ this.predicate = predicate;
+ this.searchCallback = searchCallback;
+ this.operationalComponents = operationalComponents;
+ this.isDiskComponentScan = false;
+ }
+
+ public ITreeIndexFrameFactory getInteriorFrameFactory() {
+ return interiorFrameFactory;
+ }
+
+ public ITreeIndexFrameFactory getLeafFrameFactory() {
+ return leafFrameFactory;
+ }
+
+ public ITreeIndexFrameFactory getMetadataFrameFactory() {
+ return metadataFrameFactory;
+ }
+
+ public ITreeIndexFrameFactory getDataFrameFactory() {
+ return dataFrameFactory;
+ }
+
+ public MultiComparator getOriginalKeyComparator() {
+ return cmp;
+ }
+
+ @Override
+ public void setOriginialKeyComparator(MultiComparator originalCmp) {
+ // Intentionally inert: the comparator is supplied via the constructor (getOriginalKeyComparator);
+ // the VTree cursors never re-set it through the ICursorInitialState interface.
+ }
+
+ public ILSMHarness getLSMHarness() {
+ return lsmHarness;
+ }
+
+ public ISearchPredicate getSearchPredicate() {
+ return predicate;
+ }
+
+ public ISearchOperationCallback getSearchOperationCallback() {
+ return searchCallback;
+ }
+
+ @Override
+ public void setSearchOperationCallback(ISearchOperationCallback searchCallback) {
+ // Intentionally inert: the callback is supplied via the constructor (getSearchOperationCallback);
+ // the VTree cursors never re-set it through the ICursorInitialState interface.
+ }
+
+ public List<ILSMComponent> getOperationalComponents() {
+ return operationalComponents;
+ }
+
+ @Override
+ public ICachedPage getPage() {
+ return null;
+ }
+
+ @Override
+ public void setPage(ICachedPage page) {
+ // No-op for LSM trees
+ }
+
+ // Make the cursor initial state re-usable
+ public void reset(ISearchPredicate predicate, List<ILSMComponent> operationalComponents) {
+ this.isDiskComponentScan = false;
+ this.predicate = predicate;
+ this.operationalComponents = operationalComponents;
+ }
+
+ public void setDiskComponentScan(boolean isDiskComponentScan) {
+ this.isDiskComponentScan = isDiskComponentScan;
+ }
+
+ public boolean isDiskComponentScan() {
+ return isDiskComponentScan;
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/LSMVTreeDiskComponent.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/LSMVTreeDiskComponent.java
new file mode 100644
index 0000000000..9cc0d83493
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/LSMVTreeDiskComponent.java
@@ -0,0 +1,262 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.lsm.vector.impls;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.control.common.controllers.NCConfig;
+import org.apache.hyracks.storage.am.common.api.IMetadataPageManager;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexAccessor;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import org.apache.hyracks.storage.am.common.impls.NoOpIndexAccessParameters;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilter;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation.LSMIOOperationType;
+import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMDiskComponent;
+import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndex;
+import org.apache.hyracks.storage.am.lsm.common.impls.ChainedLSMDiskComponentBulkLoader;
+import org.apache.hyracks.storage.am.lsm.common.impls.IChainedComponentBulkLoader;
+import org.apache.hyracks.storage.am.lsm.common.impls.LSMIndexBulkLoader;
+import org.apache.hyracks.storage.am.lsm.vector.tuples.LSMVTreeDataTupleWriterFactory;
+import org.apache.hyracks.storage.am.vector.frames.VTreeDataFrameFactory;
+import org.apache.hyracks.storage.am.vector.impls.VTree;
+import org.apache.hyracks.storage.common.IIndexBulkLoader;
+import org.apache.hyracks.storage.common.buffercache.IPageWriteCallback;
+import org.apache.hyracks.storage.common.buffercache.NoOpPageWriteCallback;
+import org.apache.hyracks.util.annotations.AiProvenance;
+
+/**
+ * LSM disk component for Vector Clustering Trees. Wraps a materialized {@link VTree} on persistent
+ * storage; the wrapped tree may have been produced by a memory-component flush, by initial
+ * bulk-load at CREATE INDEX, or by a merge.
+ */
+public class LSMVTreeDiskComponent extends AbstractLSMDiskComponent {
+
+ // Operation-parameter keys recognized by createBulkLoader(). These are written into the
+ // operation's parameter map by the upstream operator (e.g. the static-structure creator) and
+ // dispatched on in createBulkLoader.
+ public static final String PARAM_NUM_LEVELS = "numLevels";
+ public static final String PARAM_CLUSTERS_PER_LEVEL = "clustersPerLevel";
+ public static final String PARAM_CENTROIDS_PER_CLUSTER = "centroidsPerCluster";
+ public static final String PARAM_MAX_ENTRIES_PER_PAGE = "maxEntriesPerPage";
+ public static final String PARAM_STATIC_STRUCTURE_COMPONENT = "static_structure_component";
+
+ private final VTree vTree;
+
+ /**
+ * Marks this component as carrying the index's static structure (root + interior + leaf pages,
+ * built once at CREATE INDEX time). Set by {@code LSMVTree.createStaticStructure} so that
+ * {@code LSMVTree.addBulkLoadedDiskComponent} can dispatch the just-loaded component to the
+ * static-structure slot instead of into the regular disk-component list.
+ */
+ private boolean isStaticStructure = false;
+
+ public LSMVTreeDiskComponent(AbstractLSMIndex lsmIndex, VTree vTree, ILSMComponentFilter filter) {
+ super(lsmIndex, getMetadataPageManager(vTree), filter);
+ this.vTree = vTree;
+ }
+
+ @Override
+ public VTree getIndex() {
+ return vTree;
+ }
+
+ @Override
+ public VTree getMetadataHolder() {
+ return vTree;
+ }
+
+ @Override
+ public long getComponentSize() {
+ return getComponentSize(vTree);
+ }
+
+ @Override
+ public int getFileReferenceCount() {
+ return getFileReferenceCount(vTree);
+ }
+
+ @Override
+ public Set<String> getLSMComponentPhysicalFiles() {
+ return getFiles(vTree);
+ }
+
+ @Override
+ public String toString() {
+ return getClass().getSimpleName() + ":" + vTree.getFileReference().getRelativePath();
+ }
+
+ @Override
+ public void validate() throws HyracksDataException {
+ vTree.validate();
+ }
+
+ /**
+ * A static-structure build is signalled by the presence of the clustering-shape descriptor in the
+ * bulk-load parameters (canonically {@link #PARAM_NUM_LEVELS}; the producer always supplies the full set
+ * — {@link #PARAM_CLUSTERS_PER_LEVEL} / {@link #PARAM_CENTROIDS_PER_CLUSTER} /
+ * {@link #PARAM_MAX_ENTRIES_PER_PAGE} — which {@link #createVTreeStaticStructureBulkLoader} then
+ * validates). Any other bulk load is a data load.
+ */
+ public static boolean isStaticStructureLoad(Map<String, Object> parameters) {
+ return parameters.containsKey(PARAM_NUM_LEVELS);
+ }
+
+ /**
+ * Creates a bulk loader for VTree with static structure support.
+ *
+ * Follows the LSMBTree chained-bulk-loader pattern. If the operation parameters carry the
+ * static-structure descriptor ({@link #isStaticStructureLoad}), a static-structure loader is wired in;
+ * otherwise a regular data-loading bulk loader is used, sourcing centroids from the pre-built static
+ * structure component.
+ */
+ @Override
+ public ChainedLSMDiskComponentBulkLoader createBulkLoader(NCConfig storageConfig, ILSMIOOperation operation,
+ float fillFactor, boolean verifyInput, long numElementsHint, boolean checkIfEmptyIndex, boolean withFilter,
+ boolean cleanupEmptyComponent, IPageWriteCallback callback) throws HyracksDataException {
+
+ ChainedLSMDiskComponentBulkLoader chainedBulkLoader =
+ new ChainedLSMDiskComponentBulkLoader(operation, this, cleanupEmptyComponent);
+
+ // Add filter bulk loader if needed
+ if (withFilter && getLsmIndex().getFilterFields() != null) {
+ chainedBulkLoader.addBulkLoader(createFilterBulkLoader());
+ }
+
+ // Check if this is a static structure bulk load operation
+ Map<String, Object> parameters = operation.getParameters();
+ boolean isStaticStructureLoad = isStaticStructureLoad(parameters);
+ IChainedComponentBulkLoader indexBulkLoader = isStaticStructureLoad
+ ? createVTreeStaticStructureBulkLoader(operation, callback) : createVTreeBulkLoader(operation);
+
+ chainedBulkLoader.addBulkLoader(indexBulkLoader);
+ callback.initialize(chainedBulkLoader);
+
+ return chainedBulkLoader;
+ }
+
+ /**
+ * Creates a VTreeStaticStructureLoader configured from the operation parameters
+ * ({@link #PARAM_NUM_LEVELS} / {@link #PARAM_CLUSTERS_PER_LEVEL} /
+ * {@link #PARAM_CENTROIDS_PER_CLUSTER} / {@link #PARAM_MAX_ENTRIES_PER_PAGE}) and wraps it as an
+ * {@link IChainedComponentBulkLoader}.
+ */
+ private IChainedComponentBulkLoader createVTreeStaticStructureBulkLoader(ILSMIOOperation operation,
+ IPageWriteCallback callback) throws HyracksDataException {
+
+ // These are required parameters: every producer (the static-structure creator operator and the
+ // test harness) always supplies all four, and the caller only invokes this method when the three
+ // structural keys are present. Fail loudly if any is missing/null rather than silently building the
+ // index with an arbitrary default clustering shape (which would corrupt the static structure).
+ Map<String, Object> parameters = operation.getParameters();
+ Integer numLevelsParam = (Integer) parameters.get(PARAM_NUM_LEVELS);
+ @SuppressWarnings("unchecked")
+ List<Integer> clustersPerLevel = (List<Integer>) parameters.get(PARAM_CLUSTERS_PER_LEVEL);
+ @SuppressWarnings("unchecked")
+ List<List<Integer>> centroidsPerCluster = (List<List<Integer>>) parameters.get(PARAM_CENTROIDS_PER_CLUSTER);
+ Integer maxEntriesParam = (Integer) parameters.get(PARAM_MAX_ENTRIES_PER_PAGE);
+ if (numLevelsParam == null || clustersPerLevel == null || centroidsPerCluster == null
+ || maxEntriesParam == null) {
+ throw new HyracksDataException("static-structure bulk load requires non-null " + PARAM_NUM_LEVELS + ", "
+ + PARAM_CLUSTERS_PER_LEVEL + ", " + PARAM_CENTROIDS_PER_CLUSTER + ", and "
+ + PARAM_MAX_ENTRIES_PER_PAGE + " parameters");
+ }
+ int numLevels = numLevelsParam;
+ int maxEntriesPerPage = maxEntriesParam;
+
+ IIndexBulkLoader ssbuilder = getIndex().createStaticStructureBulkLoader(numLevels, clustersPerLevel,
+ centroidsPerCluster, maxEntriesPerPage, callback);
+ return new LSMIndexBulkLoader(ssbuilder, getMetadata(), getSampler());
+ }
+
+ /**
+ * Creates a regular data-loading bulk loader that uses the already-built static structure
+ * component (passed via the {@link #PARAM_STATIC_STRUCTURE_COMPONENT} parameter) to assign
+ * incoming records to leaf clusters.
+ */
+ private IChainedComponentBulkLoader createVTreeBulkLoader(ILSMIOOperation operation) throws HyracksDataException {
+ Object staticParam = operation.getParameters().get(PARAM_STATIC_STRUCTURE_COMPONENT);
+ if (!(staticParam instanceof LSMVTreeDiskComponent staticComponent)) {
+ throw new HyracksDataException(
+ PARAM_STATIC_STRUCTURE_COMPONENT + " must be provided in parameters for data loading");
+ }
+ ITreeIndexAccessor staticAccessor =
+ staticComponent.getIndex().createAccessor(NoOpIndexAccessParameters.INSTANCE);
+ // Merges drain tuples from the merge cursor, which may hand over PRESERVED antimatter
+ // tuples (returnDeletedTuples=true when the merging set excludes the oldest disk
+ // component). The tree's default data frames use the insert (matter) tuple writer, which
+ // would silently re-encode preserved antimatter as matter and resurrect deletes — so
+ // merges write through an antimatter-preserving copy tuple writer instead (LSMBTree's
+ // copy-tuple-writer pattern). Initial bulk load (LOAD) keeps the default matter writer.
+ ITreeIndexFrameFactory dataFrameFactoryOverride =
+ operation.getIOOperationType() == LSMIOOperationType.MERGE ? createCopyDataFrameFactory() : null;
+ IIndexBulkLoader builder = getIndex().createComponentBulkLoader(NoOpPageWriteCallback.INSTANCE, staticAccessor,
+ getSampler(), dataFrameFactoryOverride);
+ return new LSMIndexBulkLoader(builder, getMetadata(), getSampler());
+ }
+
+ /**
+ * Builds a data-frame factory identical to the tree's default except that its tuple writer
+ * carries over the source tuple's antimatter bit.
+ */
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_FABLE_5, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.ASSISTED)
+ private ITreeIndexFrameFactory createCopyDataFrameFactory() {
+ VTreeDataFrameFactory dataFrameFactory = (VTreeDataFrameFactory) getIndex().getDataFrameFactory();
+ LSMVTreeDataTupleWriterFactory tupleWriterFactory =
+ (LSMVTreeDataTupleWriterFactory) dataFrameFactory.getTupleWriterFactory();
+ return new VTreeDataFrameFactory(tupleWriterFactory.createCopyWriterFactory(),
+ dataFrameFactory.getVectorDimensions());
+ }
+
+ static IMetadataPageManager getMetadataPageManager(VTree vTree) {
+ return (IMetadataPageManager) vTree.getPageManager();
+ }
+
+ static long getComponentSize(VTree vTree) {
+ return vTree.getFileReference().getFile().length();
+ }
+
+ static int getFileReferenceCount(VTree vTree) {
+ return vTree.getBufferCache().getFileReferenceCount(vTree.getFileId());
+ }
+
+ static Set<String> getFiles(VTree vTree) {
+ Set<String> files = new HashSet<>();
+ files.add(vTree.getFileReference().getFile().getAbsolutePath());
+ return files;
+ }
+
+ public boolean isStaticStructure() {
+ return isStaticStructure;
+ }
+
+ public void setStaticStructure(boolean isStaticStructure) {
+ this.isStaticStructure = isStaticStructure;
+ }
+
+ public void setInitialized() throws HyracksDataException {
+ int rootPageId = vTree.getPageManager().getRootPageId();
+ vTree.setRootPageId(rootPageId);
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/LSMVTreeDiskComponentFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/LSMVTreeDiskComponentFactory.java
new file mode 100644
index 0000000000..53155165e0
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/LSMVTreeDiskComponentFactory.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.lsm.vector.impls;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.lsm.common.api.IComponentFilterHelper;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentFactory;
+import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndex;
+import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
+import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
+import org.apache.hyracks.storage.am.vector.impls.VTree;
+
+/**
+ * Factory for creating LSM Vector Clustering Tree disk components.
+ *
+ * This factory creates disk components for LSM Vector Clustering Trees,
+ * which contain materialized VTree instances that have been
+ * flushed from memory to persistent storage.
+ */
+public class LSMVTreeDiskComponentFactory implements ILSMDiskComponentFactory {
+
+ protected final TreeIndexFactory<VTree> vTreeFactory;
+ protected final IComponentFilterHelper filterHelper;
+
+ public LSMVTreeDiskComponentFactory(TreeIndexFactory<VTree> vTreeFactory, IComponentFilterHelper filterHelper) {
+ this.vTreeFactory = vTreeFactory;
+ this.filterHelper = filterHelper;
+ }
+
+ @Override
+ public LSMVTreeDiskComponent createComponent(AbstractLSMIndex lsmIndex, LSMComponentFileReferences cfr)
+ throws HyracksDataException {
+ return new LSMVTreeDiskComponent(lsmIndex, vTreeFactory.createIndexInstance(cfr.getInsertIndexFileReference()),
+ filterHelper == null ? null : filterHelper.createFilter());
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/LSMVTreeFileManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/LSMVTreeFileManager.java
new file mode 100644
index 0000000000..fc8135ace6
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/LSMVTreeFileManager.java
@@ -0,0 +1,213 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.lsm.vector.impls;
+
+import java.io.FilenameFilter;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.hyracks.api.exceptions.ErrorCode;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.IIOManager;
+import org.apache.hyracks.storage.am.common.api.ITreeIndex;
+import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexFileManager;
+import org.apache.hyracks.storage.am.lsm.common.impls.IndexComponentFileReference;
+import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
+import org.apache.hyracks.storage.am.lsm.common.impls.LSMVTreeComponentFileReferences;
+import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+/**
+ * File manager for LSM Vector Clustering Trees.
+ *
+ * This class manages the files associated with LSM Vector Clustering Tree components,
+ * including naming conventions, file creation, and cleanup operations for both
+ * in-memory and disk components.
+ */
+public class LSMVTreeFileManager extends AbstractLSMIndexFileManager {
+ private static final Logger LOGGER = LogManager.getLogger();
+
+ private static final String VCTREE_SUFFIX = "vct";
+ private static final String STATIC_STRUCTURE_SUFFIX = ".staticstructure";
+
+ private final TreeIndexFactory<? extends ITreeIndex> vTreeFactory;
+
+ private static final FilenameFilter vTreeFilter =
+ (dir, name) -> !name.startsWith(".") && name.endsWith(VCTREE_SUFFIX);
+
+ public LSMVTreeFileManager(IIOManager ioManager, FileReference file,
+ TreeIndexFactory<? extends ITreeIndex> vTreeFactory) {
+ super(ioManager, file, null);
+ this.vTreeFactory = vTreeFactory;
+ }
+
+ @Override
+ public LSMComponentFileReferences getRelFlushFileReference() throws HyracksDataException {
+ String baseName = getNextComponentSequence(vTreeFilter);
+ return new LSMVTreeComponentFileReferences(baseDir.getChild(baseName + DELIMITER + VCTREE_SUFFIX), null, null,
+ baseDir.getChild(STATIC_STRUCTURE_SUFFIX));
+ }
+
+ @Override
+ public LSMComponentFileReferences getRelMergeFileReference(String firstFileName, String lastFileName) {
+ final String baseName = IndexComponentFileReference.getMergeSequence(firstFileName, lastFileName);
+ return new LSMComponentFileReferences(baseDir.getChild(baseName + DELIMITER + VCTREE_SUFFIX), null,
+ baseDir.getChild(baseName + DELIMITER + STATIC_STRUCTURE_SUFFIX));
+ }
+
+ @Override
+ public List<LSMComponentFileReferences> cleanupAndGetValidFiles() throws HyracksDataException {
+ List<LSMComponentFileReferences> validFiles = new ArrayList<>();
+ ArrayList<IndexComponentFileReference> allVTreeFiles = new ArrayList<>();
+
+ // Gather all VTree data component files.
+ collectVTreeFiles(allVTreeFiles);
+
+ // The single shared static structure (one per index) is required to read any component; without a
+ // valid one no VTree component can be opened, so drop everything.
+ FileReference staticStructureFile = baseDir.getChild(STATIC_STRUCTURE_SUFFIX);
+ if (!validateStaticStructureFile(staticStructureFile)) {
+ LOGGER.log(Level.TRACE, "Invalid or missing shared .staticstructure file: {}",
+ staticStructureFile.getAbsolutePath());
+ for (IndexComponentFileReference vTreeFile : allVTreeFiles) {
+ cleanupOrphanedVTreeFile(vTreeFile.getFileRef());
+ }
+ return validFiles; // Return empty list
+ }
+
+ if (allVTreeFiles.isEmpty()) {
+ return validFiles;
+ }
+
+ // Drop (and delete) any component whose sequence interval is contained in a more-recent (merged)
+ // component. A merge writes the merged component and then deletes its inputs as a SEPARATE step; a
+ // crash in between leaves the pre-merge inputs on disk. Loading both the merged component and its
+ // inputs would duplicate every record and resurface deletes the merge reconciled away. Mirrors
+ // AbstractLSMIndexFileManager.cleanupAndGetValidFiles (isMoreRecentThan / isWithin). Natural order
+ // sorts by sequenceStart asc then sequenceEnd desc, so a merged component precedes the flushed
+ // inputs it contains and becomes the running "widest" reference.
+ Collections.sort(allVTreeFiles);
+ ArrayList<IndexComponentFileReference> survivors = new ArrayList<>();
+ IndexComponentFileReference last = allVTreeFiles.get(0);
+ survivors.add(last);
+ for (int i = 1; i < allVTreeFiles.size(); i++) {
+ IndexComponentFileReference current = allVTreeFiles.get(i);
+ if (current.isMoreRecentThan(last)) {
+ survivors.add(current);
+ last = current;
+ } else if (current.isWithin(last)) {
+ LOGGER.log(Level.TRACE, "Deleting merged-away VTree component {} (within {})", current.getSequence(),
+ last.getSequence());
+ cleanupOrphanedVTreeFile(current.getFileRef());
+ } else {
+ throw HyracksDataException.create(ErrorCode.FOUND_OVERLAPPING_LSM_FILES, baseDir);
+ }
+ }
+
+ // LSM expects disk components ordered newest -> oldest: [2,2], [1,1], [0,0].
+ survivors.sort(Collections.reverseOrder());
+ for (IndexComponentFileReference vTreeFile : survivors) {
+ LOGGER.log(Level.TRACE, "Valid VTree component: {} (using shared .staticstructure)",
+ vTreeFile.getSequence());
+ validFiles.add(new LSMComponentFileReferences(vTreeFile.getFileRef(), null, staticStructureFile));
+ }
+ return validFiles;
+ }
+
+ @Override
+ protected boolean areHolesAllowed() {
+ return false; // VTree components must be contiguous
+ }
+
+ /**
+ * Validates and returns the shared static structure file reference.
+ * The static structure is shared across all LSM components and contains
+ * the hierarchical k-means clustering metadata.
+ *
+ * @return LSMVTreeComponentFileReferences with only the static structure, or null if invalid
+ * @throws HyracksDataException if validation fails
+ */
+ public LSMVTreeComponentFileReferences getStaticStructureFileReference() throws HyracksDataException {
+ FileReference staticStructureFile = baseDir.getChild(STATIC_STRUCTURE_SUFFIX);
+ if (validateStaticStructureFile(staticStructureFile)) {
+ return new LSMVTreeComponentFileReferences(null, null, null, staticStructureFile);
+ }
+ return null;
+ }
+
+ /**
+ * Lists all VTree component files in the base directory and appends their
+ * IndexComponentFileReference forms to the supplied list.
+ */
+ private void collectVTreeFiles(List<IndexComponentFileReference> files) {
+ String[] fileNames = baseDir.getFile().list(vTreeFilter);
+ if (fileNames == null) {
+ return;
+ }
+ for (String fileName : fileNames) {
+ FileReference fileRef = baseDir.getChild(fileName);
+ files.add(IndexComponentFileReference.of(fileRef));
+ }
+ }
+
+ /**
+ * Validates that a .staticstructure file exists and is valid.
+ *
+ * @param staticStructureFile The .staticstructure file to validate
+ * @return true if the file exists and is valid, false otherwise
+ */
+ private boolean validateStaticStructureFile(FileReference staticStructureFile) {
+ try {
+ // Check if file exists
+ if (!ioManager.exists(staticStructureFile)) {
+ LOGGER.log(Level.TRACE, "Static structure file does not exist: {}",
+ staticStructureFile.getAbsolutePath());
+ return false;
+ }
+
+ LOGGER.log(Level.TRACE, "Static structure file is valid: {}", staticStructureFile.getAbsolutePath());
+ return true;
+
+ } catch (Exception e) {
+ LOGGER.log(Level.TRACE, "Error validating static structure file {}: {}",
+ staticStructureFile.getAbsolutePath(), e.getMessage());
+ return false;
+ }
+ }
+
+ /**
+ * Cleans up an orphaned VTree file when its .staticstructure file is missing or invalid.
+ *
+ * @param vTreeFile The orphaned VTree file to clean up
+ */
+ private void cleanupOrphanedVTreeFile(FileReference vTreeFile) {
+ try {
+ LOGGER.log(Level.TRACE, "Cleaning up orphaned VTree file: {}", vTreeFile.getAbsolutePath());
+ ioManager.delete(vTreeFile);
+ } catch (Exception e) {
+ LOGGER.log(Level.TRACE, "Failed to clean up orphaned VTree file {}: {}", vTreeFile.getAbsolutePath(),
+ e.getMessage());
+ }
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/LSMVTreeFlushOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/LSMVTreeFlushOperation.java
new file mode 100644
index 0000000000..01d4aeeed9
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/LSMVTreeFlushOperation.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.vector.impls;
+
+import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
+import org.apache.hyracks.storage.am.lsm.common.impls.FlushOperation;
+import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
+
+/**
+ * Flush operation for LSM Vector Clustering Tree.
+ * This operation flushes a memory component to disk, creating a new disk component.
+ */
+public class LSMVTreeFlushOperation extends FlushOperation {
+ private final LSMComponentFileReferences fileReferences;
+
+ public LSMVTreeFlushOperation(ILSMIndexAccessor accessor, FileReference flushTarget,
+ ILSMIOOperationCallback callback, String indexIdentifier) {
+ super(accessor, flushTarget, callback, indexIdentifier);
+ // Vector clustering tree only has a single primary file (no bloom filter or secondary indexes)
+ fileReferences = new LSMComponentFileReferences(target, null, null);
+ }
+
+ @Override
+ public LSMComponentFileReferences getComponentFiles() {
+ return fileReferences;
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/LSMVTreeIndexAccessor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/LSMVTreeIndexAccessor.java
new file mode 100644
index 0000000000..1cf27e13d9
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/LSMVTreeIndexAccessor.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.lsm.vector.impls;
+
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMHarness;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
+import org.apache.hyracks.storage.am.lsm.common.impls.LSMTreeIndexAccessor;
+import org.apache.hyracks.storage.common.IIndexAccessParameters;
+import org.apache.hyracks.storage.common.IIndexCursor;
+
+/**
+ * LSM Vector Clustering Tree Index Accessor.
+ *
+ * <p>Default search routes through the streaming {@link LSMVTreeSearchCursor} (registered on the
+ * parent accessor as the {@code cursorFactory}). Production ANN queries opt in to the quantized
+ * top-K cursor by setting {@link LSMVTreeTopKSearchCursor#IAP_KEY} to {@code Boolean.TRUE} in the
+ * index-access parameters; the test fixtures that don't set the flag get the streaming cursor,
+ * which is the same cursor used by component merges.
+ */
+public class LSMVTreeIndexAccessor extends LSMTreeIndexAccessor {
+
+ private final LSMVTree lsmVTree;
+
+ public LSMVTreeIndexAccessor(ILSMHarness lsmHarness, ILSMIndexOperationContext ctx, ICursorFactory cursorFactory,
+ LSMVTree lsmVTree) {
+ super(lsmHarness, ctx, cursorFactory);
+ this.lsmVTree = lsmVTree;
+ }
+
+ @Override
+ public IIndexCursor createSearchCursor(boolean exclusive) {
+ if (ctx instanceof LSMVTreeOpContext) {
+ IIndexAccessParameters iap = ((LSMVTreeOpContext) ctx).getIndexAccessParameters();
+ if (iap != null) {
+ Boolean useTopK = iap.getParameter(LSMVTreeTopKSearchCursor.IAP_KEY, Boolean.class);
+ if (Boolean.TRUE.equals(useTopK)) {
+ return lsmVTree.createTopKSearchCursor(ctx);
+ }
+ }
+ }
+ return super.createSearchCursor(exclusive);
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/LSMVTreeMemoryComponent.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/LSMVTreeMemoryComponent.java
new file mode 100644
index 0000000000..7061f52744
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/LSMVTreeMemoryComponent.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.lsm.vector.impls;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilter;
+import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
+import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMMemoryComponent;
+import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
+import org.apache.hyracks.storage.am.vector.impls.VTree;
+
+/**
+ * LSM Vector Clustering Tree memory component wrapper.
+ *
+ * This class wraps a VTree for use as an in-memory component
+ * in the LSM framework, providing the necessary lifecycle management and
+ * component filter support.
+ */
+public class LSMVTreeMemoryComponent extends AbstractLSMMemoryComponent {
+
+ private final VTree vtree;
+ private final LSMVTree lsmTree;
+
+ public LSMVTreeMemoryComponent(LSMVTree lsmTree, VTree vtree, IVirtualBufferCache vbc, ILSMComponentFilter filter) {
+ super(lsmTree, vbc, filter);
+ this.lsmTree = lsmTree;
+ this.vtree = vtree;
+ }
+
+ @Override
+ public VTree getIndex() {
+ return vtree;
+ }
+
+ @Override
+ public void validate() throws HyracksDataException {
+ vtree.validate();
+ }
+
+ @Override
+ public LSMComponentFileReferences getComponentFileRefs() {
+ return new LSMComponentFileReferences(vtree.getFileReference(), null, null);
+ }
+
+ @Override
+ public void cleanup() throws HyracksDataException {
+ // Reset VTree initialization state before cleanup so that
+ // static structure directory pages are re-created when the
+ // recycled component is next allocated for writes.
+ vtree.resetInitialization();
+ super.cleanup();
+ }
+
+ @Override
+ protected void doAllocate() throws HyracksDataException {
+ super.doAllocate();
+ // After re-creating the index, re-initialize the static structure
+ // directory pages from the shared static structure disk component.
+ lsmTree.reinitializeMemoryComponent(this);
+ }
+
+ /**
+ * Gets the vector dimensions from the underlying vector clustering tree.
+ */
+ public int getVectorDimensions() {
+ return lsmTree.getVectorDimensions();
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/LSMVTreeMergeOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/LSMVTreeMergeOperation.java
new file mode 100644
index 0000000000..bbe06c0a9f
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/LSMVTreeMergeOperation.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.vector.impls;
+
+import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
+import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
+import org.apache.hyracks.storage.am.lsm.common.impls.MergeOperation;
+import org.apache.hyracks.storage.common.IIndexCursor;
+import org.apache.hyracks.storage.common.IIndexCursorStats;
+
+/**
+ * Merge operation for LSM Vector Clustering Tree.
+ * This operation merges multiple disk components into a single new disk component.
+ */
+public class LSMVTreeMergeOperation extends MergeOperation {
+ private final LSMComponentFileReferences fileReferences;
+
+ public LSMVTreeMergeOperation(ILSMIndexAccessor accessor, IIndexCursor cursor, IIndexCursorStats stats,
+ FileReference target, ILSMIOOperationCallback callback, String indexIdentifier) {
+ super(accessor, target, callback, indexIdentifier, cursor, stats);
+ // Vector clustering tree only has a single primary file (no bloom filter or secondary indexes)
+ fileReferences = new LSMComponentFileReferences(target, null, null);
+ }
+
+ @Override
+ public LSMComponentFileReferences getComponentFiles() {
+ return fileReferences;
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/LSMVTreeOpContext.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/LSMVTreeOpContext.java
new file mode 100644
index 0000000000..97ff203e39
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/LSMVTreeOpContext.java
@@ -0,0 +1,177 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.lsm.vector.impls;
+
+import java.util.List;
+
+import org.apache.hyracks.api.dataflow.value.IBinaryComparator;
+import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.util.CleanupUtils;
+import org.apache.hyracks.storage.am.common.api.IExtendedModificationOperationCallback;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import org.apache.hyracks.storage.am.common.impls.IndexAccessParameters;
+import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback;
+import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMMemoryComponent;
+import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexOperationContext;
+import org.apache.hyracks.storage.am.vector.api.IVTreeDataFrame;
+import org.apache.hyracks.storage.am.vector.impls.VTree;
+import org.apache.hyracks.storage.am.vector.impls.VTreeOpContext;
+import org.apache.hyracks.storage.common.IIndexAccessParameters;
+import org.apache.hyracks.storage.common.ISearchOperationCallback;
+import org.apache.hyracks.storage.common.MultiComparator;
+import org.apache.hyracks.util.trace.ITracer;
+
+/**
+ * Operation context for LSM Vector Clustering Tree operations.
+ * Holds pre-created accessors and op contexts for every mutable component, plus the per-operation
+ * mutable cursor state. Mirrors the LSM B+Tree op-context pattern.
+ */
+public class LSMVTreeOpContext extends AbstractLSMIndexOperationContext {
+
+ // Immutable: pre-created resources, reused across operations
+ private final ITreeIndexFrameFactory insertDataFrameFactory;
+ private final ITreeIndexFrameFactory deleteDataFrameFactory;
+ private final IVTreeDataFrame insertDataFrame;
+ private final IVTreeDataFrame deleteDataFrame;
+ private final VTree[] mutableVTrees;
+ private final VTree.VTreeAccessor[] mutableVTreeAccessors;
+ private final VTreeOpContext[] mutableVTreeOpCtxs;
+ private final MultiComparator cmp;
+ private final LSMVTreeCursorInitialState searchInitialState;
+ private final IIndexAccessParameters iap;
+
+ // Mutable: switched per operation
+ private VTree.VTreeAccessor currentMutableVTreeAccessor;
+ private VTreeOpContext currentMutableVTreeOpCtx;
+ private boolean destroyed = false;
+
+ public LSMVTreeOpContext(LSMVTree lsmTree, List<ILSMMemoryComponent> mutableComponents, int[] treeFields,
+ int[] filterFields, IBinaryComparatorFactory[] filterCmpFactories,
+ IExtendedModificationOperationCallback modificationCallback, ISearchOperationCallback searchCallback,
+ ITracer tracer, IIndexAccessParameters iap) {
+ super(lsmTree, treeFields, filterFields, filterCmpFactories, searchCallback, modificationCallback, tracer);
+
+ this.iap = iap;
+ this.insertDataFrameFactory = lsmTree.getInsertDataFrameFactory();
+ this.deleteDataFrameFactory = lsmTree.getDeleteDataFrameFactory();
+
+ // ADM-aware comparators: handle the 1-byte type tag prefix for ADOUBLE, AINT32, etc.
+ IBinaryComparatorFactory[] cmpFactories = lsmTree.getCmpFactories();
+ IBinaryComparator[] comparators = new IBinaryComparator[cmpFactories.length];
+ for (int i = 0; i < comparators.length; i++) {
+ comparators[i] = cmpFactories[i].createBinaryComparator();
+ }
+ this.cmp = new MultiComparator(comparators);
+
+ this.insertDataFrame = (IVTreeDataFrame) insertDataFrameFactory.createFrame();
+ this.deleteDataFrame = (IVTreeDataFrame) deleteDataFrameFactory.createFrame();
+
+ // Pre-create accessors and op contexts for ALL mutable components
+ mutableVTrees = new VTree[mutableComponents.size()];
+ mutableVTreeAccessors = new VTree.VTreeAccessor[mutableComponents.size()];
+ mutableVTreeOpCtxs = new VTreeOpContext[mutableComponents.size()];
+
+ for (int i = 0; i < mutableComponents.size(); i++) {
+ LSMVTreeMemoryComponent mutableComponent = (LSMVTreeMemoryComponent) mutableComponents.get(i);
+ mutableVTrees[i] = mutableComponent.getIndex();
+ IIndexAccessParameters componentIap =
+ new IndexAccessParameters(modificationCallback, NoOpOperationCallback.INSTANCE);
+ mutableVTreeAccessors[i] = (VTree.VTreeAccessor) mutableVTrees[i].createAccessor(componentIap);
+ mutableVTreeOpCtxs[i] = mutableVTreeAccessors[i].getOpContext();
+ }
+
+ // Search state defaults to insert-mode data frame; switched on delete/upsert path
+ this.searchInitialState = new LSMVTreeCursorInitialState(lsmTree.getInteriorFrameFactory(),
+ lsmTree.getLeafFrameFactory(), lsmTree.getMetadataFrameFactory(), insertDataFrameFactory, cmp,
+ lsmTree.getHarness(), null, searchCallback, componentHolder);
+ }
+
+ @Override
+ public void setOperation(IndexOperation newOp) {
+ reset();
+ this.op = newOp;
+ }
+
+ public void setInsertMode() {
+ currentMutableVTreeOpCtx.setDataFrame(insertDataFrame);
+ currentMutableVTreeOpCtx.setDataFrameFactory(insertDataFrameFactory);
+ }
+
+ public void setDeleteMode() {
+ currentMutableVTreeOpCtx.setDataFrame(deleteDataFrame);
+ currentMutableVTreeOpCtx.setDataFrameFactory(deleteDataFrameFactory);
+ }
+
+ @Override
+ public void setCurrentMutableComponentId(int currentMutableComponentId) {
+ setCurrentMutableVTreeAccessor(mutableVTreeAccessors[currentMutableComponentId]);
+ currentMutableVTreeOpCtx = mutableVTreeOpCtxs[currentMutableComponentId];
+ switch (op) {
+ case SEARCH:
+ case DISKORDERSCAN:
+ case UPDATE:
+ // Data frame is set by the calling path; nothing to do here.
+ break;
+ case UPSERT:
+ case INSERT:
+ setInsertMode();
+ break;
+ case PHYSICALDELETE:
+ case DELETE:
+ setDeleteMode();
+ break;
+ }
+ }
+
+ public VTree.VTreeAccessor getCurrentMutableVTreeAccessor() {
+ return currentMutableVTreeAccessor;
+ }
+
+ public void setCurrentMutableVTreeAccessor(VTree.VTreeAccessor accessor) {
+ this.currentMutableVTreeAccessor = accessor;
+ }
+
+ public LSMVTreeCursorInitialState getSearchInitialState() {
+ return searchInitialState;
+ }
+
+ public MultiComparator getCmp() {
+ return cmp;
+ }
+
+ public IIndexAccessParameters getIndexAccessParameters() {
+ return iap;
+ }
+
+ @Override
+ public void destroy() throws HyracksDataException {
+ if (destroyed) {
+ return;
+ }
+ destroyed = true;
+ Throwable failure = CleanupUtils.destroy(null, mutableVTreeAccessors);
+ failure = CleanupUtils.destroy(failure, mutableVTreeOpCtxs);
+ if (failure != null) {
+ throw HyracksDataException.create(failure);
+ }
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/LSMVTreeSearchCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/LSMVTreeSearchCursor.java
new file mode 100644
index 0000000000..74c1f8422b
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/LSMVTreeSearchCursor.java
@@ -0,0 +1,1087 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.lsm.vector.impls;
+
+import java.util.Arrays;
+import java.util.PriorityQueue;
+import java.util.Set;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.util.CleanupUtils;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.storage.am.common.api.ITupleFilter;
+import org.apache.hyracks.storage.am.common.tuples.ReferenceFrameTupleReference;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent.LSMComponentType;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMTreeTupleReference;
+import org.apache.hyracks.storage.am.lsm.common.impls.LSMIndexSearchCursor;
+import org.apache.hyracks.storage.am.vector.impls.ClusterSearchResult;
+import org.apache.hyracks.storage.am.vector.impls.VTree;
+import org.apache.hyracks.storage.am.vector.impls.VTree.VTreeAccessor;
+import org.apache.hyracks.storage.am.vector.impls.VTreeSearchCursor;
+import org.apache.hyracks.storage.am.vector.impls.VTreeSearchPredicate;
+import org.apache.hyracks.storage.am.vector.utils.VTreeDataTupleAccessor;
+import org.apache.hyracks.storage.common.ICursorInitialState;
+import org.apache.hyracks.storage.common.IIndexCursor;
+import org.apache.hyracks.storage.common.IIndexCursorStats;
+import org.apache.hyracks.storage.common.ISearchPredicate;
+import org.apache.hyracks.storage.common.MultiComparator;
+import org.apache.hyracks.storage.common.NoOpIndexCursorStats;
+import org.apache.hyracks.storage.common.util.IndexCursorUtils;
+import org.apache.hyracks.util.annotations.AiProvenance;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+/**
+ * LSM search cursor for Vector Clustering Tree.
+ *
+ * This cursor coordinates searches across multiple LSM components (memory and disk)
+ * by delegating to VTreeSearchCursor for each component and merging results.
+ *
+ * Supports two modes:
+ *
+ * 1. Query Mode (fullScanMode=false):
+ * - Used for ANN (Approximate Nearest Neighbor) queries
+ * - Level-wise + DFS cluster selection (nprobe logic)
+ * - All components advance to the SAME cluster (synchronized exploration)
+ * - Early termination based on nprobe and K parameters
+ * - INCLUDE field filtering support
+ *
+ * 2. Full Scan Mode (fullScanMode=true):
+ * - Used for merge operations where we need ALL data
+ * - Sequential cluster iteration (0 → 1 → 2 → ...)
+ * - All components advance through clusters TOGETHER in lock-step
+ * - No early termination (must process all data)
+ * - Returns antimatter tuples for proper reconciliation
+ *
+ * Following the pattern of LSMBTreeRangeSearchCursor:
+ * - Extends LSMIndexSearchCursor for priority queue and component switching infrastructure
+ * - Creates VTreeAccessor for each component
+ * - Handles component state changes (memory → disk transitions)
+ * - Uses priority queue to merge results sorted by <distance, primary_key> (query) or <primary_key> (merge)
+ * - Filters antimatter tuples and handles matter/antimatter cancellation
+ */
+public class LSMVTreeSearchCursor extends LSMIndexSearchCursor {
+ private static final Logger LOGGER = LogManager.getLogger();
+
+ // Accessor array for each component's VTree
+ private VTreeAccessor[] vTreeAccessors;
+
+ // Track component types to detect memory → disk transitions
+ protected boolean[] isMemoryComponent;
+
+ // Store search predicate for component switching
+ private ISearchPredicate searchPredicate;
+
+ // Track K (target limit) for cluster advancement decisions
+ private int K;
+
+ // Track nprobe (minimum clusters to probe before checking K)
+ private int nprobe;
+
+ // Cluster selection strategy (encapsulates level-wise + DFS logic)
+ private IClusterSelectionStrategy clusterStrategy;
+
+ // Track cluster exhaustion state for synchronized advancement
+ private boolean[] clusterExhausted; // Whether each component exhausted its current cluster
+ private boolean stopAdvancing; // Flag to stop advancing after K reached
+
+ // Debug counters to track reconciliation
+ private int totalTuplesPopped; // Total tuples popped from priority queue (including cancelled)
+ private int antimatterTuplesDetected; // Antimatter tuples detected
+ private int cancellationsMade; // Matter tuples cancelled by antimatter
+ private int getTupleCallCount; // Track how many times doGetTuple() is called
+ private int tuplesFilteredOut; // Tuples that failed INCLUDE field filter
+ private int validTuplesFromCurrentCluster; // Valid tuples output from current cluster (for empty-cluster nprobe)
+
+ // Tuple filter for INCLUDE field predicates (e.g., year > 2000)
+ // When set, only tuples passing this filter are returned and counted toward K
+ private ITupleFilter tupleFilter;
+
+ // Wrapper to convert ITupleReference to IFrameTupleReference for filter evaluation
+ private ReferenceFrameTupleReference referenceFilterTuple;
+
+ // Full-scan mode flag (for merge operations)
+ // true = merge mode (sequential cluster iteration, no early termination)
+ // false = query mode (level-wise + DFS, nprobe/K-based early termination)
+ private final boolean fullScanMode;
+
+ // Field index where primary keys start in the data tuple
+ // Non-quantized format: 2 (distance, centroidId, PK...)
+ // Quantized format: 4 (distance, centroidId, quantized_distance, quantized_embedding, PK...)
+ private int pkStartField;
+
+ // Number of primary key fields (from the index). The reconciliation/ordering key is exactly
+ // <distance (field 0), PK fields> — secondary fields between them (centroidId, and for
+ // quantized layouts quantized_distance/quantized_embedding) AND trailing INCLUDE fields are
+ // excluded, since they may legitimately differ between matter/antimatter twins.
+ private int numPrimaryKeyFields;
+
+ public LSMVTreeSearchCursor(ILSMIndexOperationContext opCtx) {
+ this(opCtx, false, false, NoOpIndexCursorStats.INSTANCE);
+ }
+
+ public LSMVTreeSearchCursor(ILSMIndexOperationContext opCtx, boolean returnDeletedTuples, boolean fullScanMode,
+ IIndexCursorStats stats) {
+ super(opCtx, returnDeletedTuples, stats);
+ this.fullScanMode = fullScanMode;
+ }
+
+ @Override
+ public void doOpen(ICursorInitialState initialState, ISearchPredicate searchPred) throws HyracksDataException {
+ // Get LSM-specific initial state
+ LSMVTreeCursorInitialState lsmInitialState = (LSMVTreeCursorInitialState) initialState;
+
+ // Save search predicate for component switching
+ this.searchPredicate = searchPred;
+
+ // Extract K from search predicate for cluster advancement decisions
+ this.K = extractK(searchPred);
+
+ // Field layout depends only on whether the index is quantized; derive pkStartField from that
+ // single source (VTreeDataTupleAccessor) rather than a predicate value set at a distance.
+ this.pkStartField = new VTreeDataTupleAccessor(((LSMVTree) opCtx.getIndex()).isQuantized()).pkStartField();
+
+ // The comparison key ends after the primary keys (INCLUDE fields are not part of it)
+ this.numPrimaryKeyFields = ((LSMVTree) opCtx.getIndex()).getNumPrimaryKeyFields();
+
+ // Extract minProbeFraction and epsilon from search predicate
+ double minProbeFraction = extractMinProbeFraction(searchPred);
+ double epsilon = extractEpsilon(searchPred);
+ this.nprobe = 1; // Will be computed by NprobeClusterSelectionStrategy from minProbeFraction
+
+ // Create cluster selection strategy based on mode
+ if (fullScanMode) {
+ // Merge mode: sequential cluster iteration, no early termination
+ this.clusterStrategy = new SequentialClusterSelectionStrategy();
+ } else {
+ // Query mode: level-wise + DFS, minProbeFraction/K-based early termination
+ this.clusterStrategy = new NprobeClusterSelectionStrategy(minProbeFraction, epsilon);
+ }
+
+ // Extract tuple filter from search predicate for INCLUDE field predicates
+ this.tupleFilter = extractTupleFilter(searchPred);
+ if (this.tupleFilter != null) {
+ this.referenceFilterTuple = new ReferenceFrameTupleReference();
+ }
+
+ // Initialize debug counters
+ this.totalTuplesPopped = 0;
+ this.antimatterTuplesDetected = 0;
+ this.cancellationsMade = 0;
+ this.getTupleCallCount = 0;
+ this.tuplesFilteredOut = 0;
+ this.validTuplesFromCurrentCluster = 0;
+
+ // Set up comparator and operational components
+ cmp = lsmInitialState.getOriginalKeyComparator();
+ operationalComponents = lsmInitialState.getOperationalComponents();
+ lsmHarness = lsmInitialState.getLSMHarness();
+
+ // For vector index, we don't need mutable component special handling initially
+ includeMutableComponent = false;
+
+ int numVTrees = operationalComponents.size();
+
+ // Initialize cluster tracking arrays for synchronized advancement
+ clusterExhausted = new boolean[numVTrees];
+ Arrays.fill(clusterExhausted, false);
+ stopAdvancing = false;
+
+ // Initialize or resize accessor/cursor arrays
+ if (rangeCursors == null) {
+ // First open: create arrays
+ rangeCursors = new IIndexCursor[numVTrees];
+ vTreeAccessors = new VTreeAccessor[numVTrees];
+ isMemoryComponent = new boolean[numVTrees];
+ } else if (rangeCursors.length != numVTrees) {
+ // Component count changed (due to flush/merge): destroy and recreate
+ Throwable failure = CleanupUtils.destroy(null, vTreeAccessors);
+ vTreeAccessors = null;
+ failure = CleanupUtils.destroy(failure, rangeCursors);
+ if (failure != null) {
+ throw HyracksDataException.create(failure);
+ }
+ rangeCursors = new IIndexCursor[numVTrees];
+ vTreeAccessors = new VTreeAccessor[numVTrees];
+ isMemoryComponent = new boolean[numVTrees];
+ }
+
+ // Create accessors and cursors for each component
+ for (int i = 0; i < numVTrees; i++) {
+ ILSMComponent component = operationalComponents.get(i);
+ LSMComponentType type = component.getType();
+
+ // Track if this is a memory component
+ if (component.getType() == LSMComponentType.MEMORY) {
+ includeMutableComponent = true;
+ }
+
+ // Check if we need to destroy incompatible accessor/cursor
+ if (vTreeAccessors[i] == null || destroyIncompatible(component, i)) {
+ vTreeAccessors[i] = createAccessor(component, i);
+ rangeCursors[i] = createCursor(type, vTreeAccessors[i]);
+ } else {
+ // Re-use existing cursor
+ rangeCursors[i].close();
+ }
+
+ isMemoryComponent[i] = type == LSMComponentType.MEMORY;
+ }
+
+ // Open all cursors with the search predicate
+ IndexCursorUtils.open(vTreeAccessors, rangeCursors, searchPred);
+
+ LOGGER.trace("doOpen: numComponents={}, K={}, nprobe={}, pkStartField={}", numVTrees, K, nprobe, pkStartField);
+
+ // Initialize strategy and set up DFS fallback
+ if (numVTrees > 0) {
+ VTreeSearchCursor firstCursor = (VTreeSearchCursor) rangeCursors[0];
+ double[] queryVector = firstCursor.getQueryVector();
+
+ // Initialize strategy with first component's tree
+ ILSMComponent firstComponent = operationalComponents.get(0);
+ VTree vTree = (VTree) firstComponent.getIndex();
+ clusterStrategy.initialize(vTree, queryVector, firstCursor.getDistanceFunction(), K);
+
+ // Set first cursor for DFS fallback
+ clusterStrategy.setFirstCursorForDFS(firstCursor);
+
+ // Pass shared visited set from strategy to all VTreeSearchCursors
+ Set<Integer> visitedSet = clusterStrategy.getVisitedCentroidIds();
+ for (int i = 0; i < numVTrees; i++) {
+ if (rangeCursors[i] instanceof VTreeSearchCursor) {
+ ((VTreeSearchCursor) rangeCursors[i]).setSharedVisitedSet(visitedSet);
+ }
+ }
+
+ // DFS in each cursor may pick a DIFFERENT closest cluster than level-wise (level-wise
+ // does a global sort across ALL centroids, DFS is greedy). Re-open ALL cursors to
+ // level-wise[0] so every component agrees on the first cluster.
+ ClusterSearchResult firstCluster = clusterStrategy.getFirstCluster();
+ if (firstCluster != null) {
+ ClusterSearchResult dfsCluster = firstCursor.getCurrentClusterResult();
+ if (dfsCluster != null && dfsCluster.centroidId != firstCluster.centroidId) {
+ LOGGER.trace(
+ "DFS found cid={} but level-wise[0] is cid={} - re-opening all cursors to level-wise[0]",
+ dfsCluster.centroidId, firstCluster.centroidId);
+
+ for (int i = 0; i < numVTrees; i++) {
+ if (rangeCursors[i] instanceof VTreeSearchCursor) {
+ VTreeSearchCursor vcCursor = (VTreeSearchCursor) rangeCursors[i];
+ // Reset cluster counter since we're replacing the initial DFS cluster
+ // with level-wise[0], not advancing to a second cluster
+ vcCursor.resetClustersProbed();
+ vcCursor.openClusterByResult(firstCluster);
+ }
+ }
+
+ visitedSet.remove(dfsCluster.centroidId);
+ }
+
+ LOGGER.trace("Computed {} level-wise clusters, first cluster cid={} marked visited",
+ clusterStrategy.getLevelWiseClusterCount(), firstCluster.centroidId);
+ } else {
+ // No level-wise clusters - mark first cluster from DFS as visited
+ ClusterSearchResult dfsFallbackCluster = firstCursor.getCurrentClusterResult();
+ if (dfsFallbackCluster != null) {
+ visitedSet.add(dfsFallbackCluster.centroidId);
+ LOGGER.trace("Level-wise disabled, marked first DFS cluster as visited: cid={}",
+ dfsFallbackCluster.centroidId);
+ }
+ }
+ }
+
+ // Initialize priority queue for merging results from all components
+ try {
+ setPriorityQueueComparator();
+ initPriorityQueue();
+ } catch (Throwable th) { // NOSONAR Must catch all
+ IndexCursorUtils.close(rangeCursors, th);
+ throw HyracksDataException.create(th);
+ }
+ }
+
+ @Override
+ public void initPriorityQueue() throws HyracksDataException {
+ // Initialize priority queue and populate with first element from each cursor
+ int pqInitSize = (rangeCursors.length > 0) ? rangeCursors.length : 1;
+ if (outputPriorityQueue == null) {
+ outputPriorityQueue = new PriorityQueue<>(pqInitSize, pqCmp);
+ pqes = new PriorityQueueElement[pqInitSize];
+ for (int i = 0; i < pqInitSize; i++) {
+ pqes[i] = new PriorityQueueElement(i);
+ }
+ } else {
+ outputPriorityQueue.clear();
+ if (pqInitSize != pqes.length) {
+ pqes = new PriorityQueueElement[pqInitSize];
+ for (int i = 0; i < pqInitSize; i++) {
+ pqes[i] = new PriorityQueueElement(i);
+ }
+ }
+ }
+
+ // Populate priority queue with first element from each cursor
+ // If a cursor has no data (empty cluster), mark it as exhausted
+ for (int i = 0; i < rangeCursors.length; i++) {
+ if (rangeCursors[i].hasNext()) {
+ rangeCursors[i].next();
+ ITupleReference tuple = rangeCursors[i].getTuple();
+ pqes[i].reset(tuple);
+ outputPriorityQueue.offer(pqes[i]);
+ } else {
+ // Cursor has no data in initial cluster - mark as exhausted
+ clusterExhausted[i] = true;
+ LOGGER.trace("Component {} has empty initial cluster (marked exhausted)", i);
+ }
+ }
+
+ // If ALL components started with empty clusters, advance all to the next cluster.
+ boolean allInitiallyExhausted = true;
+ for (int i = 0; i < clusterExhausted.length; i++) {
+ if (!clusterExhausted[i]) {
+ allInitiallyExhausted = false;
+ break;
+ }
+ }
+
+ if (allInitiallyExhausted) {
+ LOGGER.trace("All components have empty initial cluster, advancing to next");
+ advanceAllComponentsToNextCluster();
+ }
+ }
+
+ @Override
+ protected void pushIntoQueueFromCursorAndReplaceThisElement(PriorityQueueElement e) throws HyracksDataException {
+ // Get next tuple from this cursor and add to priority queue
+ int cursorIndex = e.getCursorIndex();
+ if (rangeCursors[cursorIndex].hasNext()) {
+ rangeCursors[cursorIndex].next();
+ e.reset(rangeCursors[cursorIndex].getTuple());
+ outputPriorityQueue.offer(e);
+ }
+ }
+
+ /**
+ * Check if accessor/cursor needs to be destroyed due to component type change.
+ * This happens when a memory component is replaced with a disk component.
+ */
+ private boolean destroyIncompatible(ILSMComponent component, int index) throws HyracksDataException {
+ // XOR: if component type changed (memory → disk or disk → memory)
+ if (component.getType() == LSMComponentType.MEMORY ^ isMemoryComponent[index]) {
+ Throwable failure = CleanupUtils.destroy(null, vTreeAccessors[index]);
+ vTreeAccessors[index] = null;
+ failure = CleanupUtils.destroy(failure, rangeCursors[index]);
+ rangeCursors[index] = null;
+ if (failure != null) {
+ throw HyracksDataException.create(failure);
+ }
+ return true;
+ }
+ return false;
+ }
+
+ /**
+ * Create accessor for a VTree component.
+ */
+ protected VTreeAccessor createAccessor(ILSMComponent component, int index) throws HyracksDataException {
+ VTree vTree = (VTree) component.getIndex();
+ // Get iap from operation context instead of using cursor's default iap
+ LSMVTreeOpContext vcOpCtx = (LSMVTreeOpContext) opCtx;
+ return (VTreeAccessor) vTree.createAccessor(vcOpCtx.getIndexAccessParameters());
+ }
+
+ /**
+ * Create cursor for a VTree component.
+ * Passes fullScanMode to enable sequential cluster iteration for merge operations.
+ */
+ protected IIndexCursor createCursor(LSMComponentType type, VTreeAccessor accessor) throws HyracksDataException {
+ return accessor.createSearchCursor(false, fullScanMode);
+ }
+
+ @Override
+ public void doClose() throws HyracksDataException {
+ if (LOGGER.isTraceEnabled()) {
+ LOGGER.trace(
+ "Search summary: mode={}, K={}, nprobe={}, levelWiseClusters={}, levelWiseComplete={},"
+ + " minClustersProbed={}, tuplesProcessed={}, antimatter={}, cancellations={},"
+ + " filteredOut={}, getTupleCalls={}",
+ fullScanMode ? "MERGE" : "QUERY", K, nprobe,
+ clusterStrategy != null ? clusterStrategy.getLevelWiseClusterCount() : 0,
+ clusterStrategy != null && clusterStrategy.isLevelWisePhaseComplete(), getMinClustersProbed(),
+ totalTuplesPopped, antimatterTuplesDetected, cancellationsMade, tuplesFilteredOut,
+ getTupleCallCount);
+ }
+ super.doClose();
+ }
+
+ @Override
+ public boolean doHasNext() throws HyracksDataException {
+ hasNextCallCount++;
+ checkPriorityQueue();
+ // Use priority queue - check if there's a valid element at the top
+ return !outputPriorityQueue.isEmpty();
+ }
+
+ @Override
+ public void doNext() throws HyracksDataException {
+ // Pop element from priority queue and mark for replacement
+ outputElement = outputPriorityQueue.poll();
+ needPushElementIntoQueue = true;
+
+ // Track total tuples popped (including those that may be cancelled)
+ totalTuplesPopped++;
+ }
+
+ @Override
+ protected void checkPriorityQueue() throws HyracksDataException {
+ // Periodically check if memory components need to be swapped with disk components
+ checkAndSwitchMemoryComponentsIfNeeded();
+
+ // Process queue until we find a valid output tuple or queue is exhausted
+ while (!outputPriorityQueue.isEmpty() || needPushElementIntoQueue) {
+ if (outputPriorityQueue.isEmpty()) {
+ // Queue empty but pending element exists - refill it
+ refillQueueFromPendingElement();
+ continue;
+ }
+
+ PriorityQueueElement checkElement = outputPriorityQueue.peek();
+
+ if (outputElement == null) {
+ // No pending element - process top of queue
+ if (processTopElement(checkElement)) {
+ break; // Found valid output tuple
+ }
+ } else {
+ // Have pending element - check for antimatter cancellation
+ processWithPendingElement(checkElement);
+ }
+ }
+ }
+
+ /**
+ * Check if memory components should be swapped with disk components.
+ */
+ private void checkAndSwitchMemoryComponentsIfNeeded() throws HyracksDataException {
+ if (hasNextCallCount >= SWITCH_COMPONENT_CYCLE) {
+ replaceMemoryComponentWithDiskComponentIfNeeded();
+ hasNextCallCount = 0;
+ }
+ }
+
+ /**
+ * Process the top element of the priority queue when no pending element exists.
+ *
+ * @param checkElement the element at top of queue
+ * @return true if this is a valid output tuple, false if processing should continue
+ */
+ private boolean processTopElement(PriorityQueueElement checkElement) throws HyracksDataException {
+ // Check if top element is antimatter
+ if (isDeleted(checkElement) && !returnDeletedTuples) {
+ // Antimatter tuple - hold for cancellation check with next tuple
+ outputElement = outputPriorityQueue.poll();
+ needPushElementIntoQueue = true;
+ antimatterTuplesDetected++;
+ return false; // Continue processing
+ }
+
+ // Matter tuple - apply INCLUDE field filter if configured
+ if (!passesTupleFilter(checkElement)) {
+ // Tuple fails filter - skip without counting toward K
+ PriorityQueueElement skippedElement = outputPriorityQueue.poll();
+ pushIntoQueueAndAdvanceClusterIfNeeded(skippedElement);
+ return false; // Continue processing
+ }
+
+ // Valid output tuple - passed antimatter reconciliation and filter
+ return true;
+ }
+
+ /**
+ * Check if tuple passes the INCLUDE field filter.
+ * Filter is applied AFTER antimatter reconciliation because:
+ * 1. Antimatter from newer components comes first in queue (lower cursorIndex)
+ * 2. By the time we reach here, any matching antimatter has cancelled this tuple
+ * 3. Tuples that fail filter should NOT be counted toward K
+ *
+ * @param element the element to check
+ * @return true if tuple passes filter (or no filter configured), false otherwise
+ */
+ private boolean passesTupleFilter(PriorityQueueElement element) throws HyracksDataException {
+ if (tupleFilter == null) {
+ return true;
+ }
+
+ ITupleReference tuple = element.getTuple();
+ referenceFilterTuple.reset(tuple);
+
+ if (tupleFilter.accept(referenceFilterTuple)) {
+ return true;
+ }
+
+ // Tuple fails filter
+ tuplesFilteredOut++;
+ return false;
+ }
+
+ /**
+ * Process queue element when we have a pending element (potential antimatter cancellation).
+ *
+ * @param checkElement the element at top of queue
+ */
+ private void processWithPendingElement(PriorityQueueElement checkElement) throws HyracksDataException {
+ if (compare(cmp, outputElement.getTuple(), checkElement.getTuple()) == 0) {
+ // Same <distance, PK> as the reference (a held antimatter, or the just-output matter). This
+ // head is an older, superseded version of that key. Discard ONLY this head and pull the next
+ // tuple from ITS cursor, keeping outputElement as the running reference and letting
+ // checkPriorityQueue() keep looping so EVERY older same-key version drains — not just one.
+ //
+ // This mirrors LSMIndexSearchCursor's reconciliation. Draining only a single matter/antimatter
+ // pair (and resetting the reference) would leak the 3rd+ version of the same key: e.g.
+ // insert -> delete -> re-insert of the same vector across >=3 components would resurface the
+ // deleted row and duplicate the key.
+ //
+ // Safe w.r.t. cluster advancing: the queue holds at most one element per component and the
+ // reference's own component is held out (not advanced) during the drain, so its
+ // clusterExhausted[] entry stays false and advanceAllComponentsToNextCluster() cannot fire
+ // here (which would otherwise reset the reused pqes[] element backing outputElement).
+ cancellationsMade++;
+ PriorityQueueElement duplicate = outputPriorityQueue.poll();
+ pushIntoQueueAndAdvanceClusterIfNeeded(duplicate);
+ } else {
+ // Different key - refill pending element's cursor
+ refillPendingElementCursor();
+ }
+ }
+
+ /**
+ * Refill the pending element's cursor and reset state.
+ */
+ private void refillPendingElementCursor() throws HyracksDataException {
+ if (needPushElementIntoQueue) {
+ pushIntoQueueAndAdvanceClusterIfNeeded(outputElement);
+ needPushElementIntoQueue = false;
+ }
+ outputElement = null;
+ }
+
+ /**
+ * Refill queue from pending element when queue is empty.
+ */
+ private void refillQueueFromPendingElement() throws HyracksDataException {
+ pushIntoQueueAndAdvanceClusterIfNeeded(outputElement);
+ needPushElementIntoQueue = false;
+ outputElement = null;
+ }
+
+ /**
+ * Replace memory components with disk components if they were flushed.
+ * This is called periodically to handle concurrent flushes during search.
+ */
+ private void replaceMemoryComponentWithDiskComponentIfNeeded() throws HyracksDataException {
+ int replaceFrom = findFirstComponentToReplace();
+ if (replaceFrom < 0) {
+ // No switch needed
+ return;
+ }
+
+ // Ask LSM harness to replace memory components with their flushed disk versions
+ opCtx.getIndex().getHarness().replaceMemoryComponentsWithDiskComponents(getOpCtx(), replaceFrom);
+
+ // Redo searches on the new disk components
+ for (int i = replaceFrom; i < switchRequest.length && i < operationalComponents.size(); i++) {
+ if (switchRequest[i]) {
+ ILSMComponent component = operationalComponents.get(i);
+
+ // Check if first component is now disk (no more mutable component)
+ if (i == 0 && component.getType() != LSMComponentType.MEMORY) {
+ includeMutableComponent = false;
+ }
+
+ // If we had an active element from this component, restart search from that point
+ if (switchedElements[i] != null) {
+ rangeCursors[i].close();
+ // Use the op-context's real access parameters (with the vector accessor factory + task
+ // context), NOT the cursor's NoOp default iap — otherwise the reopened disk cursor
+ // searches with degraded/missing config and can drop that component's results.
+ vTreeAccessors[i] = createAccessor(component, i);
+ vTreeAccessors[i].search(rangeCursors[i], searchPredicate);
+
+ // Try to position cursor at the same element
+ if (rangeCursors[i].hasNext()) {
+ rangeCursors[i].next();
+ switchedElements[i].reset(rangeCursors[i].getTuple());
+ }
+ }
+ }
+ switchRequest[i] = false;
+ switchedElements[i] = null;
+ // Any failed switch makes further switches pointless
+ switchPossible = switchPossible && operationalComponents.get(i).getType() == LSMComponentType.DISK;
+ }
+ }
+
+ /**
+ * Find the first component that needs to be replaced (has been flushed).
+ * Returns the index of the first component to replace, or -1 if no replacement needed.
+ */
+ private int findFirstComponentToReplace() throws HyracksDataException {
+ int replaceFrom = -1;
+
+ if (!switchPossible) {
+ return replaceFrom;
+ }
+
+ for (int i = 0; i < operationalComponents.size(); i++) {
+ ILSMComponent component = operationalComponents.get(i);
+
+ if (component.getType() == LSMComponentType.DISK) {
+ if (i == 0) {
+ // First component is already disk, no more switching possible
+ switchPossible = false;
+ }
+ break;
+ } else if (component.getState() == ILSMComponent.ComponentState.UNREADABLE_UNWRITABLE) {
+ // Component was flushed while cursor is active - mark for replacement
+ if (replaceFrom < 0) {
+ replaceFrom = i;
+ }
+
+ // Find the element from this cursor (if any)
+ PriorityQueueElement element = findElementInQueue(i);
+
+ // Mark this cursor for switching
+ rangeCursors[i].close();
+ switchRequest[i] = true;
+ switchedElements[i] = element;
+ }
+ }
+
+ return replaceFrom;
+ }
+
+ /**
+ * Find an element in the priority queue or output element from a specific cursor.
+ */
+ private PriorityQueueElement findElementInQueue(int cursorIndex) {
+ // Check if output element is from this cursor
+ if (outputElement != null && outputElement.getCursorIndex() == cursorIndex) {
+ return outputElement;
+ }
+
+ // Search in priority queue
+ for (PriorityQueueElement element : outputPriorityQueue) {
+ if (element.getCursorIndex() == cursorIndex) {
+ return element;
+ }
+ }
+
+ return null;
+ }
+
+ @Override
+ protected void setPriorityQueueComparator() {
+ // For vector index: sort by distance (field 0), then primary key (field pkStartField+)
+ // Skip secondary fields between distance and PKs (centroidId, and optionally quantized fields)
+ if (pqCmp == null || pqCmp.getMultiComparator() != cmp) {
+ pqCmp = new VectorPriorityQueueComparator(cmp);
+ }
+ }
+
+ /**
+ * Custom priority queue comparator for vector index tuples.
+ * Compares field 0 (distance) then fields from pkStartField onward (PKs + includes),
+ * skipping secondary fields (centroidId, and optionally quantized_distance/quantized_embedding).
+ */
+ private class VectorPriorityQueueComparator extends PriorityQueueComparator {
+
+ public VectorPriorityQueueComparator(MultiComparator cmp) {
+ super(cmp);
+ }
+
+ @Override
+ public int compare(PriorityQueueElement elementA, PriorityQueueElement elementB) {
+ ITupleReference tupleA = elementA.getTuple();
+ ITupleReference tupleB = elementB.getTuple();
+
+ int result;
+ try {
+ // Merge ordering shares the exact <distance, PK> key with antimatter cancellation.
+ result = compareKey(cmp, tupleA, tupleB);
+ } catch (Throwable e) {
+ throw new IllegalArgumentException(e);
+ }
+ if (result != 0) {
+ return result;
+ }
+
+ // Tiebreaker: prefer tuples from earlier components
+ if (elementA.getCursorIndex() > elementB.getCursorIndex()) {
+ return 1;
+ } else {
+ return -1;
+ }
+ }
+ }
+
+ @Override
+ public ITupleReference doGetTuple() {
+ getTupleCallCount++;
+ if (outputElement != null) {
+ validTuplesFromCurrentCluster++;
+ }
+ return outputElement != null ? outputElement.getTuple() : null;
+ }
+
+ @Override
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_FABLE_5, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.ASSISTED)
+ protected int compare(MultiComparator cmp, ITupleReference tupleA, ITupleReference tupleB)
+ throws HyracksDataException {
+ // Antimatter cancellation uses the same <distance, PK> key as the merge ordering.
+ return compareKey(cmp, tupleA, tupleB);
+ }
+
+ /**
+ * The <distance, PK> key shared by antimatter cancellation
+ * ({@link #compare(MultiComparator, ITupleReference, ITupleReference)}) and merge ordering
+ * ({@link VectorPriorityQueueComparator#compare}). Both must define the exact same key, otherwise a
+ * delete marker and its live twin could sort apart and fail to cancel.
+ *
+ * <p>Tuple format (non-quantized): {@code [distance, centroidId, PKs..., includes...]};
+ * quantized: {@code [distance, centroidId, quantized_distance, quantized_embedding, PKs..., includes...]}.
+ * The key is exactly {@code <distance (field 0), PK fields>}: secondary fields (centroidId, and for
+ * quantized layouts quantized_distance/quantized_embedding) and trailing INCLUDE fields are excluded —
+ * they may differ between matter/antimatter twins.
+ */
+ private int compareKey(MultiComparator cmp, ITupleReference tupleA, ITupleReference tupleB)
+ throws HyracksDataException {
+ // Compare field 0 (distance) using ADM-aware comparator 0
+ int result = cmp.getComparators()[0].compare(tupleA.getFieldData(0), tupleA.getFieldStart(0),
+ tupleA.getFieldLength(0), tupleB.getFieldData(0), tupleB.getFieldStart(0), tupleB.getFieldLength(0));
+
+ if (result != 0) {
+ return result;
+ }
+
+ // Compare the PK fields starting at pkStartField
+ int numRemainingFields = Math.min(cmp.getComparators().length - pkStartField, numPrimaryKeyFields);
+ for (int i = 0; i < numRemainingFields; i++) {
+ int fieldIdx = pkStartField + i;
+ int cmpIdx = pkStartField + i;
+
+ // Check if field exists in tuple before comparing
+ if (fieldIdx >= tupleA.getFieldCount() || fieldIdx >= tupleB.getFieldCount()) {
+ break;
+ }
+
+ result = cmp.getComparators()[cmpIdx].compare(tupleA.getFieldData(fieldIdx), tupleA.getFieldStart(fieldIdx),
+ tupleA.getFieldLength(fieldIdx), tupleB.getFieldData(fieldIdx), tupleB.getFieldStart(fieldIdx),
+ tupleB.getFieldLength(fieldIdx));
+ if (result != 0) {
+ return result;
+ }
+ }
+
+ return 0;
+ }
+
+ @Override
+ protected boolean isDeleted(PriorityQueueElement element) throws HyracksDataException {
+ // Check if tuple has antimatter bit set (indicates deleted record)
+ // During merge with full-scan mode, tuples may be rebuilt as ArrayTupleReference
+ // which doesn't have antimatter bit - treat those as matter tuples
+ ITupleReference tuple = element.getTuple();
+ return ((ILSMTreeTupleReference) tuple).isAntimatter();
+ }
+
+ /**
+ * Extract K value from search predicate for cluster advancement decisions.
+ */
+ private int extractK(ISearchPredicate searchPred) {
+ return ((VTreeSearchPredicate) searchPred).getK();
+ }
+
+ /**
+ * Extract minProbeFraction from search predicate (fraction of leaf clusters to probe).
+ */
+ private double extractMinProbeFraction(ISearchPredicate searchPred) {
+ return ((VTreeSearchPredicate) searchPred).getMinProbeFraction();
+ }
+
+ /**
+ * Extract epsilon value from search predicate (distance threshold for level-wise).
+ */
+ private double extractEpsilon(ISearchPredicate searchPred) {
+ return ((VTreeSearchPredicate) searchPred).getEpsilon();
+ }
+
+ /**
+ * Get the minimum number of clusters probed across all VTreeSearchCursors.
+ * This ensures synchronized nprobe checking across all LSM components.
+ */
+ private int getMinClustersProbed() {
+ int minProbed = Integer.MAX_VALUE;
+
+ for (int i = 0; i < rangeCursors.length; i++) {
+ VTreeSearchCursor vcCursor = (VTreeSearchCursor) rangeCursors[i];
+ int probed = vcCursor.getClustersProbed();
+ if (probed < minProbed) {
+ minProbed = probed;
+ }
+ }
+
+ // If no VTreeSearchCursors found, return 0
+ return minProbed == Integer.MAX_VALUE ? 0 : minProbed;
+ }
+
+ /**
+ * Extract tuple filter from search predicate for INCLUDE field predicates.
+ * When set, only tuples passing this filter are returned and counted toward K.
+ */
+ private ITupleFilter extractTupleFilter(ISearchPredicate searchPred) {
+ if (searchPred instanceof VTreeSearchPredicate) {
+ return ((VTreeSearchPredicate) searchPred).getTupleFilter();
+ }
+ return null;
+ }
+
+ /**
+ * Push next element from component cursor into queue.
+ * If cursor's current cluster is exhausted, mark it as exhausted.
+ * When ALL components' clusters are exhausted, decide whether to advance ALL to next cluster.
+ *
+ * @param e the priority queue element to refill
+ * @throws HyracksDataException if an error occurs
+ */
+ private void pushIntoQueueAndAdvanceClusterIfNeeded(PriorityQueueElement e) throws HyracksDataException {
+ int cursorIndex = e.getCursorIndex();
+ IIndexCursor cursor = rangeCursors[cursorIndex];
+
+ if (cursor.hasNext()) {
+ // Current cluster/page has more data
+ cursor.next();
+ ITupleReference tuple = cursor.getTuple();
+ e.reset(tuple);
+ outputPriorityQueue.offer(e);
+
+ return;
+ }
+
+ // Current cluster exhausted for THIS component
+ clusterExhausted[cursorIndex] = true;
+
+ // Check if ALL components have exhausted their current cluster
+ boolean allExhausted = true;
+ for (int i = 0; i < clusterExhausted.length; i++) {
+ if (!clusterExhausted[i]) {
+ allExhausted = false;
+ break;
+ }
+ }
+
+ if (!allExhausted) {
+ // Some components still have data in their current cluster
+ // Don't advance yet - wait for all to exhaust
+ return;
+ }
+
+ // ALL components exhausted their current cluster
+ // Decision: Should we advance ALL components to next cluster?
+ if (stopAdvancing) {
+ return;
+ }
+
+ // Calculate minimum clusters explored across all components
+ int minClustersExplored = getMinClustersProbed();
+ int resultsCollected = getTupleCallCount;
+
+ if (clusterStrategy.shouldStopAdvancing(minClustersExplored, resultsCollected)) {
+ // We have enough results AND probed enough clusters - stop advancing
+ stopAdvancing = true;
+ LOGGER.trace("Early termination: minClustersExplored={}, returned={}", minClustersExplored,
+ resultsCollected);
+ return;
+ }
+
+ // Not enough results or haven't probed enough clusters yet - advance ALL components
+ // Exclude empty clusters from nprobe (query mode only)
+ if (!fullScanMode && validTuplesFromCurrentCluster == 0) {
+ for (int i = 0; i < rangeCursors.length; i++) {
+ if (rangeCursors[i] instanceof VTreeSearchCursor) {
+ ((VTreeSearchCursor) rangeCursors[i]).decrementClustersProbed();
+ }
+ }
+ }
+ validTuplesFromCurrentCluster = 0;
+ advanceAllComponentsToNextCluster();
+ }
+
+ /**
+ * Advance ALL component cursors to the SAME next cluster (determined by the strategy in query
+ * mode, or by the first cursor's own iterator in full-scan/merge mode). Synchronized
+ * advancement is required so nprobe accounting is consistent across components.
+ * <p>
+ * Iterates instead of recursing to tolerate long runs of consecutive empty clusters without
+ * blowing the stack.
+ */
+ private void advanceAllComponentsToNextCluster() throws HyracksDataException {
+ while (true) {
+ // Reset exhaustion flags for new cluster
+ Arrays.fill(clusterExhausted, false);
+
+ if (fullScanMode) {
+ // Full-scan mode (merge): each cursor advances independently using its own
+ // allDirectoryPageIds. Directory page IDs are component-local and cannot be
+ // shared across components (bulk-loaded vs flushed components have different layouts).
+ boolean anyAdvanced = false;
+ for (int i = 0; i < rangeCursors.length; i++) {
+ if (!(rangeCursors[i] instanceof VTreeSearchCursor)) {
+ clusterExhausted[i] = true;
+ continue;
+ }
+ VTreeSearchCursor vcCursor = (VTreeSearchCursor) rangeCursors[i];
+ boolean advanced = vcCursor.advanceToNextCluster();
+ if (!advanced) {
+ clusterExhausted[i] = true;
+ continue;
+ }
+ anyAdvanced = true;
+ if (vcCursor.hasNext()) {
+ vcCursor.next();
+ pqes[i].reset(vcCursor.getTuple());
+ outputPriorityQueue.offer(pqes[i]);
+ } else {
+ clusterExhausted[i] = true;
+ }
+ }
+ if (!anyAdvanced) {
+ // All cursors exhausted all clusters
+ return;
+ }
+ if (!shouldSkipToNextCluster()) {
+ return;
+ }
+ continue; // All empty, try next cluster
+ }
+
+ // Query mode: strategy determines the cluster ALL components should advance to
+ ClusterSearchResult nextCluster = clusterStrategy.getNextCluster();
+
+ if (nextCluster == null) {
+ LOGGER.trace("No more clusters available globally");
+ Arrays.fill(clusterExhausted, true);
+ return;
+ }
+
+ LOGGER.trace("Global cluster selected: cid={}, distance={}, dirPage={}", nextCluster.centroidId,
+ nextCluster.distance, nextCluster.directoryPageId);
+
+ // Tell ALL components to open this SAME cluster
+ for (int i = 0; i < rangeCursors.length; i++) {
+ advanceComponentToCluster(i, nextCluster);
+ }
+
+ // Check if all components found empty cluster
+ // If so, continue loop to try next cluster (instead of recursion)
+ if (!shouldSkipToNextCluster()) {
+ return; // At least one component has data, or we should stop advancing
+ }
+ // All components empty and should continue - loop to next cluster
+ }
+ }
+
+ /**
+ * Advance a single component to a specific cluster using ClusterSearchResult.
+ * Uses O(1) directoryPageId access when available.
+ */
+ private void advanceComponentToCluster(int componentIndex, ClusterSearchResult cluster)
+ throws HyracksDataException {
+ IIndexCursor cursor = rangeCursors[componentIndex];
+
+ if (!(cursor instanceof VTreeSearchCursor)) {
+ clusterExhausted[componentIndex] = true;
+ return;
+ }
+
+ VTreeSearchCursor vcCursor = (VTreeSearchCursor) cursor;
+
+ // O(1) access via directoryPageId
+ boolean hasData = vcCursor.openClusterByResult(cluster);
+
+ if (hasData && vcCursor.hasNext()) {
+ vcCursor.next();
+ ITupleReference tuple = vcCursor.getTuple();
+ PriorityQueueElement pqe = pqes[componentIndex];
+ pqe.reset(tuple);
+ outputPriorityQueue.offer(pqe);
+ } else {
+ clusterExhausted[componentIndex] = true;
+ LOGGER.trace("advCluster: comp={} cluster={} is empty", componentIndex, cluster.centroidId);
+ }
+ }
+
+ /**
+ * Check if we should skip to next cluster (all components empty) or stop.
+ * Returns true if we should continue to next cluster, false otherwise.
+ */
+ private boolean shouldSkipToNextCluster() {
+ // Check if ALL components found the cluster empty
+ boolean allExhausted = true;
+ for (int i = 0; i < clusterExhausted.length; i++) {
+ if (!clusterExhausted[i]) {
+ allExhausted = false;
+ break;
+ }
+ }
+
+ if (!allExhausted) {
+ // At least one component has data - don't skip
+ return false;
+ }
+
+ // All components exhausted this cluster - check if we should continue
+ if (stopAdvancing) {
+ return false;
+ }
+
+ // Check if any component has more clusters available
+ boolean hasMoreClusters;
+ if (fullScanMode) {
+ // In fullScanMode, check cursors directly (strategy is not used for advancement)
+ hasMoreClusters = false;
+ for (IIndexCursor cursor : rangeCursors) {
+ if (cursor instanceof VTreeSearchCursor && ((VTreeSearchCursor) cursor).hasMoreClusters()) {
+ hasMoreClusters = true;
+ break;
+ }
+ }
+ } else {
+ hasMoreClusters = clusterStrategy.hasMoreClusters();
+ }
+ if (!hasMoreClusters) {
+ return false;
+ }
+
+ return true; // Continue to next cluster
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/LSMVTreeTopKSearchCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/LSMVTreeTopKSearchCursor.java
new file mode 100644
index 0000000000..ae1af91c32
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/LSMVTreeTopKSearchCursor.java
@@ -0,0 +1,810 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.vector.impls;
+
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.List;
+import java.util.PriorityQueue;
+import java.util.Set;
+
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.util.CleanupUtils;
+import org.apache.hyracks.api.util.HyracksConstants;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.dataflow.common.utils.TupleUtils;
+import org.apache.hyracks.storage.am.common.api.ITupleFilter;
+import org.apache.hyracks.storage.am.common.tuples.ReferenceFrameTupleReference;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMTreeTupleReference;
+import org.apache.hyracks.storage.am.vector.api.IVTreeBinaryAccessor;
+import org.apache.hyracks.storage.am.vector.api.IVTreeBinaryAccessorFactory;
+import org.apache.hyracks.storage.am.vector.api.IVTreeDistanceFunction;
+import org.apache.hyracks.storage.am.vector.api.IVTreeQuantizer;
+import org.apache.hyracks.storage.am.vector.impls.ClusterSearchResult;
+import org.apache.hyracks.storage.am.vector.impls.VTree;
+import org.apache.hyracks.storage.am.vector.impls.VTree.VTreeAccessor;
+import org.apache.hyracks.storage.am.vector.impls.VTreeSearchCursor;
+import org.apache.hyracks.storage.am.vector.impls.VTreeSearchPredicate;
+import org.apache.hyracks.storage.am.vector.utils.VTreeDataTupleAccessor;
+import org.apache.hyracks.storage.common.EnforcedIndexCursor;
+import org.apache.hyracks.storage.common.ICursorInitialState;
+import org.apache.hyracks.storage.common.IIndexAccessParameters;
+import org.apache.hyracks.storage.common.IIndexCursor;
+import org.apache.hyracks.storage.common.ISearchPredicate;
+import org.apache.hyracks.storage.common.MultiComparator;
+import org.apache.hyracks.storage.common.util.IndexCursorUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+/**
+ * Blocked top-K cursor for vector ANN search.
+ * <p>
+ * Lifecycle: all per-component cursor work — priority-queue merge across LSM components, antimatter
+ * reconciliation, INCLUDE-field filtering, approximate distance computation against the quantized
+ * embedding — happens inside {@link #open}. Surviving candidates are collected into a
+ * {@link SpillableTopKBuffer} (in-memory MaxHeap keyed by approximate distance, with optional
+ * disk spill on memory pressure). {@link #hasNext()}/{@link #next()}/{@link #getTuple()} then
+ * drain that buffer in approximate-distance ascending order.
+ * <p>
+ * This cursor is for quantized indexes: the tuple format is
+ * {@code [distance, centroidId, quantized_distance, quantized_embedding, PKs..., includes...]}
+ * (pkStartField=4). Reranking against the unquantized vector is the caller's responsibility.
+ */
+public class LSMVTreeTopKSearchCursor extends EnforcedIndexCursor implements IVectorSearchCursor {
+
+ /**
+ * Index-access-parameters key — set {@code Boolean.TRUE} under this key to route search to
+ * this cursor (the quantized top-K window cursor used by production ANN queries). When absent
+ * or false, {@link LSMVTreeIndexAccessor#createSearchCursor(boolean)} returns the streaming
+ * {@link LSMVTreeSearchCursor} — also the cursor used by component merges and by test
+ * fixtures that verify inserts/deletes through full-scan iteration.
+ */
+ public static final String IAP_KEY = "USE_TOPK_SEARCH";
+
+ private static final Logger LOGGER = LogManager.getLogger();
+
+ // Operation context
+ private ILSMIndexOperationContext opCtx;
+ private List<ILSMComponent> operationalComponents;
+
+ // Per-component accessors and cursors (same as LSMVTreeSearchCursor)
+ private VTreeAccessor[] vTreeAccessors;
+ private IIndexCursor[] rangeCursors;
+ private int numComponents;
+
+ // Priority queue for merging results from multiple components
+ private PriorityQueue<PriorityQueueElement> outputPriorityQueue;
+ private PriorityQueueElement[] pqes;
+ private MultiComparator cmp;
+
+ // Antimatter reconciliation: stable COPIES of matters that survived reconciliation of the current
+ // equal-distance group, buffered for emission one at a time. See getNextValidTuple.
+ private final ArrayDeque<ITupleReference> readyMatters = new ArrayDeque<>();
+ // Number of primary-key fields (INCLUDE fields excluded) — the reconciliation key.
+ private int numPrimaryKeyFields;
+
+ // Spillable top-K buffer: frame-backed in-memory heap with disk spill on budget exceeded.
+ // Encapsulates MaxHeap + VariableDeletableTupleMemoryManager + RunFileWriter spill.
+ private SpillableTopKBuffer topKBuffer;
+ private SpillableTopKDrainIterator drainIterator;
+
+ // Search parameters
+ private int K;
+ /** Number of candidates to collect for reranking: {@code K * max(1, kMultiplier)} from the predicate. */
+ private int candidateLimit;
+ private double epsilon;
+ private double[] queryVector;
+ private IVTreeDistanceFunction distanceFunction;
+
+ // Vector accessor for extracting vectors from tuples
+ private IVTreeBinaryAccessor vectorAccessor;
+
+ // Quantization state (propagated from first search cursor)
+ private double[] quantizedQueryVector;
+ private IVTreeQuantizer quantizer;
+
+ // Data-tuple layout authority. This cursor is quantized-only (see class javadoc), so the layout
+ // is the quantized one.
+ private final VTreeDataTupleAccessor dataAccessor = new VTreeDataTupleAccessor(true);
+
+ // Cluster selection strategy (nprobe + DFS fallback)
+ private IClusterSelectionStrategy clusterStrategy;
+
+ // First component's search cursor (for query vector/distance function extraction and DFS)
+ private VTreeSearchCursor firstSearchCursor;
+
+ // Cluster tracking (synchronized advancement like LSMVTreeSearchCursor)
+ private boolean[] clusterExhausted;
+ private boolean stopAdvancing;
+ private int clustersExplored;
+
+ // Tuple filter for INCLUDE field predicates (e.g., year > 2000)
+ private ITupleFilter tupleFilter;
+ private ReferenceFrameTupleReference referenceFilterTuple;
+
+ // Field index where primary keys start in the data tuple
+ private int pkStartField;
+
+ // Statistics
+ private int totalTuplesProcessed;
+ private int nextCallCount;
+ private int antimatterCancellations;
+ private int tuplesFilteredOut;
+ private int validTuplesFromCurrentCluster; // Valid tuples from current cluster (for empty-cluster nprobe)
+
+ public LSMVTreeTopKSearchCursor(ILSMIndexOperationContext opCtx) {
+ this.opCtx = opCtx;
+ }
+
+ @Override
+ protected void doOpen(ICursorInitialState initialState, ISearchPredicate searchPred) throws HyracksDataException {
+ this.totalTuplesProcessed = 0;
+ this.nextCallCount = 0;
+ this.antimatterCancellations = 0;
+ this.tuplesFilteredOut = 0;
+ this.validTuplesFromCurrentCluster = 0;
+ this.readyMatters.clear();
+
+ // Get initial state
+ LSMVTreeCursorInitialState lsmInitialState = (LSMVTreeCursorInitialState) initialState;
+ this.cmp = lsmInitialState.getOriginalKeyComparator();
+ this.operationalComponents = lsmInitialState.getOperationalComponents();
+ this.numComponents = operationalComponents.size();
+
+ // Extract search parameters from predicate
+ VTreeSearchPredicate vectorPred = (VTreeSearchPredicate) searchPred;
+ this.K = vectorPred.getK();
+ int mult = vectorPred.getKMultiplier();
+ this.candidateLimit = K * Math.max(1, mult); // Send K*kMultiplier to PK for reranking
+ this.epsilon = vectorPred.getEpsilon();
+ this.pkStartField = dataAccessor.pkStartField();
+ this.numPrimaryKeyFields = ((LSMVTree) opCtx.getIndex()).getNumPrimaryKeyFields();
+
+ // Extract tuple filter from search predicate for INCLUDE field predicates
+ this.tupleFilter = vectorPred.getTupleFilter();
+ if (this.tupleFilter != null) {
+ this.referenceFilterTuple = new ReferenceFrameTupleReference();
+ }
+
+ // Get index access parameters
+ IIndexAccessParameters iap = ((LSMVTreeOpContext) opCtx).getIndexAccessParameters();
+
+ // Initialize vector accessor from factory in parameters
+ IVTreeBinaryAccessorFactory vectorAccessorFactory =
+ (IVTreeBinaryAccessorFactory) iap.getParameters().get(IVTreeBinaryAccessorFactory.IAP_KEY);
+ if (vectorAccessorFactory != null) {
+ this.vectorAccessor = vectorAccessorFactory.createAccessor();
+ }
+
+ // Create cluster selection strategy (minProbeFraction → nprobe + DFS fallback)
+ this.clusterStrategy = new NprobeClusterSelectionStrategy(vectorPred.getMinProbeFraction(), epsilon);
+
+ // Create spillable top-K buffer (follows inverted index pattern: pass ctx via IAP)
+ IHyracksTaskContext ctx = (IHyracksTaskContext) iap.getParameters().get(HyracksConstants.HYRACKS_TASK_CONTEXT);
+ this.topKBuffer = new SpillableTopKBuffer(candidateLimit, ctx);
+
+ // Initialize cluster tracking arrays
+ clusterExhausted = new boolean[numComponents];
+ Arrays.fill(clusterExhausted, false);
+ stopAdvancing = false;
+ clustersExplored = 0;
+
+ // Create accessors and cursors for each component (same as LSMVTreeSearchCursor)
+ vTreeAccessors = new VTreeAccessor[numComponents];
+ rangeCursors = new IIndexCursor[numComponents];
+
+ try {
+ openComponentsAndSearch(searchPred, iap);
+ } catch (Throwable th) { // NOSONAR must catch all: release per-component cursors/accessors + the
+ // spill buffer here, since a failed open() leaves the caller no handle to close.
+ Throwable failure = CleanupUtils.destroy(th, vTreeAccessors);
+ failure = CleanupUtils.destroy(failure, rangeCursors);
+ if (topKBuffer != null) {
+ try {
+ topKBuffer.close();
+ } catch (Throwable t) { // NOSONAR
+ failure.addSuppressed(t);
+ }
+ topKBuffer = null;
+ }
+ throw HyracksDataException.create(failure);
+ }
+ }
+
+ /**
+ * Body of {@link #open}: create and open a search cursor per component, initialize the cluster
+ * selection strategy, and run the blocked search. Extracted so {@code open()} can wrap it in one
+ * try/catch that releases all of open()'s resources on any failure.
+ */
+ private void openComponentsAndSearch(ISearchPredicate searchPred, IIndexAccessParameters iap)
+ throws HyracksDataException {
+ for (int i = 0; i < numComponents; i++) {
+ ILSMComponent component = operationalComponents.get(i);
+ VTree vTree = (VTree) component.getIndex();
+ vTreeAccessors[i] = (VTreeAccessor) vTree.createAccessor(iap);
+ rangeCursors[i] = vTreeAccessors[i].createSearchCursor(false);
+ }
+
+ // Open all cursors with the search predicate
+ IndexCursorUtils.open(vTreeAccessors, rangeCursors, searchPred);
+
+ // Initialize strategy and set up DFS fallback (same as LSMVTreeSearchCursor)
+ if (numComponents > 0) {
+ this.firstSearchCursor = (VTreeSearchCursor) rangeCursors[0];
+ this.queryVector = firstSearchCursor.getQueryVector();
+ this.distanceFunction = firstSearchCursor.getDistanceFunction();
+
+ // Extract quantized state from first cursor (null = non-quantized path)
+ this.quantizedQueryVector = firstSearchCursor.getQuantizedQueryVector();
+ this.quantizer = firstSearchCursor.getQuantizer();
+
+ if (this.queryVector == null) {
+ throw HyracksDataException
+ .create(new IllegalArgumentException("Query vector must be provided for naive blocked search"));
+ }
+
+ // Initialize strategy with first component's tree (candidateLimit so we collect 2*K for reranking)
+ ILSMComponent firstComponent = operationalComponents.get(0);
+ VTree vTree = (VTree) firstComponent.getIndex();
+ clusterStrategy.initialize(vTree, queryVector, distanceFunction, candidateLimit);
+
+ // Set first cursor for DFS fallback
+ clusterStrategy.setFirstCursorForDFS(firstSearchCursor);
+
+ // Pass shared visited set from strategy to all cursors
+ Set<Integer> visitedSet = clusterStrategy.getVisitedCentroidIds();
+ for (int i = 0; i < numComponents; i++) {
+ if (rangeCursors[i] instanceof VTreeSearchCursor) {
+ ((VTreeSearchCursor) rangeCursors[i]).setSharedVisitedSet(visitedSet);
+ }
+ }
+
+ // Re-open all cursors to the first level-wise cluster for consistency
+ ClusterSearchResult firstCluster = clusterStrategy.getFirstCluster();
+ if (firstCluster != null) {
+ ClusterSearchResult dfsCluster = firstSearchCursor.getCurrentClusterResult();
+ if (dfsCluster != null && dfsCluster.centroidId != firstCluster.centroidId) {
+ LOGGER.trace("DFS found cid={} but level-wise[0] is cid={} - re-opening", dfsCluster.centroidId,
+ firstCluster.centroidId);
+ // The greedy-DFS seed (dfsCluster) was consumed from every component cursor's DFS
+ // iterator and marked visited by initializeClusterIterator. We are displacing it here
+ // for the globally-closest level-wise[0]. Defer it to the strategy so it is probed once
+ // after the level-wise phase — otherwise its cluster is never scanned and records routed
+ // there (cross_pollination_m=1: no replica) silently disappear from results. All
+ // components share one static structure, so every cursor's greedy seed is this centroid.
+ clusterStrategy.deferSeedCluster(dfsCluster);
+ for (int i = 0; i < numComponents; i++) {
+ if (rangeCursors[i] instanceof VTreeSearchCursor) {
+ VTreeSearchCursor vcCursor = (VTreeSearchCursor) rangeCursors[i];
+ vcCursor.resetClustersProbed();
+ vcCursor.openClusterByResult(firstCluster);
+ }
+ }
+ }
+ LOGGER.trace("Initialized with K={}, epsilon={}, level-wise clusters={}", K, epsilon,
+ clusterStrategy.getLevelWiseClusterCount());
+ }
+ }
+
+ // Initialize priority queue for merging results from all components
+ initPriorityQueue();
+
+ // Perform the blocked search: drain all clusters and collect results
+ performBlockedSearch();
+
+ // Prepare drain: sort entries by dqx ascending for output (with merge if spilled)
+ this.drainIterator = topKBuffer.drain();
+
+ LOGGER.trace("Search complete: topK={}, processed={}, filtered={}, cancellations={}, clusters={}",
+ topKBuffer.getNumEntries(), totalTuplesProcessed, tuplesFilteredOut, antimatterCancellations,
+ clustersExplored);
+ }
+
+ /**
+ * Initialize priority queue and populate with first element from each cursor.
+ */
+ private void initPriorityQueue() throws HyracksDataException {
+ int pqInitSize = Math.max(numComponents, 1);
+ outputPriorityQueue = new PriorityQueue<>(pqInitSize, new NaivePriorityQueueComparator());
+ pqes = new PriorityQueueElement[pqInitSize];
+ for (int i = 0; i < pqInitSize; i++) {
+ pqes[i] = new PriorityQueueElement(i);
+ }
+
+ // Populate priority queue with first element from each cursor
+ for (int i = 0; i < numComponents; i++) {
+ if (rangeCursors[i].hasNext()) {
+ rangeCursors[i].next();
+ pqes[i].reset(rangeCursors[i].getTuple());
+ outputPriorityQueue.offer(pqes[i]);
+ } else {
+ clusterExhausted[i] = true;
+ }
+ }
+
+ clustersExplored = 1; // First cluster opened
+
+ // If all components started empty, advance to next cluster
+ if (allComponentsExhausted()) {
+ advanceAllComponentsToNextCluster();
+ }
+ }
+
+ /**
+ * Perform the blocked search: drain the priority queue, apply antimatter reconciliation
+ * and filtering, compute distances, and collect results into topKWindow.
+ *
+ * This continues until we've probed enough clusters and have enough results,
+ * or all clusters are exhausted.
+ */
+ private void performBlockedSearch() throws HyracksDataException {
+ while (true) {
+ // Process current cluster's data via priority queue (getNextValidTuple returns null once the
+ // queue AND the reconciled-matter buffer are both drained).
+ ITupleReference validTuple;
+ while ((validTuple = getNextValidTuple()) != null) {
+ // Apply INCLUDE field filter
+ if (passesTupleFilter(validTuple)) {
+ validTuplesFromCurrentCluster++;
+ // Compute approximate distance using quantized embedding
+ double dqx = computeApproximateDistance(validTuple);
+ topKBuffer.insert(validTuple, dqx);
+ }
+ totalTuplesProcessed++;
+ }
+
+ // Current cluster(s) exhausted - check if we should advance
+ if (stopAdvancing) {
+ break;
+ }
+
+ // Check strategy for stop condition
+ int minClustersExplored = getMinClustersProbed();
+ if (clusterStrategy.shouldStopAdvancing(minClustersExplored, topKBuffer.getNumEntries())) {
+ stopAdvancing = true;
+ LOGGER.trace("Early termination: clusters={}, topK={}", minClustersExplored,
+ topKBuffer.getNumEntries());
+ break;
+ }
+
+ // Try to advance to next cluster
+ if (!clusterStrategy.hasMoreClusters()) {
+ LOGGER.trace("No more clusters available");
+ break;
+ }
+
+ // Exclude empty clusters from nprobe
+ if (validTuplesFromCurrentCluster == 0) {
+ for (int i = 0; i < rangeCursors.length; i++) {
+ if (rangeCursors[i] instanceof VTreeSearchCursor) {
+ ((VTreeSearchCursor) rangeCursors[i]).decrementClustersProbed();
+ }
+ }
+ }
+ validTuplesFromCurrentCluster = 0;
+ advanceAllComponentsToNextCluster();
+ }
+ }
+
+ /**
+ * Get next valid tuple with antimatter reconciliation.
+ * Follows the checkPriorityQueue() pattern from LSMVTreeSearchCursor.
+ *
+ * @return next valid matter tuple, or null if queue exhausted
+ */
+ private ITupleReference getNextValidTuple() throws HyracksDataException {
+ while (true) {
+ if (!readyMatters.isEmpty()) {
+ return readyMatters.poll();
+ }
+ if (outputPriorityQueue.isEmpty()) {
+ return null;
+ }
+
+ // Drain the entire run of elements that share field-0 (distance-to-centroid) across ALL
+ // component streams, then reconcile by primary key. This is required because the merge order
+ // is by distance, NOT by PK: a per-component cursor iterates same-distance tuples in insertion
+ // order, so a delete marker and its live twin can be separated by an unrelated same-distance
+ // record. Reconciling the whole group by PK — instead of only cancelling against the adjacent
+ // queue element — cancels the twin wherever it lands in the run, so a deleted record cannot
+ // resurface (see LSMVTreeAntimatterCollisionQuantizedTest).
+ //
+ // Each element's tuple points into its component cursor's frame and is invalidated when that
+ // cursor advances, so capture a stable COPY before pushIntoQueueAndAdvanceClusterIfNeeded.
+ PriorityQueueElement first = outputPriorityQueue.poll();
+ ITupleReference groupKey = TupleUtils.copyTuple(first.tuple);
+ List<ReconcileEntry> group = new ArrayList<>();
+ group.add(new ReconcileEntry(groupKey, first.componentId, isAntimatter(first.tuple)));
+ pushIntoQueueAndAdvanceClusterIfNeeded(first);
+ while (!outputPriorityQueue.isEmpty() && compareField0(outputPriorityQueue.peek().tuple, groupKey) == 0) {
+ PriorityQueueElement e = outputPriorityQueue.poll();
+ group.add(new ReconcileEntry(TupleUtils.copyTuple(e.tuple), e.componentId, isAntimatter(e.tuple)));
+ pushIntoQueueAndAdvanceClusterIfNeeded(e);
+ }
+ reconcileGroupByPrimaryKey(group);
+ }
+ }
+
+ /**
+ * Reconcile one equal-distance group by primary key using LSM newest-wins semantics: for each PK, the
+ * entry from the newest component (lowest componentId) determines presence — if it is matter it is
+ * emitted once, if it is a delete marker the record is suppressed and no older matter for that PK
+ * survives. Surviving matters are appended to {@link #readyMatters}.
+ */
+ private void reconcileGroupByPrimaryKey(List<ReconcileEntry> group) throws HyracksDataException {
+ for (ReconcileEntry e : group) {
+ boolean newestForPk = true;
+ for (ReconcileEntry other : group) {
+ if (other != e && other.componentId < e.componentId && samePrimaryKey(e.tuple, other.tuple)) {
+ newestForPk = false;
+ break;
+ }
+ }
+ if (!newestForPk) {
+ continue; // shadowed by a newer tuple for the same PK
+ }
+ if (e.antimatter) {
+ antimatterCancellations++; // newest is a delete marker → record is absent
+ } else {
+ readyMatters.add(e.tuple);
+ }
+ }
+ }
+
+ /** A stable copy of one queue element, captured before its cursor advances (see getNextValidTuple). */
+ private static final class ReconcileEntry {
+ private final ITupleReference tuple;
+ private final int componentId;
+ private final boolean antimatter;
+
+ private ReconcileEntry(ITupleReference tuple, int componentId, boolean antimatter) {
+ this.tuple = tuple;
+ this.componentId = componentId;
+ this.antimatter = antimatter;
+ }
+ }
+
+ /** Compare field 0 (distance-to-centroid), the priority-queue ordering key. */
+ private int compareField0(ITupleReference a, ITupleReference b) throws HyracksDataException {
+ return cmp.getComparators()[0].compare(a.getFieldData(0), a.getFieldStart(0), a.getFieldLength(0),
+ b.getFieldData(0), b.getFieldStart(0), b.getFieldLength(0));
+ }
+
+ /**
+ * True iff two tuples carry the same primary key. Only the {@code numPrimaryKeyFields} PK fields are
+ * compared — trailing INCLUDE fields are excluded, since a delete marker and its live twin may differ
+ * in INCLUDE values and must still reconcile.
+ */
+ private boolean samePrimaryKey(ITupleReference a, ITupleReference b) throws HyracksDataException {
+ int numPkFields = Math.min(cmp.getComparators().length - pkStartField, numPrimaryKeyFields);
+ for (int i = 0; i < numPkFields; i++) {
+ int fieldIdx = pkStartField + i;
+ if (fieldIdx >= a.getFieldCount() || fieldIdx >= b.getFieldCount()) {
+ return false;
+ }
+ if (cmp.getComparators()[fieldIdx].compare(a.getFieldData(fieldIdx), a.getFieldStart(fieldIdx),
+ a.getFieldLength(fieldIdx), b.getFieldData(fieldIdx), b.getFieldStart(fieldIdx),
+ b.getFieldLength(fieldIdx)) != 0) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Push next element from component cursor into queue.
+ * If cursor's current cluster is exhausted, mark it as exhausted.
+ * When ALL components' clusters are exhausted, the loop in performBlockedSearch handles advancement.
+ */
+ private void pushIntoQueueAndAdvanceClusterIfNeeded(PriorityQueueElement e) throws HyracksDataException {
+ int cursorIndex = e.componentId;
+ IIndexCursor cursor = rangeCursors[cursorIndex];
+
+ if (cursor.hasNext()) {
+ cursor.next();
+ e.reset(cursor.getTuple());
+ outputPriorityQueue.offer(e);
+ return;
+ }
+
+ // Current cluster exhausted for this component
+ clusterExhausted[cursorIndex] = true;
+ }
+
+ /**
+ * Advance ALL component cursors to the next cluster.
+ * Uses iterative loop to handle consecutive empty clusters.
+ */
+ private void advanceAllComponentsToNextCluster() throws HyracksDataException {
+ while (true) {
+ Arrays.fill(clusterExhausted, false);
+
+ ClusterSearchResult nextCluster = clusterStrategy.getNextCluster();
+ if (nextCluster == null) {
+ LOGGER.trace("No more clusters available globally");
+ Arrays.fill(clusterExhausted, true);
+ stopAdvancing = true;
+ return;
+ }
+
+ LOGGER.trace("Advancing to cluster cid={}, distance={}, dirPage={}", nextCluster.centroidId,
+ nextCluster.distance, nextCluster.directoryPageId);
+
+ // Open all components to this cluster
+ for (int i = 0; i < numComponents; i++) {
+ advanceComponentToCluster(i, nextCluster);
+ }
+ clustersExplored++;
+
+ // Check if all components found empty cluster - try next
+ if (!allComponentsExhausted()) {
+ return; // At least one component has data
+ }
+
+ // All empty - check if should continue
+ if (!clusterStrategy.hasMoreClusters()) {
+ stopAdvancing = true;
+ return;
+ }
+ // Loop to try next cluster
+ }
+ }
+
+ /**
+ * Advance a single component to a specific cluster.
+ */
+ private void advanceComponentToCluster(int componentIndex, ClusterSearchResult cluster)
+ throws HyracksDataException {
+ IIndexCursor cursor = rangeCursors[componentIndex];
+
+ if (!(cursor instanceof VTreeSearchCursor)) {
+ clusterExhausted[componentIndex] = true;
+ return;
+ }
+
+ VTreeSearchCursor vcCursor = (VTreeSearchCursor) cursor;
+ boolean hasData = vcCursor.openClusterByResult(cluster);
+
+ if (hasData && vcCursor.hasNext()) {
+ vcCursor.next();
+ pqes[componentIndex].reset(vcCursor.getTuple());
+ outputPriorityQueue.offer(pqes[componentIndex]);
+ } else {
+ clusterExhausted[componentIndex] = true;
+ }
+ }
+
+ /**
+ * Check if all components have exhausted their current cluster.
+ */
+ private boolean allComponentsExhausted() {
+ for (boolean exhausted : clusterExhausted) {
+ if (!exhausted) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Get the minimum number of clusters probed across all VTreeSearchCursors.
+ */
+ private int getMinClustersProbed() {
+ int minProbed = Integer.MAX_VALUE;
+ for (int i = 0; i < rangeCursors.length; i++) {
+ if (rangeCursors[i] instanceof VTreeSearchCursor) {
+ int probed = ((VTreeSearchCursor) rangeCursors[i]).getClustersProbed();
+ if (probed < minProbed) {
+ minProbed = probed;
+ }
+ }
+ }
+ return minProbed == Integer.MAX_VALUE ? 0 : minProbed;
+ }
+
+ /**
+ * Check if tuple is antimatter.
+ */
+ private boolean isAntimatter(ITupleReference tuple) {
+ if (tuple instanceof ILSMTreeTupleReference) {
+ return ((ILSMTreeTupleReference) tuple).isAntimatter();
+ }
+ return false;
+ }
+
+ /**
+ * Check if tuple passes the INCLUDE field filter.
+ * Applied AFTER antimatter reconciliation. Tuples that fail filter don't enter topKWindow.
+ */
+ private boolean passesTupleFilter(ITupleReference tuple) throws HyracksDataException {
+ if (tupleFilter == null) {
+ return true;
+ }
+ referenceFilterTuple.reset(tuple);
+ if (tupleFilter.accept(referenceFilterTuple)) {
+ return true;
+ }
+ tuplesFilteredOut++;
+ return false;
+ }
+
+ /**
+ * Compute approximate distance D(q, x) using quantized embedding.
+ *
+ * This cursor is dedicated for quantized vector indexes.
+ * Quantized data tuple format (pkStartField=4):
+ * Field 0: distance_to_centroid, Field 1: centroidId,
+ * Field 2: quantized_distance, Field 3: quantized_embedding, Field 4+: PKs
+ *
+ * Dequantizes the stored embedding bytes (field 3) and computes distance
+ * against the quantized query vector.
+ */
+ private double computeApproximateDistance(ITupleReference tuple) throws HyracksDataException {
+ // Quantized embedding content bytes (field 3, ByteArrayPointable prefix stripped) → dequantize.
+ byte[] qBytes = dataAccessor.getQuantizedEmbedding(tuple);
+ double[] dequantized = quantizer.dequantize(qBytes);
+ return distanceFunction.apply(quantizedQueryVector, dequantized);
+ }
+
+ // ==================== IIndexCursor Interface (EnforcedIndexCursor template methods) ====================
+
+ @Override
+ protected boolean doHasNext() throws HyracksDataException {
+ return drainIterator != null && drainIterator.hasNext();
+ }
+
+ @Override
+ protected void doNext() throws HyracksDataException {
+ if (!doHasNext()) {
+ throw HyracksDataException.create(new IllegalStateException("No more tuples"));
+ }
+ drainIterator.next();
+ nextCallCount++;
+ }
+
+ @Override
+ protected ITupleReference doGetTuple() {
+ if (drainIterator == null) {
+ return null;
+ }
+ return drainIterator.getTuple();
+ }
+
+ /**
+ * {@inheritDoc}
+ *
+ * <p>Delegates to {@link SpillableTopKDrainIterator#getCurrentDqx()}, which returns the
+ * {@code D(q,x)} computed from the quantized embedding during the blocked-search phase.
+ * Valid between {@link #next()} and the next {@link #next()} / {@link #close()}; returns
+ * {@link Double#NaN} otherwise.
+ */
+ @Override
+ public double getCurrentDistance() {
+ return drainIterator != null ? drainIterator.getCurrentDqx() : Double.NaN;
+ }
+
+ @Override
+ protected void doClose() throws HyracksDataException {
+ // EnforcedIndexCursor only routes here on an OPENED -> CLOSED transition, so no isOpen guard is
+ // needed. doClose() ends the current search (the cursor can be re-opened); the null-guards below
+ // still cover a failure early in doOpen() that left these unallocated.
+ if (LOGGER.isTraceEnabled()) {
+ LOGGER.trace(
+ "Search summary: K={}, epsilon={}, clustersExplored={}, tuplesProcessed={},"
+ + " antimatterCancellations={}, filteredOut={}, resultsReturned={}",
+ K, epsilon, clustersExplored, totalTuplesProcessed, antimatterCancellations, tuplesFilteredOut,
+ nextCallCount);
+ }
+
+ if (rangeCursors != null) {
+ for (int i = 0; i < rangeCursors.length; i++) {
+ if (rangeCursors[i] != null) {
+ rangeCursors[i].close();
+ }
+ }
+ }
+ if (drainIterator != null) {
+ drainIterator.close();
+ drainIterator = null;
+ }
+ if (topKBuffer != null) {
+ topKBuffer.close();
+ topKBuffer = null;
+ }
+ }
+
+ @Override
+ protected void doDestroy() throws HyracksDataException {
+ // The enforced contract guarantees doClose() already ran (destroy requires the CLOSED state), which
+ // ended the current search. doDestroy() reclaims the per-component accessors + cursors for good,
+ // matching LSMIndexSearchCursor.doDestroy.
+ Throwable failure = CleanupUtils.destroy(null, vTreeAccessors);
+ failure = CleanupUtils.destroy(failure, rangeCursors);
+ vTreeAccessors = null;
+ rangeCursors = null;
+ if (failure != null) {
+ throw HyracksDataException.create(failure);
+ }
+ }
+
+ // ==================== Inner Classes ====================
+
+ /**
+ * Priority queue element holding tuple and component info (for multi-component merging).
+ */
+ private static class PriorityQueueElement {
+ int componentId;
+ ITupleReference tuple;
+
+ PriorityQueueElement(int componentId) {
+ this.componentId = componentId;
+ }
+
+ void reset(ITupleReference tuple) {
+ this.tuple = tuple;
+ }
+ }
+
+ /**
+ * Priority queue comparator for merging results from multiple components.
+ * Compares by distance (field 0), then PK fields, then component ID.
+ */
+ private class NaivePriorityQueueComparator implements Comparator<PriorityQueueElement> {
+ @Override
+ public int compare(PriorityQueueElement a, PriorityQueueElement b) {
+ ITupleReference tupleA = a.tuple;
+ ITupleReference tupleB = b.tuple;
+
+ try {
+ int result = cmp.getComparators()[0].compare(tupleA.getFieldData(0), tupleA.getFieldStart(0),
+ tupleA.getFieldLength(0), tupleB.getFieldData(0), tupleB.getFieldStart(0),
+ tupleB.getFieldLength(0));
+ if (result != 0) {
+ return result;
+ }
+
+ // Cap at the PK fields; trailing INCLUDE fields are excluded from the ordering key so a
+ // delete marker and its live twin (which may differ in INCLUDE values) order together.
+ int numRemainingFields = Math.min(cmp.getComparators().length - pkStartField, numPrimaryKeyFields);
+ for (int i = 0; i < numRemainingFields; i++) {
+ int fieldIdx = pkStartField + i;
+ if (fieldIdx >= tupleA.getFieldCount() || fieldIdx >= tupleB.getFieldCount()) {
+ break;
+ }
+ result = cmp.getComparators()[pkStartField + i].compare(tupleA.getFieldData(fieldIdx),
+ tupleA.getFieldStart(fieldIdx), tupleA.getFieldLength(fieldIdx),
+ tupleB.getFieldData(fieldIdx), tupleB.getFieldStart(fieldIdx),
+ tupleB.getFieldLength(fieldIdx));
+ if (result != 0) {
+ return result;
+ }
+ }
+ } catch (Throwable e) {
+ throw new IllegalArgumentException(e);
+ }
+
+ return Integer.compare(a.componentId, b.componentId);
+ }
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/NprobeClusterSelectionStrategy.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/NprobeClusterSelectionStrategy.java
new file mode 100644
index 0000000000..db252f90f9
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/NprobeClusterSelectionStrategy.java
@@ -0,0 +1,297 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.lsm.vector.impls;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.vector.api.IVTreeDistanceFunction;
+import org.apache.hyracks.storage.am.vector.api.IVTreeQuantizer;
+import org.apache.hyracks.storage.am.vector.impls.ClusterSearchResult;
+import org.apache.hyracks.storage.am.vector.impls.VTree;
+import org.apache.hyracks.storage.am.vector.impls.VTreeSearchCursor;
+import org.apache.hyracks.storage.am.vector.utils.VTreeNavigationUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+/**
+ * Nprobe-based cluster selection strategy.
+ *
+ * Explores clusters in two phases:
+ * 1. Level-wise: Computes clusters within epsilon distance threshold, sorted globally by distance
+ * 2. DFS fallback: When level-wise exhausted, uses DFS backtracking from first component
+ *
+ * Stopping condition: minClustersExplored >= nprobe AND resultsCollected >= K
+ */
+public class NprobeClusterSelectionStrategy implements IClusterSelectionStrategy {
+ private static final Logger LOGGER = LogManager.getLogger();
+
+ // Parameters
+ private final double minProbeFraction;
+ private final double epsilon;
+ private int nprobe; // Computed from minProbeFraction * totalLeafClusters at initialize time
+ private int K;
+
+ // Level-wise state
+ private List<ClusterSearchResult> globalLevelWiseClusters;
+ private int globalClusterIndex;
+ private boolean levelWisePhaseComplete;
+
+ // Shared state for cross-component deduplication
+ private Set<Integer> visitedCentroidIds;
+
+ // Quantizer state for computing quantized D(q,C) in ClusterSearchResult
+ private double[] quantizedQueryVector;
+ private IVTreeQuantizer quantizer;
+
+ // For DFS fallback
+ private VTreeSearchCursor firstCursor;
+
+ // Seed cluster displaced by the level-wise re-open; probed once after the level-wise phase (see
+ // deferSeedCluster).
+ private ClusterSearchResult deferredSeedCluster;
+
+ public NprobeClusterSelectionStrategy(double minProbeFraction, double epsilon) {
+ this.minProbeFraction = minProbeFraction;
+ this.epsilon = epsilon;
+ this.nprobe = 1; // Will be computed in initialize() from minProbeFraction * totalLeafClusters
+ this.visitedCentroidIds = new HashSet<>();
+ }
+
+ @Override
+ public void setQuantizer(double[] quantizedQueryVector, IVTreeQuantizer quantizer) {
+ this.quantizedQueryVector = quantizedQueryVector;
+ this.quantizer = quantizer;
+ }
+
+ @Override
+ public void initialize(VTree vTree, double[] queryVector, IVTreeDistanceFunction distFunc, int k)
+ throws HyracksDataException {
+ this.K = k;
+ this.globalClusterIndex = 0;
+ this.levelWisePhaseComplete = false;
+
+ // Compute level-wise clusters using VTreeNavigationUtils
+ if (queryVector != null && epsilon > 0.0 && vTree != null) {
+ try {
+ globalLevelWiseClusters = VTreeNavigationUtils.findCloseCentroidsLevelWiseGlobalSort(
+ vTree.getNavigationBufferCache(), vTree.getNavigationFileId(), vTree.getNavigationRootPageId(),
+ vTree.getInteriorFrameFactory(), vTree.getLeafFrameFactory(), queryVector, distFunc, epsilon,
+ quantizedQueryVector, quantizer);
+
+ // Compute nprobe from minProbeFraction * totalLeafClusters
+ int totalLeafClusters = globalLevelWiseClusters != null ? globalLevelWiseClusters.size() : 1;
+ // Always probe at least one cluster: for small indexes floor(total*fraction) can round to
+ // 0, which would make the search entirely K-gated. The max(1,...) clamp is intentional and
+ // makes nprobe non-monotonic vs the configured fraction for tiny datasets.
+ this.nprobe = Math.max(1, (int) Math.floor(totalLeafClusters * minProbeFraction));
+
+ // Mark first cluster as visited and start getNextCluster() from index 1
+ // The cursor handles index 0 separately via getFirstCluster()
+ if (globalLevelWiseClusters != null && !globalLevelWiseClusters.isEmpty()) {
+ visitedCentroidIds.add(globalLevelWiseClusters.get(0).centroidId);
+ globalClusterIndex = 1; // Skip first cluster in getNextCluster()
+ }
+
+ LOGGER.trace("Computed {} level-wise clusters with epsilon={}, minProbeFraction={}, nprobe={}",
+ totalLeafClusters, epsilon, minProbeFraction, nprobe);
+ } catch (HyracksDataException e) {
+ // Level-wise probing is an optimization over the DFS fallback; on a storage error we can
+ // still answer via DFS, but that reduces recall, so surface it at WARN with the cause
+ // rather than hiding a degraded search at TRACE. Only HyracksDataException is tolerated
+ // here — an unexpected RuntimeException (NPE, contract violation) propagates so real bugs
+ // are not masked as a silent recall drop.
+ LOGGER.warn("Level-wise cluster computation failed; falling back to DFS-only cluster "
+ + "selection (recall may be reduced)", e);
+ globalLevelWiseClusters = null;
+ }
+ }
+ }
+
+ @Override
+ public ClusterSearchResult getNextCluster() throws HyracksDataException {
+ // Phase 1: Level-wise clusters (pre-computed, globally sorted by distance)
+ if (!levelWisePhaseComplete && globalLevelWiseClusters != null
+ && globalClusterIndex < globalLevelWiseClusters.size()) {
+
+ ClusterSearchResult nextCluster = globalLevelWiseClusters.get(globalClusterIndex);
+ globalClusterIndex++;
+
+ // Mark visited for DFS fallback deduplication
+ visitedCentroidIds.add(nextCluster.centroidId);
+
+ LOGGER.trace("Level-wise: cluster {}/{} (cid={}, distance={}, dirPage={})", globalClusterIndex,
+ globalLevelWiseClusters.size(), nextCluster.centroidId, nextCluster.distance,
+ nextCluster.directoryPageId);
+
+ if (globalClusterIndex >= globalLevelWiseClusters.size()) {
+ levelWisePhaseComplete = true;
+ LOGGER.trace("Level-wise phase complete. Visited centroids: {}", visitedCentroidIds);
+ }
+
+ return nextCluster;
+ }
+
+ // Phase 2: DFS fallback - get next from first component's DFS
+ levelWisePhaseComplete = true;
+
+ // After the level-wise clusters and before the DFS fallback, yield the displaced seed once.
+ // The DFS below can't reach it (already consumed and marked visited), so it must be explicit.
+ if (deferredSeedCluster != null) {
+ ClusterSearchResult seed = deferredSeedCluster;
+ deferredSeedCluster = null;
+ visitedCentroidIds.add(seed.centroidId);
+ LOGGER.trace("Probing deferred greedy-seed cluster cid={} (dirPage={})", seed.centroidId,
+ seed.directoryPageId);
+ return seed;
+ }
+
+ if (firstCursor == null) {
+ return null;
+ }
+
+ ClusterSearchResult next = firstCursor.findNextClusterDFS();
+
+ if (next == null) {
+ LOGGER.trace("DFS exhausted, no more clusters. Visited centroids: {}", visitedCentroidIds);
+ return null;
+ }
+
+ LOGGER.trace("DFS fallback: cluster cid={}, distance={}, dirPage={}", next.centroidId, next.distance,
+ next.directoryPageId);
+
+ return next;
+ }
+
+ @Override
+ public boolean hasMoreClusters() {
+ // Check level-wise first
+ if (!levelWisePhaseComplete && globalLevelWiseClusters != null
+ && globalClusterIndex < globalLevelWiseClusters.size()) {
+ return true;
+ }
+
+ // Deferred seed still pending.
+ if (deferredSeedCluster != null) {
+ return true;
+ }
+
+ // Check DFS via first cursor
+ if (firstCursor != null) {
+ return firstCursor.hasMoreClusters();
+ }
+
+ return false;
+ }
+
+ @Override
+ public void deferSeedCluster(ClusterSearchResult seed) {
+ if (seed == null || isInLevelWiseList(seed.centroidId)) {
+ // Nothing to rescue: a seed already in the level-wise window is scanned by the level-wise
+ // phase, so deferring it too would double-scan the cluster.
+ this.deferredSeedCluster = null;
+ } else {
+ // Out-of-window seed: the level-wise phase will not scan it, so probe it afterwards.
+ this.deferredSeedCluster = seed;
+ }
+ }
+
+ private boolean isInLevelWiseList(int centroidId) {
+ if (globalLevelWiseClusters == null) {
+ return false;
+ }
+ for (ClusterSearchResult c : globalLevelWiseClusters) {
+ if (c.centroidId == centroidId) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public boolean shouldStopAdvancing(int minClustersExplored, int resultsCollected) {
+ // Stop when: explored at least nprobe clusters AND collected at least K results
+ return minClustersExplored >= nprobe && resultsCollected >= K;
+ }
+
+ /**
+ * Returns the live visited-centroid set, which {@code getNextCluster()} also mutates. This strategy
+ * is single-threaded per cursor: the set is shared with the component cursors for dedup and must not
+ * be iterated concurrently with a {@code getNextCluster()} call. Not safe for parallel component probing.
+ */
+ @Override
+ public Set<Integer> getVisitedCentroidIds() {
+ return visitedCentroidIds;
+ }
+
+ @Override
+ public void setFirstCursorForDFS(VTreeSearchCursor firstCursor) {
+ this.firstCursor = firstCursor;
+ }
+
+ @Override
+ public ClusterSearchResult getFirstCluster() {
+ if (globalLevelWiseClusters != null && !globalLevelWiseClusters.isEmpty()) {
+ return globalLevelWiseClusters.get(0);
+ }
+ return null;
+ }
+
+ @Override
+ public void reset() {
+ this.globalLevelWiseClusters = null;
+ this.globalClusterIndex = 0;
+ this.levelWisePhaseComplete = false;
+ this.visitedCentroidIds.clear();
+ this.firstCursor = null;
+ this.deferredSeedCluster = null;
+ this.quantizedQueryVector = null;
+ this.quantizer = null;
+ }
+
+ @Override
+ public int getLevelWiseClusterCount() {
+ return globalLevelWiseClusters != null ? globalLevelWiseClusters.size() : 0;
+ }
+
+ @Override
+ public boolean isLevelWisePhaseComplete() {
+ return levelWisePhaseComplete;
+ }
+
+ // Getters for logging/debugging
+ public int getNprobe() {
+ return nprobe;
+ }
+
+ public double getMinProbeFraction() {
+ return minProbeFraction;
+ }
+
+ public double getEpsilon() {
+ return epsilon;
+ }
+
+ public int getK() {
+ return K;
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/PKOnlyTupleProjector.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/PKOnlyTupleProjector.java
new file mode 100644
index 0000000000..3bff0f09ca
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/PKOnlyTupleProjector.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.vector.impls;
+
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.storage.common.projection.ITupleProjector;
+
+/**
+ * Tuple projector that extracts only primary key fields from vector index search results.
+ *
+ * Vector index tuples contain: <distance, cosine, embedding, pk1, pk2, ...>
+ * This projector skips the secondary key fields (distance, cosine, embedding) and
+ * writes only the primary key fields to the output.
+ *
+ * This optimization avoids transferring large embedding vectors (4KB-16KB) when only
+ * primary keys are needed for subsequent primary index lookup.
+ */
+public class PKOnlyTupleProjector implements ITupleProjector {
+
+ private final int numSecondaryKeys; // Number of fields to skip
+ private final int numPrimaryKeys; // Number of PK fields to write
+
+ public PKOnlyTupleProjector(int numSecondaryKeys, int numPrimaryKeys) {
+ this.numSecondaryKeys = numSecondaryKeys;
+ this.numPrimaryKeys = numPrimaryKeys;
+ }
+
+ /**
+ * Writes only the primary-key fields into {@code dos}/{@code tb}; the meaningful projected output is
+ * that builder, NOT the return value. The original (unprojected) tuple is returned merely as a
+ * non-null sentinel — callers must consume {@code tb}, not the returned reference.
+ */
+ @Override
+ public ITupleReference project(ITupleReference tuple, DataOutput dos, ArrayTupleBuilder tb) throws IOException {
+ int totalFields = tuple.getFieldCount();
+ int startField = numSecondaryKeys;
+ int endField = numSecondaryKeys + numPrimaryKeys;
+
+ if (endField > totalFields) {
+ throw new IOException("Invalid field range: trying to extract fields [" + startField + ", " + endField
+ + ") from tuple with " + totalFields + " fields");
+ }
+
+ for (int i = startField; i < endField; i++) {
+ dos.write(tuple.getFieldData(i), tuple.getFieldStart(i), tuple.getFieldLength(i));
+ tb.addFieldEndOffset();
+ }
+
+ return tuple;
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/PKOnlyTupleProjectorFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/PKOnlyTupleProjectorFactory.java
new file mode 100644
index 0000000000..7518418e31
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/PKOnlyTupleProjectorFactory.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.vector.impls;
+
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.common.projection.ITupleProjector;
+import org.apache.hyracks.storage.common.projection.ITupleProjectorFactory;
+
+/**
+ * Factory for creating PKOnlyTupleProjector instances.
+ *
+ * Used by vector index search operators to create projectors that extract only
+ * primary key fields from vector index search results, skipping secondary key fields
+ * (distance, cosine similarity, embedding vector).
+ */
+public class PKOnlyTupleProjectorFactory implements ITupleProjectorFactory {
+
+ private static final long serialVersionUID = 1L;
+
+ private final int numSecondaryKeys; // Number of fields to skip
+ private final int numPrimaryKeys; // Number of PK fields to write
+
+ public PKOnlyTupleProjectorFactory(int numSecondaryKeys, int numPrimaryKeys) {
+ this.numSecondaryKeys = numSecondaryKeys;
+ this.numPrimaryKeys = numPrimaryKeys;
+ }
+
+ @Override
+ public ITupleProjector createTupleProjector(IHyracksTaskContext context) throws HyracksDataException {
+ return new PKOnlyTupleProjector(numSecondaryKeys, numPrimaryKeys);
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/SequentialClusterSelectionStrategy.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/SequentialClusterSelectionStrategy.java
new file mode 100644
index 0000000000..575c735dfb
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/SequentialClusterSelectionStrategy.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.lsm.vector.impls;
+
+import java.util.Collections;
+import java.util.Set;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.vector.api.IVTreeDistanceFunction;
+import org.apache.hyracks.storage.am.vector.api.IVTreeQuantizer;
+import org.apache.hyracks.storage.am.vector.impls.ClusterSearchResult;
+import org.apache.hyracks.storage.am.vector.impls.VTree;
+import org.apache.hyracks.storage.am.vector.impls.VTreeSearchCursor;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+/**
+ * Sequential cluster selection strategy.
+ * Iterates through all clusters in order (0 → 1 → 2 → ...) without early termination.
+ * Used for:
+ * - Merge operations (process ALL data from all components)
+ * - Index-driven KNN (scan all clusters with bidirectional pruning within each)
+ *
+ * Key differences from NprobeClusterSelectionStrategy:
+ * - No level-wise or DFS cluster selection
+ * - No nprobe/K-based early termination (shouldStopAdvancing always returns false)
+ * - Sequential cluster iteration via first cursor's advanceToNextCluster()
+ * - All components advance through clusters TOGETHER in lock-step
+ */
+public class SequentialClusterSelectionStrategy implements IClusterSelectionStrategy {
+
+ private static final Logger LOGGER = LogManager.getLogger();
+
+ // First component cursor for sequential cluster iteration
+ private VTreeSearchCursor firstCursor;
+
+ public SequentialClusterSelectionStrategy() {
+ }
+
+ @Override
+ public void initialize(VTree vTree, double[] queryVector, IVTreeDistanceFunction distFunc, int k)
+ throws HyracksDataException {
+ // Sequential strategy doesn't need tree/query/distance info.
+ // Cluster iteration is handled by the underlying cursor in full-scan mode.
+ }
+
+ @Override
+ public ClusterSearchResult getNextCluster() throws HyracksDataException {
+ if (firstCursor == null) {
+ return null;
+ }
+
+ // Check if there are more clusters available
+ if (!firstCursor.hasMoreClusters()) {
+ LOGGER.trace("No more clusters");
+ return null;
+ }
+
+ // Advance the first cursor to the next cluster (sequential iteration)
+ boolean advanced = firstCursor.advanceToNextCluster();
+ if (!advanced) {
+ LOGGER.trace("Failed to advance to next cluster");
+ return null;
+ }
+
+ // Get the cluster result after advancement
+ ClusterSearchResult next = firstCursor.getCurrentClusterResult();
+
+ if (next != null) {
+ LOGGER.trace("Next cluster: cid={}, dirPage={}", next.centroidId, next.directoryPageId);
+ } else {
+ LOGGER.trace("advanceToNextCluster succeeded but getCurrentClusterResult is null");
+ }
+
+ return next;
+ }
+
+ @Override
+ public boolean hasMoreClusters() {
+ return firstCursor != null && firstCursor.hasMoreClusters();
+ }
+
+ @Override
+ public boolean shouldStopAdvancing(int minClustersExplored, int resultsCollected) {
+ // Never stop early in merge mode - must process ALL data
+ return false;
+ }
+
+ @Override
+ public Set<Integer> getVisitedCentroidIds() {
+ // No deduplication in sequential (full-scan) mode — each cluster is visited exactly once in order.
+ return Collections.emptySet();
+ }
+
+ @Override
+ public void setFirstCursorForDFS(VTreeSearchCursor firstCursor) {
+ this.firstCursor = firstCursor;
+ }
+
+ @Override
+ public ClusterSearchResult getFirstCluster() {
+ // First cluster is already opened by the cursor during doOpen()
+ // Return current cluster result from first cursor
+ return firstCursor != null ? firstCursor.getCurrentClusterResult() : null;
+ }
+
+ @Override
+ public void setQuantizer(double[] quantizedQueryVector, IVTreeQuantizer quantizer) {
+ // Sequential (full-scan) strategy does not use quantized distance.
+ }
+
+ @Override
+ public void deferSeedCluster(ClusterSearchResult seed) {
+ // Sequential (full-scan) strategy has no greedy seed to defer.
+ }
+
+ @Override
+ public void reset() {
+ // No per-run state to reset in sequential mode.
+ }
+
+ @Override
+ public int getLevelWiseClusterCount() {
+ // No level-wise in sequential mode
+ return 0;
+ }
+
+ @Override
+ public boolean isLevelWisePhaseComplete() {
+ // Always "complete" since we don't use level-wise
+ return true;
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/SpillableTopKBuffer.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/SpillableTopKBuffer.java
new file mode 100644
index 0000000000..7fdbb3cbed
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/SpillableTopKBuffer.java
@@ -0,0 +1,433 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.vector.impls;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Objects;
+
+import org.apache.hyracks.api.comm.FixedSizeFrame;
+import org.apache.hyracks.api.comm.IFrame;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
+import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
+import org.apache.hyracks.api.exceptions.ErrorCode;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
+import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAppender;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.dataflow.common.data.marshalling.DoubleSerializerDeserializer;
+import org.apache.hyracks.dataflow.common.io.GeneratedRunFileReader;
+import org.apache.hyracks.dataflow.common.io.RunFileWriter;
+import org.apache.hyracks.dataflow.std.buffermanager.IDeletableTupleBufferManager;
+import org.apache.hyracks.dataflow.std.buffermanager.IFramePool;
+import org.apache.hyracks.dataflow.std.buffermanager.ITuplePointerAccessor;
+import org.apache.hyracks.dataflow.std.buffermanager.VariableDeletableTupleMemoryManager;
+import org.apache.hyracks.dataflow.std.buffermanager.VariableFramePool;
+import org.apache.hyracks.dataflow.std.structures.IResetableComparable;
+import org.apache.hyracks.dataflow.std.structures.MaxHeap;
+import org.apache.hyracks.dataflow.std.structures.TuplePointer;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+/**
+ * A spillable top-K buffer for vector ANN search.
+ *
+ * In-memory: MaxHeap of VectorSearchHeapEntry + VariableDeletableTupleMemoryManager (frame-backed).
+ * On budget exhaustion: sorts current entries by dqx, writes worst half to a RunFileWriter,
+ * resets buffer with best half, and continues. At drain time, merges in-memory entries with
+ * spilled run files.
+ *
+ * Follows the inverted index precedent (InvertedIndexSearchResult) for dual-mode
+ * in-memory/disk operation with managed workspace files.
+ */
+public class SpillableTopKBuffer {
+
+ private static final Logger LOGGER = LogManager.getLogger();
+ private static final int DEFAULT_FRAMES_LIMIT = 4;
+ private static final String SPILL_FILE_PREFIX = "VectorTopKSpill";
+
+ // Configuration
+ private final int candidateLimit;
+ private final int frameSize;
+ private final int budgetBytes;
+ private final IHyracksTaskContext ctx;
+
+ // In-memory top-K
+ private MaxHeap topKHeap;
+ private IDeletableTupleBufferManager bufferManager;
+ private ITuplePointerAccessor bufferAccessor;
+ private final VectorSearchHeapEntry peekEntry;
+ private final VectorSearchHeapEntry insertEntry;
+
+ // Adapter: serialize ITupleReference into frame format for buffer manager
+ private IFrame singleTupleFrame;
+ private FrameTupleAppender singleTupleAppender;
+ private FrameTupleAccessor singleTupleAccessor;
+ private RecordDescriptor tupleRecordDescriptor;
+ private boolean initialized;
+
+ // Spill state
+ private final List<GeneratedRunFileReader> spilledRuns;
+ private int spilledTupleCount;
+ /** Worst (largest) dqx among any spilled entry. -Inf when nothing has been spilled yet. */
+ private double maxSpilledDqx = Double.NEGATIVE_INFINITY;
+
+ // Run file serialization (dqx-prefixed tuples)
+ private RecordDescriptor spillRecordDescriptor;
+ private IFrame spillOutputFrame;
+ private FrameTupleAppender spillOutputAppender;
+
+ public SpillableTopKBuffer(int candidateLimit, IHyracksTaskContext ctx) {
+ this.candidateLimit = candidateLimit;
+ this.ctx = Objects.requireNonNull(ctx, "SpillableTopKBuffer requires a non-null IHyracksTaskContext "
+ + "(set HyracksConstants.HYRACKS_TASK_CONTEXT in the IndexAccessParameters before opening the cursor)");
+ this.frameSize = ctx.getInitialFrameSize();
+ this.budgetBytes = DEFAULT_FRAMES_LIMIT * frameSize;
+ this.peekEntry = new VectorSearchHeapEntry();
+ this.insertEntry = new VectorSearchHeapEntry();
+ this.spilledRuns = new ArrayList<>();
+ this.spilledTupleCount = 0;
+ this.initialized = false;
+ this.singleTupleAppender = new FrameTupleAppender();
+ }
+
+ @SuppressWarnings("rawtypes")
+ public void ensureInitialized(ITupleReference tuple) throws HyracksDataException {
+ if (initialized) {
+ return;
+ }
+ int fieldCount = tuple.getFieldCount();
+ this.tupleRecordDescriptor = new RecordDescriptor(new ISerializerDeserializer[fieldCount]);
+ IFramePool framePool = new VariableFramePool(ctx, budgetBytes);
+ this.bufferManager = new VariableDeletableTupleMemoryManager(framePool, tupleRecordDescriptor);
+ this.bufferAccessor = bufferManager.createTuplePointerAccessor();
+ this.topKHeap = new MaxHeap(new VectorSearchHeapEntryFactory(), candidateLimit);
+ this.singleTupleFrame = new FixedSizeFrame(ctx.allocateFrame(frameSize));
+ this.singleTupleAccessor = new FrameTupleAccessor(tupleRecordDescriptor);
+
+ // Spill record descriptor: [dqx (double), original fields...]
+ ISerializerDeserializer[] spillSerdes = new ISerializerDeserializer[fieldCount + 1];
+ spillSerdes[0] = DoubleSerializerDeserializer.INSTANCE;
+ this.spillRecordDescriptor = new RecordDescriptor(spillSerdes);
+
+ initialized = true;
+ }
+
+ /**
+ * Insert a tuple with its approximate distance. Spills to disk if the memory budget is exceeded.
+ * A candidate strictly worse than the current top-K worst is rejected up front; a tuple too large for a
+ * single frame throws {@link ErrorCode#RECORD_IS_TOO_LARGE} rather than being silently dropped.
+ */
+ public void insert(ITupleReference tuple, double dqx) throws HyracksDataException {
+ ensureInitialized(tuple);
+
+ // Early reject: skip the new tuple only if it is strictly worse than the worst kept
+ // candidate across BOTH the in-memory heap and any spilled run.
+ //
+ // Spilled tuples are immutable on disk — once a tuple is in a run file we cannot
+ // evict it. So the floor for "is this new tuple worth keeping?" is the *larger* of
+ // (in-memory peekMax, maxSpilledDqx). Comparing only against the in-memory peekMax
+ // is unsafe after a spill, because the in-memory side holds the *best* half and the
+ // disk side holds the *worst* half; an incoming dqx that beats the spilled worst is
+ // a valid top-K candidate even when it is worse than every in-memory entry.
+ //
+ // Use strict '>' so ties at the K-boundary are admitted (both are equally valid).
+ if (getNumEntries() >= candidateLimit && topKHeap.getNumEntries() > 0) {
+ topKHeap.peekMax(peekEntry);
+ double globalWorst = Math.max(peekEntry.dqx, maxSpilledDqx);
+ if (dqx > globalWorst) {
+ return;
+ }
+ }
+
+ // Serialize tuple into frame format
+ singleTupleAppender.reset(singleTupleFrame, true);
+ if (!singleTupleAppender.append(tuple)) {
+ // A tuple that does not fit a fresh single-frame staging buffer is oversized. Fail loudly with
+ // RECORD_IS_TOO_LARGE (as the buffer-manager path below does), rather than silently dropping it:
+ // the contract is "never drops tuples", and a silent drop would quietly miss a valid top-K
+ // candidate.
+ int tupleBytes = 0;
+ for (int f = 0; f < tuple.getFieldCount(); f++) {
+ tupleBytes += tuple.getFieldLength(f);
+ }
+ throw HyracksDataException.create(ErrorCode.RECORD_IS_TOO_LARGE, tupleBytes,
+ singleTupleFrame.getBuffer().capacity());
+ }
+ singleTupleAccessor.reset(singleTupleFrame.getBuffer());
+
+ // Decide whether to grow the heap or replace its current max.
+ // replaceMax is only safe when the new tuple is no worse than the in-memory worst
+ // (otherwise we would evict a better candidate for a worse one). For the in-between
+ // case — new is worse than in-memory peekMax but still better than maxSpilledDqx —
+ // we must grow the heap instead, so the better in-memory candidates survive.
+ boolean heapFull = topKHeap.getNumEntries() >= candidateLimit;
+ boolean useReplaceMax = false;
+ if (heapFull) {
+ topKHeap.peekMax(peekEntry);
+ if (dqx <= peekEntry.dqx) {
+ useReplaceMax = true;
+ }
+ }
+
+ TuplePointer outPointer = new TuplePointer();
+ if (useReplaceMax) {
+ // Heap full and the new tuple is at least as good as the worst in-memory entry. Insert the
+ // newcomer FIRST and evict the victim only once it is safely stored — deleting first and then
+ // failing the insert would lose BOTH tuples and leave a heap entry dangling at a freed pointer.
+ if (bufferManager.insertTuple(singleTupleAccessor, 0, outPointer)) {
+ bufferManager.deleteTuple(peekEntry.tuplePointer);
+ insertEntry.reset(outPointer, dqx);
+ topKHeap.replaceMax(insertEntry);
+ } else {
+ // Buffer full: spill the worst half of the heap to disk to free space. After spilling the
+ // heap is no longer full (and the worst entry may itself have been spilled), so fall back
+ // to a plain grow-insert instead of replacing.
+ spillWorstHalf();
+ if (!bufferManager.insertTuple(singleTupleAccessor, 0, outPointer)) {
+ throw HyracksDataException.create(ErrorCode.RECORD_IS_TOO_LARGE,
+ singleTupleAccessor.getTupleLength(0), singleTupleFrame.getBuffer().capacity());
+ }
+ insertEntry.reset(outPointer, dqx);
+ topKHeap.insert(insertEntry);
+ }
+ } else {
+ // Heap has room, or new tuple is worse than in-memory peekMax but better than
+ // maxSpilledDqx — grow the heap (spill on memory pressure).
+ if (!bufferManager.insertTuple(singleTupleAccessor, 0, outPointer)) {
+ spillWorstHalf();
+ if (!bufferManager.insertTuple(singleTupleAccessor, 0, outPointer)) {
+ throw HyracksDataException.create(ErrorCode.RECORD_IS_TOO_LARGE,
+ singleTupleAccessor.getTupleLength(0), singleTupleFrame.getBuffer().capacity());
+ }
+ }
+ insertEntry.reset(outPointer, dqx);
+ topKHeap.insert(insertEntry);
+ }
+ }
+
+ /**
+ * Peek the worst (maximum) dqx in the in-memory heap, or {@link Double#MAX_VALUE} when the heap is
+ * empty. The sentinel is chosen so an empty heap always accepts the next candidate (nothing is worse
+ * than {@code MAX_VALUE}); callers treat it as "no worst yet", not a real distance.
+ */
+ public double peekWorstDqx() {
+ if (topKHeap == null || topKHeap.getNumEntries() == 0) {
+ return Double.MAX_VALUE;
+ }
+ topKHeap.peekMax(peekEntry);
+ return peekEntry.dqx;
+ }
+
+ /**
+ * Total entries tracked (in-memory + spilled).
+ */
+ public int getNumEntries() {
+ int inMemory = (topKHeap != null) ? topKHeap.getNumEntries() : 0;
+ return inMemory + spilledTupleCount;
+ }
+
+ /**
+ * Spill the worst half of the in-memory heap to a sorted run file.
+ * Keeps the best half in memory for continued searching.
+ */
+ @SuppressWarnings("deprecation")
+ private void spillWorstHalf() throws HyracksDataException {
+ int n = topKHeap.getNumEntries();
+ if (n == 0) {
+ return;
+ }
+
+ LOGGER.info("[SpillableTopKBuffer] Spilling: {} entries in heap, spilling {}", n, n - n / 2);
+
+ // Sort entries ascending by dqx
+ IResetableComparable[] entries = topKHeap.getEntries();
+ Arrays.sort(entries, 0, n, Comparator.comparingDouble(e -> ((VectorSearchHeapEntry) e).dqx));
+
+ int keepCount = n / 2;
+ int spillCount = n - keepCount;
+
+ // Create run file
+ FileReference spillFile = ctx.getJobletContext().createManagedWorkspaceFile(SPILL_FILE_PREFIX);
+ RunFileWriter writer = new RunFileWriter(spillFile, ctx.getIoManager());
+ writer.open();
+
+ try {
+ // Initialize spill output frame lazily
+ if (spillOutputFrame == null) {
+ spillOutputFrame = new FixedSizeFrame(ByteBuffer.allocate(frameSize));
+ spillOutputAppender = new FrameTupleAppender();
+ }
+ spillOutputAppender.reset(spillOutputFrame, true);
+
+ // Write worst half (indices keepCount..n-1) as dqx-prefixed tuples
+ for (int i = keepCount; i < n; i++) {
+ VectorSearchHeapEntry entry = (VectorSearchHeapEntry) entries[i];
+ bufferAccessor.reset(entry.tuplePointer);
+
+ // Build dqx-prefixed tuple: [dqx_bytes][original_tuple_bytes]
+ if (!appendDqxPrefixedTuple(spillOutputAppender, entry.dqx, bufferAccessor)) {
+ // Frame full — flush and retry
+ spillOutputAppender.write(writer, true);
+ spillOutputAppender.reset(spillOutputFrame, true);
+ if (!appendDqxPrefixedTuple(spillOutputAppender, entry.dqx, bufferAccessor)) {
+ throw new HyracksDataException("Single dqx-prefixed tuple too large for spill frame");
+ }
+ }
+ }
+ // Flush remaining
+ if (spillOutputAppender.getTupleCount() > 0) {
+ spillOutputAppender.write(writer, true);
+ }
+ } finally {
+ writer.close();
+ }
+
+ spilledRuns.add(writer.createDeleteOnCloseReader());
+
+ // Track the worst dqx that has ever been spilled. Spilled tuples are immutable
+ // (we cannot evict them from the run file), so they form the floor for the
+ // early-reject decision in insert().
+ double thisSpillMax = ((VectorSearchHeapEntry) entries[n - 1]).dqx;
+ if (thisSpillMax > maxSpilledDqx) {
+ maxSpilledDqx = thisSpillMax;
+ }
+
+ // Delete spilled entries from buffer manager
+ for (int i = keepCount; i < n; i++) {
+ VectorSearchHeapEntry entry = (VectorSearchHeapEntry) entries[i];
+ bufferManager.deleteTuple(entry.tuplePointer);
+ }
+ spilledTupleCount += spillCount;
+
+ // Rebuild heap with kept entries only
+ // Save kept entries' data before resetting
+ TuplePointer[] keptPointers = new TuplePointer[keepCount];
+ double[] keptDqx = new double[keepCount];
+ for (int i = 0; i < keepCount; i++) {
+ VectorSearchHeapEntry entry = (VectorSearchHeapEntry) entries[i];
+ keptPointers[i] = new TuplePointer(entry.tuplePointer.getFrameIndex(), entry.tuplePointer.getTupleIndex());
+ keptDqx[i] = entry.dqx;
+ }
+
+ topKHeap.reset();
+ for (int i = 0; i < keepCount; i++) {
+ insertEntry.reset(keptPointers[i], keptDqx[i]);
+ topKHeap.insert(insertEntry);
+ }
+
+ LOGGER.info("[SpillableTopKBuffer] Spill complete: kept={}, spilled={} (total spilled: {}), runs={}", keepCount,
+ spillCount, spilledTupleCount, spilledRuns.size());
+ }
+
+ /**
+ * Append a dqx-prefixed tuple to the appender.
+ * Format: [8-byte dqx as serialized double][original tuple field bytes]
+ */
+ private boolean appendDqxPrefixedTuple(FrameTupleAppender appender, double dqx, ITuplePointerAccessor tupleAccessor)
+ throws HyracksDataException {
+ // Compute total size: dqx field (serialized double) + original fields
+ int numOriginalFields = tupleAccessor.getFieldCount();
+ int tupleDataSize = 8; // dqx double
+ for (int f = 0; f < numOriginalFields; f++) {
+ tupleDataSize += tupleAccessor.getFieldLength(f);
+ }
+
+ // Build the tuple bytes manually into a temp buffer
+ // Frame tuple format: [field end offsets (4 bytes each)][field data bytes]
+ int numFields = numOriginalFields + 1;
+ int headerSize = numFields * 4; // field end offsets
+ int totalTupleSize = headerSize + tupleDataSize;
+
+ byte[] tupleBytes = new byte[totalTupleSize];
+ ByteBuffer tb = ByteBuffer.wrap(tupleBytes);
+
+ // Field end offsets (relative to start of field data area)
+ int fieldDataOffset = 0;
+ // Field 0: dqx (8 bytes)
+ fieldDataOffset += 8;
+ tb.putInt(fieldDataOffset);
+ // Fields 1..N: original fields
+ for (int f = 0; f < numOriginalFields; f++) {
+ fieldDataOffset += tupleAccessor.getFieldLength(f);
+ tb.putInt(fieldDataOffset);
+ }
+
+ // Field data: dqx
+ tb.putDouble(dqx);
+ // Field data: original fields
+ byte[] srcData = tupleAccessor.getBuffer().array();
+ for (int f = 0; f < numOriginalFields; f++) {
+ int fStart = tupleAccessor.getAbsFieldStartOffset(f);
+ int fLen = tupleAccessor.getFieldLength(f);
+ tb.put(srcData, fStart, fLen);
+ }
+
+ // Use a temp single-tuple frame to append via FrameTupleAccessor
+ // Actually, we can use the raw append method
+ return appender.append(tupleBytes, 0, totalTupleSize);
+ }
+
+ /**
+ * Drain all results sorted by dqx ascending.
+ */
+ @SuppressWarnings("deprecation")
+ public SpillableTopKDrainIterator drain() throws HyracksDataException {
+ if (!initialized || topKHeap == null) {
+ return SpillableTopKDrainIterator.empty();
+ }
+
+ // Sort in-memory entries ascending by dqx
+ int n = topKHeap.getNumEntries();
+ IResetableComparable[] entries = topKHeap.getEntries();
+ Arrays.sort(entries, 0, n, Comparator.comparingDouble(e -> ((VectorSearchHeapEntry) e).dqx));
+
+ VectorSearchHeapEntry[] sortedInMemory = new VectorSearchHeapEntry[n];
+ for (int i = 0; i < n; i++) {
+ sortedInMemory[i] = (VectorSearchHeapEntry) entries[i];
+ }
+
+ if (spilledRuns.isEmpty()) {
+ // Fast path: no spill, just iterate sorted in-memory entries
+ return SpillableTopKDrainIterator.inMemoryOnly(sortedInMemory, bufferAccessor);
+ }
+
+ // Merge path: open run readers and merge with in-memory entries
+ return SpillableTopKDrainIterator.withMerge(sortedInMemory, bufferAccessor, spilledRuns, spillRecordDescriptor,
+ ctx, candidateLimit);
+ }
+
+ public void close() throws HyracksDataException {
+ if (bufferManager != null) {
+ bufferManager.close();
+ bufferManager = null;
+ }
+ for (GeneratedRunFileReader reader : spilledRuns) {
+ reader.close();
+ }
+ spilledRuns.clear();
+ topKHeap = null;
+ }
+
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/SpillableTopKDrainIterator.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/SpillableTopKDrainIterator.java
new file mode 100644
index 0000000000..b9787fdc08
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/SpillableTopKDrainIterator.java
@@ -0,0 +1,409 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.vector.impls;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.PriorityQueue;
+
+import org.apache.hyracks.api.comm.IFrame;
+import org.apache.hyracks.api.comm.VSizeFrame;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.primitive.DoublePointable;
+import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.dataflow.common.io.GeneratedRunFileReader;
+import org.apache.hyracks.dataflow.std.buffermanager.ITuplePointerAccessor;
+
+/**
+ * Iterator for draining results from SpillableTopKBuffer.
+ *
+ * Two modes:
+ * - No-spill fast path: iterates sorted VectorSearchHeapEntry[] from frame buffer (zero-copy)
+ * - Merge path: k-way merge of sorted in-memory entries + sorted run files by dqx ascending
+ */
+public class SpillableTopKDrainIterator {
+
+ // Shared state
+ private final boolean mergeMode;
+ private int outputCount;
+ private final int outputLimit;
+
+ // No-spill fast path state
+ private final VectorSearchHeapEntry[] sortedInMemory;
+ private final ITuplePointerAccessor bufferAccessor;
+ private int inMemoryIndex;
+
+ // Merge path state
+ private PriorityQueue<MergeSource> mergeQueue;
+ private MergeSource currentSource;
+ // Every source created for the merge, retained so close() releases ALL run-file readers — not just
+ // those still in the queue (an exhausted source is dropped from the queue but its reader stays open).
+ private final List<MergeSource> allSources = new ArrayList<>();
+ /**
+ * True once {@link #hasNext()} has prepared {@link #currentSource} to point at the
+ * tuple that the next {@link #next()} / {@link #getTuple()} call should yield. Cleared
+ * by {@link #next()} so the following {@link #hasNext()} prefetches again.
+ */
+ private boolean prefetched;
+
+ // Adapter for returning tuples from in-memory buffer
+ private final TuplePointerTupleReference inMemoryTupleRef;
+
+ private SpillableTopKDrainIterator(VectorSearchHeapEntry[] sortedInMemory, ITuplePointerAccessor bufferAccessor,
+ boolean mergeMode, int outputLimit) {
+ this.sortedInMemory = sortedInMemory;
+ this.bufferAccessor = bufferAccessor;
+ this.mergeMode = mergeMode;
+ this.outputLimit = outputLimit;
+ this.inMemoryIndex = 0;
+ this.outputCount = 0;
+ this.inMemoryTupleRef = new TuplePointerTupleReference();
+ }
+
+ static SpillableTopKDrainIterator empty() {
+ return new SpillableTopKDrainIterator(new VectorSearchHeapEntry[0], null, false, 0);
+ }
+
+ static SpillableTopKDrainIterator inMemoryOnly(VectorSearchHeapEntry[] sortedEntries,
+ ITuplePointerAccessor accessor) {
+ return new SpillableTopKDrainIterator(sortedEntries, accessor, false, sortedEntries.length);
+ }
+
+ static SpillableTopKDrainIterator withMerge(VectorSearchHeapEntry[] sortedInMemory,
+ ITuplePointerAccessor bufferAccessor, List<GeneratedRunFileReader> spilledRuns,
+ RecordDescriptor spillRecordDescriptor, IHyracksTaskContext ctx, int candidateLimit)
+ throws HyracksDataException {
+
+ SpillableTopKDrainIterator iter =
+ new SpillableTopKDrainIterator(sortedInMemory, bufferAccessor, true, candidateLimit);
+ iter.initMerge(spilledRuns, spillRecordDescriptor, ctx);
+ return iter;
+ }
+
+ private void initMerge(List<GeneratedRunFileReader> spilledRuns, RecordDescriptor spillRecordDescriptor,
+ IHyracksTaskContext ctx) throws HyracksDataException {
+ mergeQueue = new PriorityQueue<>(spilledRuns.size() + 1,
+ (a, b) -> Double.compare(a.getCurrentDqx(), b.getCurrentDqx()));
+
+ // Add in-memory source
+ if (sortedInMemory.length > 0) {
+ InMemoryMergeSource inMemSource = new InMemoryMergeSource(sortedInMemory, bufferAccessor);
+ allSources.add(inMemSource);
+ if (inMemSource.advance()) {
+ mergeQueue.offer(inMemSource);
+ }
+ }
+
+ // Add run file sources. Register in allSources BEFORE advance() — the reader is already open()ed,
+ // so it must be closed even if the run is empty and never enters the queue.
+ for (GeneratedRunFileReader runReader : spilledRuns) {
+ runReader.open();
+ RunFileMergeSource runSource = new RunFileMergeSource(runReader, spillRecordDescriptor, ctx);
+ allSources.add(runSource);
+ if (runSource.advance()) {
+ mergeQueue.offer(runSource);
+ }
+ }
+ }
+
+ public boolean hasNext() throws HyracksDataException {
+ if (outputCount >= outputLimit) {
+ return false;
+ }
+ if (!mergeMode) {
+ return inMemoryIndex < sortedInMemory.length;
+ }
+ // Stage the next yield. Doing this in hasNext (rather than next) keeps the
+ // hasNext/next/getTuple contract honest: hasNext returns true iff there really
+ // is a next tuple. Without this, hasNext can return true on the strength of a
+ // non-null currentSource that is actually the just-yielded source — and the
+ // following next() then advances it past its end while the queue is empty,
+ // leaving currentSource null for getTuple to NPE on.
+ if (!prefetched) {
+ if (currentSource != null) {
+ if (currentSource.advance()) {
+ mergeQueue.offer(currentSource);
+ }
+ currentSource = null;
+ }
+ currentSource = mergeQueue.poll();
+ prefetched = true;
+ }
+ return currentSource != null;
+ }
+
+ public void next() throws HyracksDataException {
+ if (!mergeMode) {
+ VectorSearchHeapEntry entry = sortedInMemory[inMemoryIndex++];
+ bufferAccessor.reset(entry.tuplePointer);
+ inMemoryTupleRef.reset(bufferAccessor);
+ } else {
+ // currentSource was prepared by hasNext(); just consume the prefetch.
+ // Callers that skip hasNext() are tolerated by re-running the staging logic.
+ if (!prefetched) {
+ hasNext();
+ }
+ prefetched = false;
+ }
+ outputCount++;
+ }
+
+ public ITupleReference getTuple() {
+ if (!mergeMode) {
+ return inMemoryTupleRef;
+ }
+ return currentSource.getCurrentTuple();
+ }
+
+ /**
+ * Returns the {@code D(q,x)} value of the entry most recently produced by {@link #next()}.
+ * Mirrors {@link #getTuple()} — only valid between a {@code next()} call and the next
+ * {@code next()} / {@code close()}. Used by the index-only ANN plan to expose per-tuple
+ * distance to the operator pushable (see {@code IVectorSearchCursor.getCurrentDistance()}).
+ *
+ * @return the dqx of the current drain entry, or {@link Double#NaN} if no current entry.
+ */
+ public double getCurrentDqx() {
+ if (!mergeMode) {
+ // next() advanced inMemoryIndex past the consumed entry; the "current" entry is at
+ // inMemoryIndex - 1. Guard the pre-first-next() case explicitly.
+ if (inMemoryIndex <= 0 || inMemoryIndex > sortedInMemory.length) {
+ return Double.NaN;
+ }
+ return sortedInMemory[inMemoryIndex - 1].dqx;
+ }
+ return currentSource != null ? currentSource.getCurrentDqx() : Double.NaN;
+ }
+
+ public void close() throws HyracksDataException {
+ // Close EVERY source created for the merge (see allSources), continuing past a failure so one
+ // bad reader can't leak the rest. RunFileReader.close() is idempotent, so this is safe even for
+ // sources still referenced by the queue/currentSource.
+ HyracksDataException failure = null;
+ for (MergeSource source : allSources) {
+ try {
+ source.close();
+ } catch (HyracksDataException e) {
+ if (failure == null) {
+ failure = e;
+ } else {
+ failure.addSuppressed(e);
+ }
+ }
+ }
+ allSources.clear();
+ if (mergeQueue != null) {
+ mergeQueue.clear();
+ }
+ currentSource = null;
+ if (failure != null) {
+ throw failure;
+ }
+ }
+
+ // ==================== Merge Sources ====================
+
+ private interface MergeSource {
+ double getCurrentDqx();
+
+ ITupleReference getCurrentTuple();
+
+ boolean advance() throws HyracksDataException;
+
+ void close() throws HyracksDataException;
+ }
+
+ /**
+ * Merge source backed by sorted in-memory VectorSearchHeapEntry array.
+ */
+ private static class InMemoryMergeSource implements MergeSource {
+ private final VectorSearchHeapEntry[] entries;
+ private final ITuplePointerAccessor accessor;
+ private final TuplePointerTupleReference tupleRef;
+ private int index;
+ double currentDqx;
+
+ InMemoryMergeSource(VectorSearchHeapEntry[] entries, ITuplePointerAccessor accessor) {
+ this.entries = entries;
+ this.accessor = accessor;
+ this.tupleRef = new TuplePointerTupleReference();
+ this.index = 0;
+ }
+
+ @Override
+ public double getCurrentDqx() {
+ return currentDqx;
+ }
+
+ @Override
+ public ITupleReference getCurrentTuple() {
+ return tupleRef;
+ }
+
+ @Override
+ public boolean advance() {
+ if (index >= entries.length) {
+ return false;
+ }
+ VectorSearchHeapEntry entry = entries[index++];
+ accessor.reset(entry.tuplePointer);
+ tupleRef.reset(accessor);
+ currentDqx = entry.dqx;
+ return true;
+ }
+
+ @Override
+ public void close() {
+ }
+ }
+
+ /**
+ * Merge source backed by a sorted run file with dqx-prefixed tuples.
+ * Run file tuple format: [8-byte dqx][original tuple fields]
+ * This source strips the dqx prefix and exposes the original tuple.
+ */
+ private static class RunFileMergeSource implements MergeSource {
+ private final GeneratedRunFileReader reader;
+ private final FrameTupleAccessor frameAccessor;
+ private final IFrame readFrame;
+ private int tupleIndex;
+ private int tupleCount;
+ double currentDqx;
+ private final RunFileTupleReference tupleRef;
+
+ RunFileMergeSource(GeneratedRunFileReader reader, RecordDescriptor spillRecordDescriptor,
+ IHyracksTaskContext ctx) throws HyracksDataException {
+ this.reader = reader;
+ this.readFrame = new VSizeFrame(ctx);
+ this.frameAccessor = new FrameTupleAccessor(spillRecordDescriptor);
+ this.tupleIndex = 0;
+ this.tupleCount = 0;
+ this.tupleRef = new RunFileTupleReference();
+ }
+
+ @Override
+ public double getCurrentDqx() {
+ return currentDqx;
+ }
+
+ @Override
+ public ITupleReference getCurrentTuple() {
+ return tupleRef;
+ }
+
+ @Override
+ public boolean advance() throws HyracksDataException {
+ while (true) {
+ if (tupleIndex < tupleCount) {
+ // Read dqx from field 0 of the spill tuple
+ byte[] data = frameAccessor.getBuffer().array();
+ int fieldStart = frameAccessor.getAbsoluteFieldStartOffset(tupleIndex, 0);
+ currentDqx = DoublePointable.getDouble(data, fieldStart);
+ // Set up tuple reference that strips field 0 (dqx) and exposes fields 1..N
+ tupleRef.reset(frameAccessor, tupleIndex);
+ tupleIndex++;
+ return true;
+ }
+ // Need to read next frame from run file
+ if (!reader.nextFrame(readFrame)) {
+ return false;
+ }
+ frameAccessor.reset(readFrame.getBuffer());
+ tupleCount = frameAccessor.getTupleCount();
+ tupleIndex = 0;
+ }
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ reader.close();
+ }
+ }
+
+ // ==================== Tuple Reference Adapters ====================
+
+ /**
+ * Adapts ITuplePointerAccessor to ITupleReference. Points into frame buffer (zero-copy).
+ */
+ static class TuplePointerTupleReference implements ITupleReference {
+ private ITuplePointerAccessor accessor;
+
+ void reset(ITuplePointerAccessor accessor) {
+ this.accessor = accessor;
+ }
+
+ @Override
+ public int getFieldCount() {
+ return accessor.getFieldCount();
+ }
+
+ @Override
+ public byte[] getFieldData(int fIdx) {
+ return accessor.getBuffer().array();
+ }
+
+ @Override
+ public int getFieldStart(int fIdx) {
+ return accessor.getAbsFieldStartOffset(fIdx);
+ }
+
+ @Override
+ public int getFieldLength(int fIdx) {
+ return accessor.getFieldLength(fIdx);
+ }
+ }
+
+ /**
+ * Adapts a dqx-prefixed run file tuple to ITupleReference, stripping field 0 (dqx).
+ * Run file format: field[0]=dqx, field[1..N]=original tuple fields.
+ * This reference exposes fields 1..N as fields 0..N-1.
+ */
+ private static class RunFileTupleReference implements ITupleReference {
+ private FrameTupleAccessor accessor;
+ private int tupleIndex;
+
+ void reset(FrameTupleAccessor accessor, int tupleIndex) {
+ this.accessor = accessor;
+ this.tupleIndex = tupleIndex;
+ }
+
+ @Override
+ public int getFieldCount() {
+ return accessor.getFieldCount() - 1;
+ }
+
+ @Override
+ public byte[] getFieldData(int fIdx) {
+ return accessor.getBuffer().array();
+ }
+
+ @Override
+ public int getFieldStart(int fIdx) {
+ return accessor.getAbsoluteFieldStartOffset(tupleIndex, fIdx + 1);
+ }
+
+ @Override
+ public int getFieldLength(int fIdx) {
+ return accessor.getFieldLength(tupleIndex, fIdx + 1);
+ }
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/VTreeFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/VTreeFactory.java
new file mode 100644
index 0000000000..0fba8f4125
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/VTreeFactory.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.lsm.vector.impls;
+
+import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.IIOManager;
+import org.apache.hyracks.storage.am.common.api.IPageManagerFactory;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
+import org.apache.hyracks.storage.am.vector.api.IVTreeBinaryAccessorFactory;
+import org.apache.hyracks.storage.am.vector.api.IVTreeDataTupleBuilderFactory;
+import org.apache.hyracks.storage.am.vector.api.IVTreeDistanceFunctionFactory;
+import org.apache.hyracks.storage.am.vector.api.VTreeQuantizationParams;
+import org.apache.hyracks.storage.am.vector.impls.VTree;
+import org.apache.hyracks.storage.am.vector.utils.CrossPollinationConfig;
+import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+
+/**
+ * Factory for creating VTree instances used as disk components in LSM Vector Clustering Trees.
+ */
+public class VTreeFactory extends TreeIndexFactory<VTree> {
+
+ private final ITreeIndexFrameFactory metadataFrameFactory;
+ private final ITreeIndexFrameFactory dataFrameFactory;
+ private final int vectorDimensions;
+ private final IVTreeBinaryAccessorFactory vectorAccessorFactory;
+ private final IVTreeDataTupleBuilderFactory dataTupleBuilderFactory;
+ private final VTreeQuantizationParams quantizationParams;
+ private final IVTreeDistanceFunctionFactory distanceFunctionFactory;
+ private final CrossPollinationConfig crossPollination;
+
+ public VTreeFactory(IIOManager ioManager, IBufferCache bufferCache, IPageManagerFactory freePageManagerFactory,
+ ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory leafFrameFactory,
+ ITreeIndexFrameFactory metadataFrameFactory, ITreeIndexFrameFactory dataFrameFactory,
+ IBinaryComparatorFactory[] cmpFactories, int fieldCount, int vectorDimensions,
+ IVTreeBinaryAccessorFactory vectorAccessorFactory, IVTreeDataTupleBuilderFactory dataTupleBuilderFactory,
+ VTreeQuantizationParams quantizationParams, IVTreeDistanceFunctionFactory distanceFunctionFactory,
+ CrossPollinationConfig crossPollination) {
+ super(ioManager, bufferCache, freePageManagerFactory, interiorFrameFactory, leafFrameFactory, cmpFactories,
+ fieldCount);
+ this.metadataFrameFactory = metadataFrameFactory;
+ this.dataFrameFactory = dataFrameFactory;
+ this.vectorDimensions = vectorDimensions;
+ this.vectorAccessorFactory = vectorAccessorFactory;
+ this.dataTupleBuilderFactory = dataTupleBuilderFactory;
+ this.quantizationParams = quantizationParams;
+ this.distanceFunctionFactory = distanceFunctionFactory;
+ this.crossPollination = crossPollination;
+ }
+
+ @Override
+ public VTree createIndexInstance(FileReference file) throws HyracksDataException {
+ return new VTree(bufferCache, freePageManagerFactory.createPageManager(bufferCache), interiorFrameFactory,
+ leafFrameFactory, metadataFrameFactory, dataFrameFactory, cmpFactories, vectorDimensions,
+ vectorDimensions, file, vectorAccessorFactory, dataTupleBuilderFactory, quantizationParams,
+ distanceFunctionFactory, crossPollination);
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/VectorSearchHeapEntry.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/VectorSearchHeapEntry.java
new file mode 100644
index 0000000000..5f74c41074
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/VectorSearchHeapEntry.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.vector.impls;
+
+import org.apache.hyracks.dataflow.std.structures.IResetableComparable;
+import org.apache.hyracks.dataflow.std.structures.TuplePointer;
+
+/**
+ * Heap entry for the frame-backed top-K window in vector search cursors.
+ * Keyed by D(q, x) (approximate distance to query) — largest dqx at the root
+ * of a MaxHeap so the worst result can be evicted in O(1) peek / O(log n) replace.
+ */
+public class VectorSearchHeapEntry implements IResetableComparable<VectorSearchHeapEntry> {
+ final TuplePointer tuplePointer;
+ double dqx;
+
+ public VectorSearchHeapEntry() {
+ this.tuplePointer = new TuplePointer();
+ this.dqx = Double.MAX_VALUE;
+ }
+
+ @Override
+ public int compareTo(VectorSearchHeapEntry other) {
+ return Double.compare(this.dqx, other.dqx);
+ }
+
+ @Override
+ public void reset(VectorSearchHeapEntry other) {
+ this.tuplePointer.reset(other.tuplePointer);
+ this.dqx = other.dqx;
+ }
+
+ public void reset(TuplePointer pointer, double dqx) {
+ this.tuplePointer.reset(pointer);
+ this.dqx = dqx;
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/VectorSearchHeapEntryFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/VectorSearchHeapEntryFactory.java
new file mode 100644
index 0000000000..b99164512c
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/VectorSearchHeapEntryFactory.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.vector.impls;
+
+import org.apache.hyracks.dataflow.std.structures.IResetableComparable;
+import org.apache.hyracks.dataflow.std.structures.IResetableComparableFactory;
+
+public class VectorSearchHeapEntryFactory implements IResetableComparableFactory<VectorSearchHeapEntry> {
+ @Override
+ public IResetableComparable<VectorSearchHeapEntry> createResetableComparable() {
+ return new VectorSearchHeapEntry();
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/tuples/LSMVTreeCopyTupleWriter.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/tuples/LSMVTreeCopyTupleWriter.java
new file mode 100644
index 0000000000..9320616728
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/tuples/LSMVTreeCopyTupleWriter.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.lsm.vector.tuples;
+
+import static org.apache.hyracks.storage.am.lsm.common.api.ILSMTreeTupleReference.ANTIMATTER_BIT_OFFSET;
+
+import org.apache.hyracks.api.dataflow.value.ITypeTraits;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.storage.am.common.api.INullIntrospector;
+import org.apache.hyracks.storage.am.common.util.BitOperationUtils;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMTreeTupleReference;
+import org.apache.hyracks.util.annotations.AiProvenance;
+
+/**
+ * Tuple writer that preserves the antimatter bit of the SOURCE tuple (LSMBTreeCopyTupleWriter
+ * pattern). Used by merges with {@code returnDeletedTuples=true} (merging set excludes the oldest
+ * disk component): preserved antimatter tuples drained from the merge cursor must be re-written
+ * as antimatter, not silently re-encoded as matter. Matter tuples are written identically to
+ * {@link LSMVTreeDataTupleWriter}.
+ */
+@AiProvenance(agent = AiProvenance.Agent.CLAUDE_FABLE_5, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.GENERATED)
+public class LSMVTreeCopyTupleWriter extends LSMVTreeDataTupleWriter {
+
+ public LSMVTreeCopyTupleWriter(ITypeTraits[] typeTraits, ITypeTraits nullTypeTraits,
+ INullIntrospector nullIntrospector) {
+ super(typeTraits, nullTypeTraits, nullIntrospector);
+ }
+
+ @Override
+ public int writeTuple(ITupleReference tuple, byte[] targetBuf, int targetOff) {
+ // The copy writer's contract is to PRESERVE the source polarity during merge. A source tuple that
+ // is not an ILSMTreeTupleReference would silently be written as matter (delete resurrection) — the
+ // exact bug class this writer guards against — so assert the merge always feeds marker-aware tuples.
+ assert tuple instanceof ILSMTreeTupleReference : "copy writer given a non-ILSMTreeTupleReference "
+ + "source; delete-marker polarity cannot be preserved";
+ int bytesWritten = super.writeTuple(tuple, targetBuf, targetOff);
+ if (tuple instanceof ILSMTreeTupleReference && ((ILSMTreeTupleReference) tuple).isAntimatter()) {
+ BitOperationUtils.setBit(targetBuf, targetOff, ANTIMATTER_BIT_OFFSET);
+ }
+ return bytesWritten;
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/tuples/LSMVTreeCopyTupleWriterFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/tuples/LSMVTreeCopyTupleWriterFactory.java
new file mode 100644
index 0000000000..5696d1611b
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/tuples/LSMVTreeCopyTupleWriterFactory.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.lsm.vector.tuples;
+
+import org.apache.hyracks.api.dataflow.value.ITypeTraits;
+import org.apache.hyracks.storage.am.common.api.INullIntrospector;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
+import org.apache.hyracks.util.annotations.AiProvenance;
+
+/**
+ * Factory for {@link LSMVTreeCopyTupleWriter} instances — merge-path data-frame writers that
+ * carry over the source tuple's antimatter bit.
+ */
+@AiProvenance(agent = AiProvenance.Agent.CLAUDE_FABLE_5, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.GENERATED)
+public class LSMVTreeCopyTupleWriterFactory extends LSMVTreeDataTupleWriterFactory {
+
+ private static final long serialVersionUID = 1L;
+
+ private final ITypeTraits[] typeTraits;
+ private final ITypeTraits nullTypeTraits;
+ private final INullIntrospector nullIntrospector;
+
+ public LSMVTreeCopyTupleWriterFactory(ITypeTraits[] typeTraits, ITypeTraits nullTypeTraits,
+ INullIntrospector nullIntrospector) {
+ super(typeTraits, false, nullTypeTraits, nullIntrospector);
+ this.typeTraits = typeTraits;
+ this.nullTypeTraits = nullTypeTraits;
+ this.nullIntrospector = nullIntrospector;
+ }
+
+ @Override
+ public ITreeIndexTupleWriter createTupleWriter() {
+ return new LSMVTreeCopyTupleWriter(typeTraits, nullTypeTraits, nullIntrospector);
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/tuples/LSMVTreeDataTupleReference.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/tuples/LSMVTreeDataTupleReference.java
new file mode 100644
index 0000000000..788e4a62c2
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/tuples/LSMVTreeDataTupleReference.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.lsm.vector.tuples;
+
+import org.apache.hyracks.api.dataflow.value.ITypeTraits;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame;
+import org.apache.hyracks.storage.am.common.tuples.TypeAwareTupleReference;
+import org.apache.hyracks.storage.am.common.util.BitOperationUtils;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMTreeTupleReference;
+
+/**
+ * Tuple reference for LSM VTree data frames.
+ *
+ * On-disk layout: [null/antimatter flag bytes][field slots][fields...]
+ * Field 0 of the null-flag byte is reserved for the antimatter bit (see
+ * {@link ILSMTreeTupleReference#ANTIMATTER_BIT_OFFSET}); user fields start at logical index 1
+ * via {@link #getAdjustedFieldIdx(int)}.
+ */
+public class LSMVTreeDataTupleReference extends TypeAwareTupleReference implements ILSMTreeTupleReference {
+
+ public LSMVTreeDataTupleReference(ITypeTraits[] typeTraits, ITypeTraits nullTypeTraits) {
+ super(typeTraits, nullTypeTraits);
+ }
+
+ @Override
+ public void resetByTupleIndex(ITreeIndexFrame frame, int tupleIndex) {
+ resetByTupleOffset(frame.getBuffer().array(), frame.getTupleOffset(tupleIndex));
+ }
+
+ @Override
+ protected int getNullFlagsBytes() {
+ // user fields + 1 antimatter bit
+ int numBits = fieldCount + 1;
+ return BitOperationUtils.getFlagBytes(numBits);
+ }
+
+ @Override
+ public boolean isAntimatter() {
+ return BitOperationUtils.getBit(buf, tupleStartOff, ILSMTreeTupleReference.ANTIMATTER_BIT_OFFSET);
+ }
+
+ public int getTupleStart() {
+ return tupleStartOff;
+ }
+
+ @Override
+ protected int getAdjustedFieldIdx(int fieldIdx) {
+ // skip the antimatter bit at index 0
+ return fieldIdx + 1;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/tuples/LSMVTreeDataTupleWriter.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/tuples/LSMVTreeDataTupleWriter.java
new file mode 100644
index 0000000000..c8a6431b58
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/tuples/LSMVTreeDataTupleWriter.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.lsm.vector.tuples;
+
+import static org.apache.hyracks.storage.am.lsm.common.api.ILSMTreeTupleReference.ANTIMATTER_BIT_OFFSET;
+
+import org.apache.hyracks.api.dataflow.value.ITypeTraits;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.storage.am.common.api.INullIntrospector;
+import org.apache.hyracks.storage.am.common.tuples.TypeAwareTupleWriter;
+import org.apache.hyracks.storage.am.common.util.BitOperationUtils;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMTreeTupleWriter;
+
+/**
+ * Tuple writer for LSM VTree data frames.
+ *
+ * On-disk layout produced: [null/antimatter flag bytes][field slots][fields...].
+ * Bit 7 (the MSB, {@link org.apache.hyracks.storage.am.lsm.common.api.ILSMTreeTupleReference#ANTIMATTER_BIT_OFFSET})
+ * of null-flag byte 0 is reserved for the antimatter (deletion) marker; user fields occupy the
+ * remaining bits. Matter and anti-matter tuples share the same physical layout and size.
+ */
+public class LSMVTreeDataTupleWriter extends TypeAwareTupleWriter implements ILSMTreeTupleWriter {
+
+ private boolean isAntimatter;
+
+ public LSMVTreeDataTupleWriter(ITypeTraits[] typeTraits, ITypeTraits nullTypeTraits,
+ INullIntrospector nullIntrospector) {
+ super(typeTraits, nullTypeTraits, nullIntrospector);
+ this.isAntimatter = false;
+ }
+
+ @Override
+ public int getCopySpaceRequired(ITupleReference tuple) {
+ return super.bytesRequired(tuple);
+ }
+
+ @Override
+ public LSMVTreeDataTupleReference createTupleReference() {
+ return new LSMVTreeDataTupleReference(typeTraits, nullTypeTraits);
+ }
+
+ @Override
+ protected int getNullFlagsBytes(int numFields) {
+ // user fields + 1 antimatter bit
+ int numBits = numFields + 1;
+ return BitOperationUtils.getFlagBytes(numBits);
+ }
+
+ @Override
+ protected int getNullFlagsBytes(ITupleReference tuple) {
+ // user fields + 1 antimatter bit
+ int numBits = tuple.getFieldCount() + 1;
+ return BitOperationUtils.getFlagBytes(numBits);
+ }
+
+ @Override
+ public int writeTuple(ITupleReference tuple, byte[] targetBuf, int targetOff) {
+ // The null-flag byte count is derived from the field count here (getNullFlagsBytes(tuple)) but from
+ // typeTraits.length on the read side (LSMVTreeDataTupleReference). A mismatch shifts every field
+ // offset and mis-locates the antimatter bit — silent corruption, not a crash. Guard the invariant.
+ assert tuple.getFieldCount() == typeTraits.length : "tuple field count " + tuple.getFieldCount()
+ + " != configured typeTraits length " + typeTraits.length;
+ int bytesWritten = super.writeTuple(tuple, targetBuf, targetOff);
+ if (isAntimatter) {
+ BitOperationUtils.setBit(targetBuf, targetOff, ANTIMATTER_BIT_OFFSET);
+ }
+ return bytesWritten;
+ }
+
+ @Override
+ public void setAntimatter(boolean isDelete) {
+ this.isAntimatter = isDelete;
+ }
+
+ @Override
+ protected int getAdjustedFieldIdx(int fieldIdx) {
+ // skip the antimatter bit at index 0
+ return fieldIdx + 1;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/tuples/LSMVTreeDataTupleWriterFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/tuples/LSMVTreeDataTupleWriterFactory.java
new file mode 100644
index 0000000000..fb38755ca5
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/tuples/LSMVTreeDataTupleWriterFactory.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.lsm.vector.tuples;
+
+import org.apache.hyracks.api.dataflow.value.ITypeTraits;
+import org.apache.hyracks.storage.am.common.api.INullIntrospector;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriterFactory;
+import org.apache.hyracks.util.annotations.AiProvenance;
+
+/**
+ * Factory for creating LSMVTreeDataTupleWriter instances.
+ *
+ * Used to create tuple writers for vector index data frames with anti-matter support.
+ */
+public class LSMVTreeDataTupleWriterFactory implements ITreeIndexTupleWriterFactory {
+
+ private static final long serialVersionUID = 1L;
+
+ private final ITypeTraits[] typeTraits;
+ private final ITypeTraits nullTypeTraits;
+ private final INullIntrospector nullIntrospector;
+ private final boolean isAntimatter;
+
+ public LSMVTreeDataTupleWriterFactory(ITypeTraits[] typeTraits, boolean isAntimatter, ITypeTraits nullTypeTraits,
+ INullIntrospector nullIntrospector) {
+ this.typeTraits = typeTraits;
+ this.isAntimatter = isAntimatter;
+ this.nullTypeTraits = nullTypeTraits;
+ this.nullIntrospector = nullIntrospector;
+ }
+
+ @Override
+ public ITreeIndexTupleWriter createTupleWriter() {
+ LSMVTreeDataTupleWriter writer = new LSMVTreeDataTupleWriter(typeTraits, nullTypeTraits, nullIntrospector);
+ writer.setAntimatter(isAntimatter);
+ return writer;
+ }
+
+ /**
+ * Derives a merge-path copy-writer factory over the same type traits: its writers preserve
+ * the SOURCE tuple's antimatter bit instead of imposing this factory's fixed matter/antimatter
+ * flag. Required when a merge preserves antimatter tuples (merging set excludes the oldest
+ * disk component).
+ */
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_FABLE_5, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.ASSISTED)
+ public LSMVTreeCopyTupleWriterFactory createCopyWriterFactory() {
+ return new LSMVTreeCopyTupleWriterFactory(typeTraits, nullTypeTraits, nullIntrospector);
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/utils/LSMVTreeUtils.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/utils/LSMVTreeUtils.java
new file mode 100644
index 0000000000..88b9046f5e
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/utils/LSMVTreeUtils.java
@@ -0,0 +1,175 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.lsm.vector.utils;
+
+import java.util.List;
+
+import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import org.apache.hyracks.api.dataflow.value.ITypeTraits;
+import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.IIOManager;
+import org.apache.hyracks.control.common.controllers.NCConfig;
+import org.apache.hyracks.storage.am.common.api.IMetadataPageManagerFactory;
+import org.apache.hyracks.storage.am.common.api.INullIntrospector;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import org.apache.hyracks.storage.am.lsm.common.api.IComponentFilterHelper;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFrameFactory;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentFactory;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackFactory;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMPageWriteCallbackFactory;
+import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
+import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFilterManager;
+import org.apache.hyracks.storage.am.lsm.vector.impls.LSMVTree;
+import org.apache.hyracks.storage.am.lsm.vector.impls.LSMVTreeDiskComponentFactory;
+import org.apache.hyracks.storage.am.lsm.vector.impls.LSMVTreeFileManager;
+import org.apache.hyracks.storage.am.lsm.vector.impls.VTreeFactory;
+import org.apache.hyracks.storage.am.lsm.vector.tuples.LSMVTreeDataTupleWriterFactory;
+import org.apache.hyracks.storage.am.vector.api.IVTreeBinaryAccessorFactory;
+import org.apache.hyracks.storage.am.vector.api.IVTreeDataTupleBuilderFactory;
+import org.apache.hyracks.storage.am.vector.api.IVTreeDistanceFunctionFactory;
+import org.apache.hyracks.storage.am.vector.api.VTreeQuantizationParams;
+import org.apache.hyracks.storage.am.vector.frames.VTreeDataFrameFactory;
+import org.apache.hyracks.storage.am.vector.frames.VTreeInteriorFrameFactory;
+import org.apache.hyracks.storage.am.vector.frames.VTreeLeafFrameFactory;
+import org.apache.hyracks.storage.am.vector.frames.VTreeMetadataFrameFactory;
+import org.apache.hyracks.storage.am.vector.utils.CrossPollinationConfig;
+import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+
+/**
+ * Factory helper that wires the four frame factories, the {@link VTreeFactory}, and the LSM glue
+ * (file manager, disk-component factory) into a configured {@link LSMVTree}.
+ */
+public final class LSMVTreeUtils {
+
+ /**
+ * Field count threaded into {@link VTreeFactory}'s {@code AbstractTreeIndex} base. Matches the
+ * production (quantized) leaf-tuple field count {@code <cid, centroid, qEmbed, pointer>}.
+ */
+ private static final int TREE_INDEX_FIELD_COUNT = 4;
+
+ private LSMVTreeUtils() {
+ }
+
+ /**
+ * Build a configured {@link LSMVTree} from the per-resource configuration. Constructs the four
+ * frame factories (interior/leaf/metadata pick up their fixed schemas internally; data takes
+ * the caller-supplied {@code typeTraits}), the matter/antimatter
+ * {@link LSMVTreeDataTupleWriterFactory} pair, and the {@link VTreeFactory} used for disk
+ * components, then hands them to {@link LSMVTree}'s constructor.
+ *
+ * @param quantizationParams {@code float[6]} sample-file params (see
+ * {@code IVTreeQuantizerFactory#createQuantizer}) or {@code null} for
+ * the non-quantized test-fixture path; controls whether the leaf
+ * frame uses the 4-field quantized layout or the 3-field bare layout.
+ */
+ public static LSMVTree createLSMTree(NCConfig storageConfig, IIOManager ioManager,
+ List<IVirtualBufferCache> virtualBufferCaches, FileReference file, IBufferCache diskBufferCache,
+ ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories, double bloomFilterFalsePositiveRate,
+ ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
+ ILSMIOOperationCallbackFactory ioOpCallbackFactory, ILSMPageWriteCallbackFactory pageWriteCallbackFactory,
+ int vectorDimensions, int[] vectorFields, int[] filterFields,
+ ILSMComponentFilterFrameFactory filterFrameFactory, LSMComponentFilterManager filterManager,
+ IComponentFilterHelper filterHelper, boolean durable,
+ IMetadataPageManagerFactory metadataPageManagerFactory, boolean atomic, RecordDescriptor inputRecDesc,
+ IVTreeBinaryAccessorFactory vectorAccessorFactory, int numPrimaryKeyFields, int numIncludeFields,
+ IVTreeDataTupleBuilderFactory dataTupleBuilderFactory, VTreeQuantizationParams quantizationParams,
+ IVTreeDistanceFunctionFactory distanceFunctionFactory) throws HyracksDataException {
+ // Legacy single-closest placement; production overload below threads the real config.
+ return createLSMTree(storageConfig, ioManager, virtualBufferCaches, file, diskBufferCache, typeTraits,
+ cmpFactories, bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallbackFactory,
+ pageWriteCallbackFactory, vectorDimensions, vectorFields, filterFields, filterFrameFactory,
+ filterManager, filterHelper, durable, metadataPageManagerFactory, atomic, inputRecDesc,
+ vectorAccessorFactory, numPrimaryKeyFields, numIncludeFields, dataTupleBuilderFactory,
+ quantizationParams, distanceFunctionFactory, CrossPollinationConfig.LEGACY);
+ }
+
+ public static LSMVTree createLSMTree(NCConfig storageConfig, IIOManager ioManager,
+ List<IVirtualBufferCache> virtualBufferCaches, FileReference file, IBufferCache diskBufferCache,
+ ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories, double bloomFilterFalsePositiveRate,
+ ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
+ ILSMIOOperationCallbackFactory ioOpCallbackFactory, ILSMPageWriteCallbackFactory pageWriteCallbackFactory,
+ int vectorDimensions, int[] vectorFields, int[] filterFields,
+ ILSMComponentFilterFrameFactory filterFrameFactory, LSMComponentFilterManager filterManager,
+ IComponentFilterHelper filterHelper, boolean durable,
+ IMetadataPageManagerFactory metadataPageManagerFactory, boolean atomic, RecordDescriptor inputRecDesc,
+ IVTreeBinaryAccessorFactory vectorAccessorFactory, int numPrimaryKeyFields, int numIncludeFields,
+ IVTreeDataTupleBuilderFactory dataTupleBuilderFactory, VTreeQuantizationParams quantizationParams,
+ IVTreeDistanceFunctionFactory distanceFunctionFactory, CrossPollinationConfig crossPollination)
+ throws HyracksDataException {
+
+ // VTree tuples contain no field that is both fixed-length AND nullable, so the null bitmap is never
+ // load-bearing and no INullIntrospector is needed (unlike BTree/RTree secondary keys):
+ // - the interior/leaf/metadata frames have fixed internal schemas whose fields are never null;
+ // - data-row include fields that can be NULL/MISSING are always ADM-tagged variable-length (nullable
+ // or optional types resolve to a UNION/ANY var-len trait), so a null self-encodes as a 1-byte
+ // NULL/MISSING tag in the value itself and the reader never consults the bitmap for them;
+ // - fixed-length data fields (distance/centroidId/quantized/pk) are always present and non-null.
+ ITypeTraits nullTypeTraits = null;
+ INullIntrospector nullIntrospector = null;
+ boolean quantized = quantizationParams != null;
+
+ // Interior, leaf, and metadata frames have fixed tuple schemas that are intrinsic to the
+ // VTree design (not user-parameterized). Each factory owns its own schema and builds its
+ // tuple writer internally — see the corresponding *FrameFactory class for the layout.
+ ITreeIndexFrameFactory interiorFrameFactory =
+ new VTreeInteriorFrameFactory(vectorDimensions, nullTypeTraits, nullIntrospector);
+ ITreeIndexFrameFactory leafFrameFactory =
+ new VTreeLeafFrameFactory(vectorDimensions, quantized, nullTypeTraits, nullIntrospector);
+ ITreeIndexFrameFactory metadataFrameFactory =
+ new VTreeMetadataFrameFactory(vectorDimensions, nullTypeTraits, nullIntrospector);
+
+ // Data frames are caller-parameterized: typeTraits carries the ADM-tagged type traits for
+ // all data-row fields (computed by VTreeResourceFactoryProvider in production, by the test
+ // harness for fixtures). Production quantized layout:
+ // [distance, qDist, qEmbed, centroidId, pk, include_fields...]
+ // INSERT operations use matter tuples, DELETE operations use antimatter tuples (LSMBTree
+ // pattern). Disk components are immutable and only ever take insert tuples.
+ LSMVTreeDataTupleWriterFactory insertDataTupleWriterFactory =
+ new LSMVTreeDataTupleWriterFactory(typeTraits, false, nullTypeTraits, nullIntrospector);
+ LSMVTreeDataTupleWriterFactory deleteDataTupleWriterFactory =
+ new LSMVTreeDataTupleWriterFactory(typeTraits, true, nullTypeTraits, nullIntrospector);
+ ITreeIndexFrameFactory insertDataFrameFactory =
+ new VTreeDataFrameFactory(insertDataTupleWriterFactory, vectorDimensions);
+ ITreeIndexFrameFactory deleteDataFrameFactory =
+ new VTreeDataFrameFactory(deleteDataTupleWriterFactory, vectorDimensions);
+
+ VTreeFactory vtreeFactory = new VTreeFactory(ioManager, diskBufferCache, metadataPageManagerFactory,
+ interiorFrameFactory, leafFrameFactory, metadataFrameFactory, insertDataFrameFactory, cmpFactories,
+ TREE_INDEX_FIELD_COUNT, vectorDimensions, vectorAccessorFactory, dataTupleBuilderFactory,
+ quantizationParams, distanceFunctionFactory, crossPollination);
+ ILSMIndexFileManager fileManager = new LSMVTreeFileManager(ioManager, file, vtreeFactory);
+ ILSMDiskComponentFactory componentFactory = new LSMVTreeDiskComponentFactory(vtreeFactory, filterHelper);
+
+ return new LSMVTree(storageConfig, ioManager, virtualBufferCaches, interiorFrameFactory, leafFrameFactory,
+ metadataFrameFactory, insertDataFrameFactory, deleteDataFrameFactory, diskBufferCache, fileManager,
+ componentFactory, componentFactory, filterHelper, filterFrameFactory, filterManager,
+ bloomFilterFalsePositiveRate, cmpFactories, mergePolicy, opTracker, ioScheduler, ioOpCallbackFactory,
+ pageWriteCallbackFactory, vectorDimensions, vectorFields, filterFields, durable, atomic,
+ vectorAccessorFactory, numPrimaryKeyFields, numIncludeFields, dataTupleBuilderFactory,
+ quantizationParams, distanceFunctionFactory, crossPollination);
+ }
+
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/api/IQuantizedResource.java b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/api/IQuantizedResource.java
new file mode 100644
index 0000000000..e8da3b8a14
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/api/IQuantizedResource.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.vector.api;
+
+/**
+ * Implemented by a local resource that carries scalar-quantization parameters, so the quantized index
+ * builder can inject them into the freshly created resource at build time.
+ */
+public interface IQuantizedResource {
+ void setQuantizationParameters(VTreeQuantizationParams parameters);
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/IResourceWrapper.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/IResourceWrapper.java
new file mode 100644
index 0000000000..bfad6427f5
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/IResourceWrapper.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.common;
+
+/**
+ * A resource that delegates to another {@link IResource}. Lets a caller reach the wrapped resource without
+ * depending on the concrete wrapper type, so index-type-specific capabilities (for example quantization) can
+ * be probed on the inner resource from a lower storage layer.
+ */
+public interface IResourceWrapper {
+
+ /**
+ * @return the wrapped resource this instance delegates to.
+ */
+ IResource getResource();
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-vtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/vector/LSMVTreeFileManagerCrashRecoveryTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-vtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/vector/LSMVTreeFileManagerCrashRecoveryTest.java
new file mode 100644
index 0000000000..ef08c41ce2
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-vtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/vector/LSMVTreeFileManagerCrashRecoveryTest.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.lsm.vector;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.util.List;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
+import org.apache.hyracks.storage.am.lsm.vector.impls.LSMVTreeFileManager;
+import org.apache.hyracks.storage.am.lsm.vector.util.LSMVTreeTestHarness;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Recovery test for {@link LSMVTreeFileManager#cleanupAndGetValidFiles()} (ASTERIXDB-3754).
+ *
+ * <p>A merge writes the merged component and then deletes its inputs as a <em>separate</em> step. A crash
+ * in between leaves the pre-merge inputs on disk alongside the merged component. On the next
+ * activation, {@code cleanupAndGetValidFiles()} must return only the merged component and delete the
+ * superseded inputs — otherwise the index would load both, duplicating every record and resurfacing
+ * deletes the merge reconciled away.
+ *
+ * <p>This is a file-manager-level test: it stages the exact post-crash on-disk file set (which is hard to
+ * produce deterministically end-to-end, since a real merge deletes the inputs) and asserts the cleanup
+ * reconciles it. Pre-fix the override returned all four components and deleted nothing.
+ */
+public class LSMVTreeFileManagerCrashRecoveryTest {
+
+ private static final String VCT = "_vct";
+ private static final String STATIC_STRUCTURE = ".staticstructure";
+
+ private final LSMVTreeTestHarness harness = new LSMVTreeTestHarness();
+
+ @Before
+ public void setUp() throws HyracksDataException {
+ harness.setUp();
+ }
+
+ @After
+ public void tearDown() throws HyracksDataException {
+ harness.tearDown();
+ }
+
+ @Test
+ public void crashAfterMergeDropsSupersededComponents() throws Exception {
+ FileReference baseDir = harness.getFileReference();
+ baseDir.getFile().mkdirs();
+
+ // Post-crash on-disk state: the merged component [0_2] PLUS its un-deleted pre-merge inputs
+ // [0_0], [1_1], [2_2], and the shared static structure.
+ create(baseDir, "0_0" + VCT);
+ create(baseDir, "1_1" + VCT);
+ create(baseDir, "2_2" + VCT);
+ create(baseDir, "0_2" + VCT); // merged component (range 0..2)
+ create(baseDir, STATIC_STRUCTURE);
+
+ LSMVTreeFileManager fileManager = new LSMVTreeFileManager(harness.getIOManager(), baseDir, null);
+ List<LSMComponentFileReferences> valid = fileManager.cleanupAndGetValidFiles();
+
+ // Only the merged component is valid.
+ assertEquals("only the merged component should survive", 1, valid.size());
+ assertTrue("survivor should be the merged [0_2] component",
+ valid.get(0).getInsertIndexFileReference().getFile().getName().startsWith("0_2"));
+
+ // The three superseded pre-merge inputs must have been deleted...
+ assertFalse("pre-merge input 0_0 must be deleted", exists(baseDir, "0_0" + VCT));
+ assertFalse("pre-merge input 1_1 must be deleted", exists(baseDir, "1_1" + VCT));
+ assertFalse("pre-merge input 2_2 must be deleted", exists(baseDir, "2_2" + VCT));
+ // ...while the merged component and the shared static structure are kept.
+ assertTrue("merged component 0_2 must be kept", exists(baseDir, "0_2" + VCT));
+ assertTrue("shared static structure must be kept", exists(baseDir, STATIC_STRUCTURE));
+ }
+
+ /** Sanity: with no overlap (plain flushes), all components survive, newest-first. */
+ @Test
+ public void nonOverlappingComponentsAllSurvive() throws Exception {
+ FileReference baseDir = harness.getFileReference();
+ baseDir.getFile().mkdirs();
+ create(baseDir, "0_0" + VCT);
+ create(baseDir, "1_1" + VCT);
+ create(baseDir, "2_2" + VCT);
+ create(baseDir, STATIC_STRUCTURE);
+
+ LSMVTreeFileManager fileManager = new LSMVTreeFileManager(harness.getIOManager(), baseDir, null);
+ List<LSMComponentFileReferences> valid = fileManager.cleanupAndGetValidFiles();
+
+ assertEquals("all three flushed components survive", 3, valid.size());
+ // LSM expects newest -> oldest.
+ assertTrue(valid.get(0).getInsertIndexFileReference().getFile().getName().startsWith("2_2"));
+ assertTrue(valid.get(2).getInsertIndexFileReference().getFile().getName().startsWith("0_0"));
+ assertTrue(exists(baseDir, "0_0" + VCT));
+ assertTrue(exists(baseDir, "2_2" + VCT));
+ }
+
+ private static void create(FileReference baseDir, String name) throws Exception {
+ File f = baseDir.getChild(name).getFile();
+ f.getParentFile().mkdirs();
+ assertTrue("failed to stage " + name, f.createNewFile() || f.exists());
+ }
+
+ private static boolean exists(FileReference baseDir, String name) {
+ return baseDir.getChild(name).getFile().exists();
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-vtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/vector/LSMVTreeMultiVersionReconciliationTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-vtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/vector/LSMVTreeMultiVersionReconciliationTest.java
new file mode 100644
index 0000000000..45e5eaeedf
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-vtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/vector/LSMVTreeMultiVersionReconciliationTest.java
@@ -0,0 +1,240 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.lsm.vector;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleReference;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.dataflow.common.data.marshalling.DoubleArraySerializerDeserializer;
+import org.apache.hyracks.dataflow.common.data.marshalling.DoubleSerializerDeserializer;
+import org.apache.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import org.apache.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
+import org.apache.hyracks.dataflow.common.utils.TupleUtils;
+import org.apache.hyracks.storage.am.common.TestOperationCallback;
+import org.apache.hyracks.storage.am.common.impls.IndexAccessParameters;
+import org.apache.hyracks.storage.am.common.impls.NoOpIndexAccessParameters;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation.LSMIOOperationStatus;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
+import org.apache.hyracks.storage.am.lsm.vector.impls.LSMVTree;
+import org.apache.hyracks.storage.am.lsm.vector.util.LSMVTreeTestContext;
+import org.apache.hyracks.storage.am.lsm.vector.util.LSMVTreeTestHarness;
+import org.apache.hyracks.storage.am.lsm.vector.util.VectorTestStructure;
+import org.apache.hyracks.storage.am.vector.AbstractVectorTreeTestContext;
+import org.apache.hyracks.storage.am.vector.TestDoubleArrayVectorAccessor;
+import org.apache.hyracks.storage.am.vector.VectorTreeTestUtils;
+import org.apache.hyracks.storage.am.vector.api.IVTreeBinaryAccessorFactory;
+import org.apache.hyracks.storage.am.vector.impls.VTreeSearchPredicate;
+import org.apache.hyracks.storage.common.IIndexAccessor;
+import org.apache.hyracks.storage.common.IIndexCursor;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Regression for multi-version antimatter reconciliation in {@code LSMVTreeSearchCursor} across
+ * >=3 components that share one {@code <distance, PK>} key (ASTERIXDB-3754).
+ *
+ * <p>The streaming cursor previously reconciled matter/antimatter <em>pairwise</em> — it cancelled a
+ * single pair and stopped, so a 3rd same-key version leaked. Concretely, re-inserting the same vector
+ * after a delete produced three components for one key (matter / antimatter / matter): the query
+ * returned the PK twice and resurfaced the deleted version.
+ *
+ * <p>Correct behaviour: the newest version wins and every older same-key version is drained — a live
+ * key appears exactly once, a net-deleted key not at all.
+ */
+public class LSMVTreeMultiVersionReconciliationTest {
+
+ private static final Logger LOGGER = LogManager.getLogger();
+
+ private final LSMVTreeTestHarness harness = new LSMVTreeTestHarness();
+ private final VectorTreeTestUtils testUtils = new VectorTreeTestUtils();
+
+ private static final int BULK_RECORDS_PER_CLUSTER = 50;
+
+ // Victim: pk_c_10_0 is the first bulk-loaded record of leaf centroid c10 {20,30,20}
+ // (ring distance 0.2, +x direction). Re-inserting the SAME vector keeps the same <distance, PK>.
+ private static final String VICTIM_PK = "pk_c_10_0";
+ private static final double[] VICTIM_VECTOR = { 20.2, 30.0, 20.0 };
+ private static final double[] QUERY_VECTOR = { 20.0, 30.0, 20.0 };
+ private static final int QUERY_K = 500;
+
+ @Before
+ public void setUp() throws HyracksDataException {
+ harness.setUp();
+ }
+
+ @After
+ public void tearDown() throws HyracksDataException {
+ harness.tearDown();
+ }
+
+ /** insert -> delete -> re-insert (same vector), each in its own disk component: key is LIVE, must appear once. */
+ @Test
+ public void reinsertAfterDeleteAppearsExactlyOnce() throws Exception {
+ AbstractVectorTreeTestContext ctx = newContext();
+ LSMVTree lsmvTree = (LSMVTree) ctx.getIndex();
+ try {
+ ctx.getIndex().create();
+ ctx.getIndex().activate();
+ testUtils.buildStaticStructure(ctx);
+
+ // C(oldest): bulk load holds the victim's original matter.
+ testUtils.bulkLoadRecords(ctx);
+ // C(middle): delete the victim -> antimatter, flush.
+ testUtils.deleteRecordsFromIndex(ctx, List.of(createVectorPkTuple(VICTIM_VECTOR, VICTIM_PK)));
+ flush(ctx);
+ // C(newest): re-insert the same vector -> matter, flush.
+ testUtils.insertRecordsIntoMemoryComponent(ctx,
+ List.of(List.of(createVectorPkTuple(VICTIM_VECTOR, VICTIM_PK))));
+ flush(ctx);
+ assertTrue(">= 3 disk components expected for the multi-version case, got "
+ + lsmvTree.getDiskComponents().size(), lsmvTree.getDiskComponents().size() >= 3);
+
+ long occurrences = searchPKs(ctx).stream().filter(VICTIM_PK::equals).count();
+ // Pre-fix: 2 (the re-insert plus the resurrected original). Correct: exactly 1.
+ assertEquals("re-inserted key must appear exactly once (no duplicate / resurrected version)", 1L,
+ occurrences);
+ } finally {
+ ctx.getIndex().deactivate();
+ }
+ }
+
+ /** insert -> delete -> re-insert -> delete: net-deleted, must be absent. */
+ @Test
+ public void netDeletedAcrossFourVersionsIsAbsent() throws Exception {
+ AbstractVectorTreeTestContext ctx = newContext();
+ try {
+ ctx.getIndex().create();
+ ctx.getIndex().activate();
+ testUtils.buildStaticStructure(ctx);
+
+ testUtils.bulkLoadRecords(ctx); // matter (oldest)
+ testUtils.deleteRecordsFromIndex(ctx, List.of(createVectorPkTuple(VICTIM_VECTOR, VICTIM_PK)));
+ flush(ctx); // antimatter
+ testUtils.insertRecordsIntoMemoryComponent(ctx,
+ List.of(List.of(createVectorPkTuple(VICTIM_VECTOR, VICTIM_PK))));
+ flush(ctx); // matter (re-insert)
+ testUtils.deleteRecordsFromIndex(ctx, List.of(createVectorPkTuple(VICTIM_VECTOR, VICTIM_PK)));
+ flush(ctx); // antimatter (newest)
+
+ long occurrences = searchPKs(ctx).stream().filter(VICTIM_PK::equals).count();
+ assertEquals("net-deleted key must be absent", 0L, occurrences);
+ } finally {
+ ctx.getIndex().deactivate();
+ }
+ }
+
+ private AbstractVectorTreeTestContext newContext() throws Exception {
+ VectorTestStructure struct = VectorTestStructure.threeDim3Level();
+ AbstractVectorTreeTestContext ctx = LSMVTreeTestContext.create(harness.getNcConfig(), harness.getIOManager(),
+ harness.getVirtualBufferCaches(), harness.getFileReference(), harness.getDiskBufferCache(),
+ struct.getDataRecordSerdes(VectorTestStructure.BulkLoadRecordFormat.NAIVE), struct.getVectorDimension(),
+ harness.getMergePolicy(), harness.getOperationTracker(), harness.getIOScheduler(),
+ harness.getIOOperationCallbackFactory(), harness.getPageWriteCallbackFactory(),
+ harness.getMetadataPageManagerFactory());
+ ctx.setStaticStructureCentroids(struct.buildCentroidTuples());
+ ctx.setNumClustersPerLevel(struct.getNumClustersPerLevel());
+ ctx.setNumCentroidsPerLevel(struct.getCentroidsPerCluster());
+ ctx.setDataRecords(struct.generateBulkLoadRecords(VectorTestStructure.BulkLoadRecordFormat.NAIVE,
+ BULK_RECORDS_PER_CLUSTER));
+ return ctx;
+ }
+
+ private void flush(AbstractVectorTreeTestContext ctx) throws HyracksDataException, InterruptedException {
+ ILSMIndexAccessor accessor =
+ (ILSMIndexAccessor) ctx.getIndex().createAccessor(NoOpIndexAccessParameters.INSTANCE);
+ ILSMIOOperation flushOp = accessor.scheduleFlush();
+ flushOp.sync();
+ if (flushOp.getStatus() == LSMIOOperationStatus.FAILURE) {
+ throw HyracksDataException.create(flushOp.getFailure());
+ }
+ }
+
+ private List<String> searchPKs(AbstractVectorTreeTestContext ctx) throws Exception {
+ ArrayTupleBuilder queryTupleBuilder = new ArrayTupleBuilder(1);
+ queryTupleBuilder.addField(DoubleArraySerializerDeserializer.INSTANCE, QUERY_VECTOR);
+ ArrayTupleReference queryTuple = new ArrayTupleReference();
+ queryTuple.reset(queryTupleBuilder.getFieldEndOffsets(), queryTupleBuilder.getByteArray());
+
+ VTreeSearchPredicate predicate = new VTreeSearchPredicate();
+ predicate.setMinProbeFraction(0.4);
+ predicate.setQueryTuple(queryTuple);
+ predicate.setQueryFieldIndex(0);
+ predicate.setK(QUERY_K);
+
+ IndexAccessParameters iap =
+ new IndexAccessParameters(TestOperationCallback.INSTANCE, TestOperationCallback.INSTANCE);
+ iap.getParameters().put(IVTreeBinaryAccessorFactory.IAP_KEY, TestDoubleArrayVectorAccessor.Factory.INSTANCE);
+
+ IIndexAccessor accessor = ctx.getIndex().createAccessor(iap);
+ IIndexCursor cursor = accessor.createSearchCursor(false);
+ List<String> foundPKs = new ArrayList<>();
+ try {
+ accessor.search(cursor, predicate);
+ try {
+ while (cursor.hasNext()) {
+ cursor.next();
+ foundPKs.add(extractPrimaryKey(cursor.getTuple()));
+ }
+ } finally {
+ cursor.close();
+ }
+ } finally {
+ cursor.destroy();
+ }
+ LOGGER.info("Search returned {} PKs", foundPKs.size());
+ return foundPKs;
+ }
+
+ /** Insert/delete tuple format is identical: {@code <vector, pk>}. */
+ private ITupleReference createVectorPkTuple(double[] vector, String primaryKey) throws HyracksDataException {
+ try {
+ ArrayTupleBuilder tupleBuilder = new ArrayTupleBuilder(2);
+ ArrayTupleReference tupleRef = new ArrayTupleReference();
+ DoubleArraySerializerDeserializer.INSTANCE.serialize(vector, tupleBuilder.getDataOutput());
+ tupleBuilder.addFieldEndOffset();
+ new UTF8StringSerializerDeserializer().serialize(primaryKey, tupleBuilder.getDataOutput());
+ tupleBuilder.addFieldEndOffset();
+ tupleRef.reset(tupleBuilder.getFieldEndOffsets(), tupleBuilder.getByteArray());
+ return tupleRef;
+ } catch (Exception e) {
+ throw HyracksDataException.create(e);
+ }
+ }
+
+ /** Naive result tuple format: {@code <distance, centroid_id, primary_key>}; PK at field 2. */
+ private String extractPrimaryKey(ITupleReference tuple) throws HyracksDataException {
+ ISerializerDeserializer[] fieldSerdes = { DoubleSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE, new UTF8StringSerializerDeserializer() };
+ Object[] values = TupleUtils.deserializeTuple(tuple, fieldSerdes);
+ return (String) values[2];
+ }
+}
diff --git a/hyracks-fullstack/hyracks/pom.xml b/hyracks-fullstack/hyracks/pom.xml
index 0d1830d1c2..5d3c577e17 100644
--- a/hyracks-fullstack/hyracks/pom.xml
+++ b/hyracks-fullstack/hyracks/pom.xml
@@ -88,6 +88,7 @@
<module>hyracks-storage-am-lsm-invertedindex</module>
<module>hyracks-storage-am-lsm-common</module>
<module>hyracks-storage-am-lsm-btree</module>
+ <module>hyracks-storage-am-lsm-vtree</module>
<module>hyracks-storage-am-lsm-btree-column</module>
<module>hyracks-storage-am-lsm-rtree</module>
<module>hyracks-storage-am-rtree</module>