You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by im...@apache.org on 2026/08/06 18:19:44 UTC
(asterixdb) 06/14: [ASTERIXDB-3754][HYR][STO] Storage layer of VTree index - patch 1
This is an automated email from the ASF dual-hosted git repository.
parshimers pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 3922c00ab8b59e7fc9d03bf4d3fa64b1e6269b34
Author: Le0shy <le...@gmail.com>
AuthorDate: Sun May 17 18:14:33 2026 -0700
[ASTERIXDB-3754][HYR][STO] Storage layer of VTree index - patch 1
- user model changes: no
- storage format changes: yes
- interface changes: yes
Details:
VTree storage layer implementation for hierarchical cluster-based
approximate nearest neighbor search.
Bring in new hyracks-storage-am-vtree package containing:
- Four specialized frame types (interior, leaf, metadata, data) for the
level clustering tree structure
- VTree core with insert, delete, and search operations
- Static structure builder invoked by k-means clustering
- Bulk loader and flush loader for LSM disk component lifecycle
- Tree navigation utilities to support index traversal
Ext-ref: MB-72975
Change-Id: Id613553fb11bf97ca9c75ae843dee470acc36b5d
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21099
Reviewed-by: Ian Maxon <im...@apache.org>
Contrib: Ian Maxon <im...@apache.org>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
---
.../asterix/cloud/AbstractCloudIOManager.java | 12 +
.../common/context/GlobalVirtualBufferCache.java | 5 +
.../apache/hyracks/control/nc/io/IOManager.java | 15 +
.../impls/MultitenantVirtualBufferCache.java | 5 +
.../am/lsm/common/impls/VirtualBufferCache.java | 5 +
.../hyracks/hyracks-storage-am-vtree/pom.xml | 85 ++
.../am/vector/api/IVTreeBinaryAccessor.java | 50 +
.../am/vector/api/IVTreeBinaryAccessorFactory.java | 36 +
.../storage/am/vector/api/IVTreeDataFrame.java | 73 ++
.../am/vector/api/IVTreeDataTupleBuilder.java | 48 +
.../vector/api/IVTreeDataTupleBuilderFactory.java | 52 +
.../am/vector/api/IVTreeDistanceFunction.java | 40 +
.../vector/api/IVTreeDistanceFunctionFactory.java | 46 +
.../hyracks/storage/am/vector/api/IVTreeFrame.java | 40 +
.../storage/am/vector/api/IVTreeInteriorFrame.java | 49 +
.../storage/am/vector/api/IVTreeLeafFrame.java | 76 ++
.../storage/am/vector/api/IVTreeMetadataFrame.java | 51 +
.../storage/am/vector/api/IVTreeQuantizer.java | 56 +
.../am/vector/api/IVTreeQuantizerFactory.java | 48 +
.../am/vector/api/VTreeQuantizationParams.java | 31 +
.../am/vector/api/VTreeStaticTupleConstants.java | 63 +
.../storage/am/vector/frames/VTreeDataFrame.java | 227 ++++
.../am/vector/frames/VTreeDataFrameFactory.java | 61 +
.../am/vector/frames/VTreeInteriorFrame.java | 94 ++
.../vector/frames/VTreeInteriorFrameFactory.java | 70 +
.../storage/am/vector/frames/VTreeLeafFrame.java | 145 ++
.../am/vector/frames/VTreeLeafFrameFactory.java | 73 ++
.../am/vector/frames/VTreeMetadataFrame.java | 158 +++
.../vector/frames/VTreeMetadataFrameFactory.java | 69 +
.../storage/am/vector/frames/VTreeNSMFrame.java | 140 ++
.../am/vector/impls/ClusterSearchResult.java | 102 ++
.../hyracks/storage/am/vector/impls/VTree.java | 1387 ++++++++++++++++++++
.../storage/am/vector/impls/VTreeBulkLoader.java | 662 ++++++++++
.../am/vector/impls/VTreeCursorInitialState.java | 167 +++
.../am/vector/impls/VTreeDataTupleBuilder.java | 155 +++
.../vector/impls/VTreeDataTupleBuilderFactory.java | 55 +
.../storage/am/vector/impls/VTreeFlushLoader.java | 249 ++++
.../storage/am/vector/impls/VTreeOpContext.java | 257 ++++
.../storage/am/vector/impls/VTreeSearchCursor.java | 1021 ++++++++++++++
.../am/vector/impls/VTreeSearchPredicate.java | 200 +++
.../vector/impls/VTreeStaticStructureBuilder.java | 569 ++++++++
.../storage/am/vector/tuples/VTreeTupleUtils.java | 82 ++
.../am/vector/utils/CrossPollinationConfig.java | 63 +
.../am/vector/utils/NoOpVectorQuantizer.java | 58 +
.../am/vector/utils/RngAcceptanceFilter.java | 135 ++
.../am/vector/utils/VTreeChildCentroid.java | 28 +
.../am/vector/utils/VTreeDataTupleAccessor.java | 176 +++
.../am/vector/utils/VTreeLeafNeighborList.java | 154 +++
.../storage/am/vector/utils/VTreeLevelNode.java | 25 +
.../storage/am/vector/utils/VTreeMetadataKeys.java | 47 +
.../vector/utils/VTreeMetadataTupleAccessor.java | 81 ++
.../am/vector/utils/VTreeNavigationFrame.java | 103 ++
.../am/vector/utils/VTreeNavigationState.java | 89 ++
.../am/vector/utils/VTreeNavigationUtils.java | 715 ++++++++++
.../am/vector/utils/VTreeStaticTupleAccessor.java | 82 ++
.../storage/common/buffercache/BufferCache.java | 13 +
.../common/buffercache/DebugBufferCache.java | 5 +
.../storage/common/buffercache/IBufferCache.java | 13 +
.../context/write/LocalOnlyWriteContext.java | 53 +
.../am/lsm/btree/impl/TestVirtualBufferCache.java | 5 +
.../hyracks/util/annotations/AiProvenance.java | 5 +
hyracks-fullstack/hyracks/pom.xml | 1 +
62 files changed, 8680 insertions(+)
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
index d1c50108d1..2e22005def 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
@@ -248,6 +248,18 @@ public abstract class AbstractCloudIOManager extends IOManager implements IParti
return localIoManager.syncWrite(fHandle, offset, data);
}
+ @Override
+ public final int localWriteOnly(IFileHandle fHandle, long offset, ByteBuffer data) throws HyracksDataException {
+ // Local copy only; the long-term (cloud) upload is intentionally deferred to a later pass.
+ return localIoManager.syncWrite(fHandle, offset, data);
+ }
+
+ @Override
+ public final long localWriteOnly(IFileHandle fHandle, long offset, ByteBuffer[] data) throws HyracksDataException {
+ // Local copy only; the long-term (cloud) upload is intentionally deferred to a later pass.
+ return localIoManager.syncWrite(fHandle, offset, data);
+ }
+
@Override
public final int cloudWrite(IFileHandle fHandle, long offset, ByteBuffer data) throws HyracksDataException {
ICloudWriter cloudWriter = ((CloudFileHandle) fHandle).getCloudWriter();
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/GlobalVirtualBufferCache.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/GlobalVirtualBufferCache.java
index aa636c8684..120efb4b63 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/GlobalVirtualBufferCache.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/GlobalVirtualBufferCache.java
@@ -417,6 +417,11 @@ public class GlobalVirtualBufferCache implements IVirtualBufferCache, ILifeCycle
return vbc.confiscatePage(dpid);
}
+ @Override
+ public ICachedPage confiscateAndLoad(long dpid) throws HyracksDataException {
+ return vbc.confiscateAndLoad(dpid);
+ }
+
@Override
public ICachedPage confiscateLargePage(long dpid, int multiplier, int extraBlockPageId)
throws HyracksDataException {
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
index 17a5f03844..0d23dde705 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
@@ -264,6 +264,21 @@ public class IOManager implements IIOManager {
}
}
+ /**
+ * Writes to local storage only, deliberately skipping any long-term (cloud) upload. Goes through
+ * {@link #syncWrite} (the IoRequest path) rather than {@link #doSyncWrite} so an interrupt on the
+ * calling thread does not close the file channel via {@code ClosedByInterruptException}; cloud IO
+ * managers override it to write the local copy without uploading. Used by multi-pass writers (e.g.
+ * the VTree static-structure builder) that publish a page locally first and upload it later, once.
+ */
+ public int localWriteOnly(IFileHandle fHandle, long offset, ByteBuffer data) throws HyracksDataException {
+ return syncWrite(fHandle, offset, data);
+ }
+
+ public long localWriteOnly(IFileHandle fHandle, long offset, ByteBuffer[] data) throws HyracksDataException {
+ return syncWrite(fHandle, offset, data);
+ }
+
/**
* Please do check the return value of this read!
*
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MultitenantVirtualBufferCache.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MultitenantVirtualBufferCache.java
index 7f4fd388e4..2e7a698dce 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MultitenantVirtualBufferCache.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MultitenantVirtualBufferCache.java
@@ -157,6 +157,11 @@ public class MultitenantVirtualBufferCache implements IVirtualBufferCache {
return vbc.confiscatePage(dpid);
}
+ @Override
+ public ICachedPage confiscateAndLoad(long dpid) throws HyracksDataException {
+ return vbc.confiscateAndLoad(dpid);
+ }
+
@Override
public ICachedPage confiscateLargePage(long dpid, int multiplier, int extraBlockPageId)
throws HyracksDataException {
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/VirtualBufferCache.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/VirtualBufferCache.java
index f9fc9068ea..96c65164e4 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/VirtualBufferCache.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/VirtualBufferCache.java
@@ -410,6 +410,11 @@ public class VirtualBufferCache implements IVirtualBufferCache {
throw new UnsupportedOperationException("Virtual buffer caches don't have FIFO writers");
}
+ @Override
+ public ICachedPage confiscateAndLoad(long dpid) throws HyracksDataException {
+ throw new UnsupportedOperationException("Virtual buffer caches don't have FIFO writers");
+ }
+
@Override
public ICachedPage confiscateLargePage(long dpid, int multiplier, int extraBlockPageId)
throws HyracksDataException {
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/pom.xml b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/pom.xml
new file mode 100644
index 0000000000..f7b8dcb890
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/pom.xml
@@ -0,0 +1,85 @@
+<!--
+ ! Licensed to the Apache Software Foundation (ASF) under one
+ ! or more contributor license agreements. See the NOTICE file
+ ! distributed with this work for additional information
+ ! regarding copyright ownership. The ASF licenses this file
+ ! to you under the Apache License, Version 2.0 (the
+ ! "License"); you may not use this file except in compliance
+ ! with the License. You may obtain a copy of the License at
+ !
+ ! http://www.apache.org/licenses/LICENSE-2.0
+ !
+ ! Unless required by applicable law or agreed to in writing,
+ ! software distributed under the License is distributed on an
+ ! "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ! KIND, either express or implied. See the License for the
+ ! specific language governing permissions and limitations
+ ! under the License.
+ !-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>hyracks-storage-am-vtree</artifactId>
+ <name>hyracks-storage-am-vtree</name>
+
+ <parent>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks</artifactId>
+ <version>0.3.10-SNAPSHOT</version>
+ </parent>
+
+ <licenses>
+ <license>
+ <name>Apache License, Version 2.0</name>
+ <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+ <distribution>repo</distribution>
+ <comments>A business-friendly OSS license</comments>
+ </license>
+ </licenses>
+
+ <properties>
+ <root.dir>${basedir}/../..</root.dir>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-storage-am-btree</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-storage-am-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-dataflow-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-api</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-storage-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-data-std</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-util</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-api</artifactId>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/api/IVTreeBinaryAccessor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/api/IVTreeBinaryAccessor.java
new file mode 100644
index 0000000000..3b87bdccee
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/api/IVTreeBinaryAccessor.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.vector.api;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+/**
+ * Schema-agnostic accessor for a serialized vector field. Lets storage-layer code read
+ * vectors without depending on AsterixDB type-system details.
+ * <p>
+ * Stateful (holds a reference to the last {@code reset} buffer) and not thread-safe; confine one
+ * instance to a single operation context.
+ */
+public interface IVTreeBinaryAccessor {
+
+ /**
+ * Reset the accessor to point at a serialized vector field.
+ *
+ * @param data byte array containing the serialized vector
+ * @param start start offset of the vector field
+ * @param length length of the vector field in bytes
+ * @throws HyracksDataException if the data format is invalid
+ */
+ void reset(byte[] data, int start, int length) throws HyracksDataException;
+
+ /**
+ * Decodes and returns the vector at the current {@code reset} position. Returns a freshly
+ * allocated {@code double[]} on each call (not aliased to the backing buffer), so callers may
+ * retain or mutate it freely.
+ */
+ double[] getVector() throws HyracksDataException;
+
+ int getDimension() throws HyracksDataException;
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/api/IVTreeBinaryAccessorFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/api/IVTreeBinaryAccessorFactory.java
new file mode 100644
index 0000000000..42ba970f75
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/api/IVTreeBinaryAccessorFactory.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.vector.api;
+
+import java.io.Serializable;
+
+import org.apache.hyracks.api.io.IJsonSerializable;
+
+/**
+ * Serializable factory for {@link IVTreeBinaryAccessor}. Passed from the AsterixDB layer
+ * to the Hyracks layer so storage code can construct accessors without a compile-time
+ * dependency on AsterixDB modules.
+ */
+public interface IVTreeBinaryAccessorFactory extends Serializable, IJsonSerializable {
+
+ /** Index-access-parameters key under which a factory instance is passed to the storage layer. */
+ String IAP_KEY = "VECTOR_QUERY";
+
+ IVTreeBinaryAccessor createAccessor();
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/api/IVTreeDataFrame.java b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/api/IVTreeDataFrame.java
new file mode 100644
index 0000000000..0b1ea60399
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/api/IVTreeDataFrame.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.vector.api;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+
+/**
+ * Interface for VTree data frames. Data frames contain vector records sorted by
+ * {@code distance_to_centroid} ascending; the exact tuple shape depends on whether the index is
+ * quantized:
+ * <ul>
+ * <li>Non-quantized: {@code <distance_to_centroid, centroid_id, PK, include_fields>}</li>
+ * <li>Quantized: {@code <distance_to_centroid, centroid_id, quantized_distance,
+ * quantized_embedding, PK, include_fields>} (the default in this build, since
+ * quantization is enforced at index creation; pkStartField=4 vs 2)</li>
+ * </ul>
+ * <p>
+ * Not thread-safe: an instance wraps one pinned page and is confined to a single operation context.
+ */
+public interface IVTreeDataFrame extends IVTreeFrame {
+
+ /** Sets this page's forward chain pointer; {@code -1} marks end-of-chain. */
+ void setNextPage(int nextPage);
+
+ /** Returns the forward chain pointer, or {@code -1} if this is the last page in the chain. */
+ int getNextPage();
+
+ /** Returns the {@code distance_to_centroid} stored in field 0 of the tuple at {@code tupleIndex}. */
+ double getDistanceToCentroid(int tupleIndex) throws HyracksDataException;
+
+ /**
+ * Inserts {@code tuple} at slot {@code tupleIndex}, shifting existing tuples right. The caller
+ * must supply an index that preserves the {@code distance_to_centroid}-ascending ordering (see
+ * {@link #findInsertPosition(double)}) and must have ensured the page has room.
+ */
+ @Override
+ void insert(ITupleReference tuple, int tupleIndex);
+
+ /**
+ * Splits this (full) page, moving the upper half of its tuples into {@code rightFrame} and then
+ * inserting {@code tuple} into whichever half keeps the ascending order. The insertion index is
+ * recomputed from the tuple's distance in the chosen half. On return both halves remain sorted by
+ * {@code distance_to_centroid}.
+ */
+ void split(IVTreeDataFrame rightFrame, ITupleReference tuple) throws HyracksDataException;
+
+ /**
+ * Returns the slot at which a tuple with the given {@code distance} would be inserted to keep
+ * the page sorted by {@code distance_to_centroid} ascending: the index of the first tuple whose
+ * distance is strictly {@code > distance} (so equal distances are inserted after existing ones),
+ * or the tuple count if none.
+ */
+
+ int findInsertPosition(double distance) throws HyracksDataException;
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/api/IVTreeDataTupleBuilder.java b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/api/IVTreeDataTupleBuilder.java
new file mode 100644
index 0000000000..42b01cf06e
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/api/IVTreeDataTupleBuilder.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.vector.api;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+
+/**
+ * Transforms input tuples from the operator format into the storage format
+ * used in VTree data pages. The output format is implementation-specific
+ * (e.g., with or without quantized vector embeddings).
+ *
+ * Implementations are created by {@link IVTreeDataTupleBuilderFactory} and
+ * held by the operation context for the lifetime of an index accessor.
+ */
+public interface IVTreeDataTupleBuilder {
+
+ /**
+ * Create a data tuple for VTree storage.
+ *
+ * @param vector the vector extracted from the input tuple
+ * @param distance the computed distance from the vector to its assigned centroid
+ * @param centroidId the ID of the assigned leaf centroid
+ * @param originalTuple the original input tuple containing vector, include fields, and primary key
+ * @return ITupleReference in storage format (valid until the next call)
+ * @throws HyracksDataException if tuple creation fails
+ */
+ ITupleReference buildDataTuple(double[] vector, double distance, int centroidId, ITupleReference originalTuple)
+ throws HyracksDataException;
+
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/api/IVTreeDataTupleBuilderFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/api/IVTreeDataTupleBuilderFactory.java
new file mode 100644
index 0000000000..e74e80065f
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/api/IVTreeDataTupleBuilderFactory.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.vector.api;
+
+import java.io.Serializable;
+
+import org.apache.hyracks.util.annotations.AiProvenance;
+
+/**
+ * Factory for {@link IVTreeDataTupleBuilder} instances.
+ * <p>
+ * {@link Serializable} so it can travel on {@code LSMVTreeLocalResourceFactory} in the Java-serialized
+ * index-creation job. Not {@code IJsonSerializable}: {@code LSMVTreeLocalResource.fromJson} rebuilds it
+ * from primitives ({@code numIncludeFields}, {@code isQuantized}) rather than serializing it as JSON.
+ */
+public interface IVTreeDataTupleBuilderFactory extends Serializable {
+
+ /**
+ * Creates a new data tuple creator.
+ *
+ * @param quantizationParams the scalar-quantization params, or {@code null} when the index is
+ * non-quantized, in which case the quantized fields are not written
+ * @return a new IVTreeDataTupleBuilder
+ */
+ IVTreeDataTupleBuilder createDataTupleBuilder(VTreeQuantizationParams quantizationParams);
+
+ /**
+ * Whether the tuples produced by this factory use the quantized data-tuple layout
+ * {@code [distance, centroidId, quantized_distance, quantized_embedding, PK..., includes...]}
+ * (pkStartField=4) rather than the non-quantized {@code [distance, centroidId, PK...,
+ * includes...]} layout (pkStartField=2).
+ */
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_FABLE_5, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.ASSISTED)
+ boolean isQuantized();
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/api/IVTreeDistanceFunction.java b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/api/IVTreeDistanceFunction.java
new file mode 100644
index 0000000000..7ff2f3ad4c
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/api/IVTreeDistanceFunction.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.vector.api;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+/**
+ * Distance function between two decoded vectors, injected into the VTree index by the Hyracks
+ * application that creates it (e.g. Euclidean, cosine, negated dot product).
+ * <p>
+ * Unlike {@link org.apache.hyracks.api.dataflow.value.IBinaryComparator}, which returns a
+ * three-valued ordering (-1/0/+1) over raw field bytes, this returns a real-valued magnitude over
+ * two already-decoded {@code double[]} vectors: smaller means nearer, and the values are compared
+ * numerically to rank candidates.
+ */
+@FunctionalInterface
+public interface IVTreeDistanceFunction {
+
+ /**
+ * @return the distance between {@code vector1} and {@code vector2}.
+ * @throws HyracksDataException if the calculation fails
+ */
+ double apply(double[] vector1, double[] vector2) throws HyracksDataException;
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/api/IVTreeDistanceFunctionFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/api/IVTreeDistanceFunctionFactory.java
new file mode 100644
index 0000000000..12229bb572
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/api/IVTreeDistanceFunctionFactory.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.vector.api;
+
+import java.io.Serializable;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.IJsonSerializable;
+
+/**
+ * Factory for {@link IVTreeDistanceFunction} instances. Supplied to the storage layer at
+ * construction time (persisted on the local resource so it survives NC restart) and, for
+ * query-time overrides, through the index-access-parameters map (key {@link #IAP_KEY}).
+ * <p>
+ * Lets the {@code asterix-runtime} layer plug in distance implementations without the
+ * {@code hyracks-storage-am-vtree} module depending on AsterixDB types. Extends
+ * {@link IJsonSerializable} so the concrete factory can be persisted on the local resource.
+ */
+public interface IVTreeDistanceFunctionFactory extends Serializable, IJsonSerializable {
+
+ /** Index-access-parameters key under which a factory instance is passed to the storage layer. */
+ String IAP_KEY = "VD_FUN_FACTORY";
+
+ /**
+ * Build the distance function for the metric this factory was created with. The metric is fixed at
+ * index creation (it is determined by the embedding model) and baked into the concrete factory, so
+ * Hyracks never passes or is aware of a metric string. Must not return {@code null}.
+ */
+ IVTreeDistanceFunction createDistanceFunction() throws HyracksDataException;
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/api/IVTreeFrame.java b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/api/IVTreeFrame.java
new file mode 100644
index 0000000000..af08a81646
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/api/IVTreeFrame.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.vector.api;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame;
+
+/**
+ * Base interface for all VTree frames.
+ * <p>
+ * Not thread-safe: an instance wraps one pinned page and is confined to a single operation context.
+ */
+public interface IVTreeFrame extends ITreeIndexFrame {
+
+ /**
+ * Appends {@code tuple} at the end of the page's sorted run. Precondition: {@code tuple}'s
+ * ordering key is {@code >=} that of the last tuple already on the page, so appending keeps the
+ * page sorted. Callers building a page in ascending key order use this to avoid a per-insert
+ * position search.
+ */
+ void insertSorted(ITupleReference tuple) throws HyracksDataException;
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/api/IVTreeInteriorFrame.java b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/api/IVTreeInteriorFrame.java
new file mode 100644
index 0000000000..9c82d8ff7e
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/api/IVTreeInteriorFrame.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.vector.api;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+/**
+ * Interface for VTree interior/root frames.
+ * Interior frames contain cluster entries: <cid, full_precision_centroid, pointer_to_clustered_page>
+ * <p>
+ * Not thread-safe: an instance wraps one pinned page and is confined to a single operation context.
+ */
+public interface IVTreeInteriorFrame extends IVTreeFrame {
+
+ /** Returns the child (next-level) page id pointed to by the entry at {@code tupleIndex}. */
+ int getChildPageId(int tupleIndex) throws HyracksDataException;
+
+ /** Sets the child page pointer for the entry at {@code tupleIndex}. */
+ void setChildPageId(int tupleIndex, int childPageId) throws HyracksDataException;
+
+ /** Sets the same-level overflow chain pointer; {@code -1} marks end-of-chain. */
+ void setNextPage(int nextPageId);
+
+ /** Returns the same-level overflow chain pointer, or {@code -1} if this is the last page. */
+ int getNextPage();
+
+ /** Sets the header bit indicating this page continues onto a {@link #getNextPage()} overflow page. */
+ void setOverflowFlagBit(boolean overflowFlag);
+
+ /** Returns whether this page has an overflow continuation (see {@link #getNextPage()}). */
+ boolean getOverflowFlagBit();
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/api/IVTreeLeafFrame.java b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/api/IVTreeLeafFrame.java
new file mode 100644
index 0000000000..72f20e5742
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/api/IVTreeLeafFrame.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.vector.api;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleReference;
+
+/**
+ * Interface for VTree leaf frames.
+ * Leaf frames contain cluster entries: <cid, full_precision_centroid, pointer_to_first_metadata_page>
+ */
+public interface IVTreeLeafFrame extends IVTreeFrame {
+
+ void setNextLeaf(int nextLeafPage);
+
+ int getNextLeaf();
+
+ int getMetadataPagePointer(int tupleIndex) throws HyracksDataException;
+
+ /**
+ * Sets the metadata page pointer for a cluster entry.
+ * @param tupleIndex the index of the cluster entry
+ * @param metadataPageId the page ID of the first metadata page
+ * @throws HyracksDataException if an error occurs
+ */
+ void setMetadataPagePointer(int tupleIndex, int metadataPageId) throws HyracksDataException;
+
+ int getCentroidId(int tupleIndex) throws HyracksDataException;
+
+ /**
+ * Inserts a cluster entry at the specified index.
+ * @param tuple the cluster entry to insert
+ * @param tupleIndex the index to insert at
+ */
+ void insert(ITupleReference tuple, int tupleIndex);
+
+ boolean getOverflowFlagBit();
+
+ void setOverflowFlagBit(boolean overflowFlag);
+
+ /**
+ * Gets the quantized centroid bytes for a cluster entry, if present.
+ * For quantized tuples (4-field format), returns the byte[] at field 2.
+ * For non-quantized tuples (3-field format), returns null.
+ *
+ * @param tupleIndex the index of the cluster entry
+ * @return the quantized centroid bytes, or null if not quantized
+ * @throws HyracksDataException if an error occurs
+ */
+ byte[] getQuantizedCentroidBytes(int tupleIndex) throws HyracksDataException;
+
+ /**
+ * Returns the frame's reusable, shared tuple reference. It is positional scratch state: reset it
+ * ({@code resetByTupleIndex}/{@code resetByTupleOffset}) before reading, and do not retain a
+ * position across another call that also drives it (e.g. {@link #getCentroidId}).
+ */
+ ITreeIndexTupleReference getFrameTuple();
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/api/IVTreeMetadataFrame.java b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/api/IVTreeMetadataFrame.java
new file mode 100644
index 0000000000..a213dfa6dd
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/api/IVTreeMetadataFrame.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.vector.api;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+
+/**
+ * Interface for VTree metadata/directory frames.
+ * Metadata frames contain entries: <max_distance, pointer_to_data_page>, sorted by
+ * {@code max_distance} ascending.
+ * <p>
+ * Not thread-safe: an instance wraps one pinned page and is confined to a single operation context.
+ */
+public interface IVTreeMetadataFrame extends IVTreeFrame {
+
+ /** Sets this directory page's forward chain pointer; {@code -1} marks end-of-chain. */
+ void setNextPage(int nextPage);
+
+ /** Returns the forward chain pointer, or {@code -1} if this is the last directory page. */
+ int getNextPage();
+
+ /** Returns the {@code max_distance} of the entry at {@code tupleIndex}. */
+ double getMaxDistance(int tupleIndex) throws HyracksDataException;
+
+ /** Returns the data-page pointer of the entry at {@code tupleIndex}. */
+ int getDataPagePointer(int tupleIndex) throws HyracksDataException;
+
+ /**
+ * Inserts {@code tuple} at slot {@code tupleIndex}, shifting existing entries right. The caller
+ * must supply an index that preserves the {@code max_distance}-ascending ordering.
+ */
+ void insert(ITupleReference tuple, int tupleIndex);
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/api/IVTreeQuantizer.java b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/api/IVTreeQuantizer.java
new file mode 100644
index 0000000000..311ad8c7a9
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/api/IVTreeQuantizer.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.vector.api;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+/**
+ * Vector quantization/dequantization at the Hyracks layer. Both methods return double[]
+ * so distance functions can consume their output directly.
+ * <p>
+ * Not thread-safe: a quantizer is built per query and confined to a single operation context.
+ */
+public interface IVTreeQuantizer {
+
+ /**
+ * Index-access-parameters key for injecting a pre-built quantizer (test-fixture path).
+ * Production code passes an {@link IVTreeQuantizerFactory} under
+ * {@link IVTreeQuantizerFactory#IAP_KEY} instead and lets the storage layer build the
+ * quantizer per query.
+ */
+ String IAP_KEY = "VECTOR_QUANTIZER";
+
+ /**
+ * Quantize a full-precision vector and immediately dequantize it back to double[].
+ *
+ * @param vector the full-precision vector
+ * @return a lossy double[] suitable for approximate distance computation
+ * @throws HyracksDataException if quantization fails
+ */
+ double[] quantize(double[] vector) throws HyracksDataException;
+
+ /**
+ * Dequantize a stored quantized byte representation back to double[].
+ *
+ * @param quantizedBytes the quantized byte representation
+ * @return the dequantized double[]
+ * @throws HyracksDataException if dequantization fails
+ */
+ double[] dequantize(byte[] quantizedBytes) throws HyracksDataException;
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/api/IVTreeQuantizerFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/api/IVTreeQuantizerFactory.java
new file mode 100644
index 0000000000..b3de34ea6b
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/api/IVTreeQuantizerFactory.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.vector.api;
+
+import java.io.Serializable;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+/**
+ * Factory for {@link IVTreeQuantizer} instances, supplied to the storage layer through the
+ * index-access-parameters map (key {@link #IAP_KEY}).
+ * <p>
+ * Replaces the previous {@code Class.forName} / constructor-reflection block in
+ * {@code VTree#search} that constructed an AsterixDB {@code ScalarVectorQuantizer} from a
+ * {@code float[]} quantization-params payload.
+ */
+public interface IVTreeQuantizerFactory extends Serializable {
+
+ /** Index-access-parameters key under which a factory instance is passed to the storage layer. */
+ String IAP_KEY = "VECTOR_QUANTIZER_FACTORY";
+
+ /**
+ * Build a quantizer for the given dimensionality. The distance metric is fixed at index creation and
+ * baked into the concrete factory (it selects symmetric vs asymmetric distance), so Hyracks does not
+ * pass a metric. The {@code params} are those returned by {@code VTree#getQuantizationParams()}.
+ *
+ * @param vectorDimensions vector dimensionality the index was built with
+ * @param params the scalar-quantization params; never {@code null} on this path,
+ * which is only reached for a quantized index
+ */
+ IVTreeQuantizer createQuantizer(int vectorDimensions, VTreeQuantizationParams params) throws HyracksDataException;
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/api/VTreeQuantizationParams.java b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/api/VTreeQuantizationParams.java
new file mode 100644
index 0000000000..074298a2fe
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/api/VTreeQuantizationParams.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.vector.api;
+
+import java.io.Serializable;
+
+/**
+ * OptimizedScalarQuantization parameters for a VTree index: the min/max clipping quantiles, the OSQ
+ * anisotropic weight ({@code alpha}), the confidence interval, the quantization bit width, and the
+ * training sample count. Carried across the Hyracks/asterix layer boundary as named, typed fields.
+ */
+public record VTreeQuantizationParams(float minQuantile, float maxQuantile, float alpha, float confidenceInterval,
+ int bits, int sampleCount) implements Serializable {
+ private static final long serialVersionUID = 0L;
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/api/VTreeStaticTupleConstants.java b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/api/VTreeStaticTupleConstants.java
new file mode 100644
index 0000000000..32d101afe9
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/api/VTreeStaticTupleConstants.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.vector.api;
+
+/**
+ * Field layout of VTree static-structure (interior and leaf) tuples.
+ * <p>
+ * The authoritative on-page binary layout is the {@code ITypeTraits[]} declared in
+ * {@code VTreeInteriorFrameFactory} and {@code VTreeLeafFrameFactory}; the field
+ * <em>content</em> is encoded by the producer
+ * ({@code VTreeStaticStructureCreatorOperatorDescriptor}). This class documents that
+ * shared schema in one place so readers and the builder agree on field positions
+ * without restating per-field serializers at each call site.
+ * <p>
+ * A stored tuple is the input centroid tuple with a trailing pointer field appended by
+ * {@code VTreeStaticStructureBuilder.createEntryTuple} — the navigation child pointer for
+ * interior tuples, the metadata-page pointer for leaf tuples. The pointer is always LAST,
+ * so {@code getMetadataPagePointer()} resolves it via {@code getFieldCount() - 1}
+ * regardless of which optional leaf fields precede it.
+ *
+ * <pre>
+ * Interior: [ cid, embedding, childPagePtr ]
+ * Leaf, non-quantized (tests): [ cid, embedding, metadataPtr ]
+ * Leaf, quantized: [ cid, embedding, quantizedBytes, metadataPtr ]
+ * Leaf, quantized + graph neighbors:[ cid, embedding, quantizedBytes, neighborList, metadataPtr ]
+ * </pre>
+ *
+ * Interior vs leaf is decided by build level (not by field count), since an interior tuple
+ * and a non-quantized leaf tuple are both 3 fields once the pointer is appended.
+ */
+public final class VTreeStaticTupleConstants {
+
+ private VTreeStaticTupleConstants() {
+ }
+
+ /** Centroid id field index (shared by interior and all leaf layouts). */
+ public static final int CENTROID_ID_FIELD = 0;
+
+ /** Full-precision centroid embedding field index (shared by interior and all leaf layouts). */
+ public static final int EMBEDDING_FIELD = 1;
+
+ /** Quantized centroid bytes field index in quantized leaf tuples. */
+ public static final int LEAF_QUANTIZED_BYTES_FIELD = 2;
+
+ /** Graph-neighbor list field index in quantized-with-neighbors leaf tuples. */
+ public static final int LEAF_NEIGHBOR_LIST_FIELD = 3;
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/frames/VTreeDataFrame.java b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/frames/VTreeDataFrame.java
new file mode 100644
index 0000000000..0ac3870725
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/frames/VTreeDataFrame.java
@@ -0,0 +1,227 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.vector.frames;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.primitive.DoublePointable;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.storage.am.btree.frames.OrderedSlotManager;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
+import org.apache.hyracks.storage.am.vector.api.IVTreeDataFrame;
+import org.apache.hyracks.storage.am.vector.utils.VTreeDataTupleAccessor;
+
+/**
+ * VTree data frame implementation.
+ * <p>
+ * Page layout: base header (from {@link VTreeNSMFrame}) followed by a 4-byte next-data-page pointer
+ * (sentinel {@code -1}). Tuples are kept sorted by {@code distance_to_centroid} ascending. The exact
+ * tuple shape depends on whether the index is quantized:
+ * <ul>
+ * <li>Non-quantized: {@code <distance_to_centroid, centroid_id, PK, included_fields>}</li>
+ * <li>Quantized: {@code <distance_to_centroid, centroid_id, quantized_distance,
+ * quantized_embedding, PK, included_fields>} (the default in this build, since
+ * quantization is enforced at index creation; pkStartField=4 vs 2)</li>
+ * </ul>
+ */
+public class VTreeDataFrame extends VTreeNSMFrame implements IVTreeDataFrame {
+
+ // Offset (in bytes from page start) of the 4-byte next-page pointer.
+ private static final int NEXT_PAGE_OFFSET = CENTROID_ID_OFFSET + Integer.BYTES;
+
+ public VTreeDataFrame(ITreeIndexTupleWriter tupleWriter) {
+ super(tupleWriter, new OrderedSlotManager());
+ }
+
+ @Override
+ public void initBuffer(byte level) {
+ super.initBuffer(level);
+ buf.putInt(NEXT_PAGE_OFFSET, -1);
+ }
+
+ @Override
+ public int getPageHeaderSize() {
+ return NEXT_PAGE_OFFSET + Integer.BYTES;
+ }
+
+ @Override
+ public void setNextPage(int nextPage) {
+ buf.putInt(NEXT_PAGE_OFFSET, nextPage);
+ }
+
+ @Override
+ public int getNextPage() {
+ return buf.getInt(NEXT_PAGE_OFFSET);
+ }
+
+ @Override
+ public double getDistanceToCentroid(int tupleIndex) {
+ frameTuple.resetByTupleIndex(this, tupleIndex);
+ // First field of a data tuple is the raw double distance (no type tag).
+ int distanceOff = frameTuple.getFieldStart(VTreeDataTupleAccessor.DISTANCE_FIELD);
+ return buf.getDouble(distanceOff);
+ }
+
+ /**
+ * Find the insertion position for a tuple based on distance to maintain sorted order.
+ * Uses RIGHT boundary search: inserts AFTER all existing tuples with the same distance.
+ * This preserves temporal ordering (FIFO) for tuples with equal distances.
+ */
+ public int findInsertPosition(double distance) {
+ int tupleCount = getTupleCount();
+
+ // Binary search for RIGHT boundary (first tuple with distance > target)
+ int left = 0;
+ int right = tupleCount;
+
+ while (left < right) {
+ int mid = (left + right) / 2;
+ double midDistance = getDistanceToCentroid(mid);
+
+ if (midDistance <= distance) {
+ // Include equal distances: move past them
+ left = mid + 1;
+ } else {
+ // midDistance > distance
+ right = mid;
+ }
+ }
+
+ return left;
+ }
+
+ /**
+ * Find tuple matching both distance and primary key using RIGHT BOUND search.
+ * Returns the rightmost (most recently inserted) tuple if multiple matches exist.
+ * This is critical for finding matter tuples inserted after a delete-marker tuple during deletion.
+ *
+ * Uses binary comparison for primary key matching - no type assumption.
+ *
+ * @param distance Target distance to centroid
+ * @param primaryKey Primary key bytes to match (binary format)
+ * @return Tuple index if found, -1 if not found
+ */
+ public int findTupleByDistanceAndPrimaryKey(double distance, byte[] primaryKey, int pkFieldIndex)
+ throws HyracksDataException {
+
+ // Step 1: Use RIGHT BOUND search to find upper boundary
+ int upperBound = findInsertPosition(distance);
+
+ // Step 2: Search BACKWARD from upperBound-1 to find matching PK
+ // This ensures we find the RIGHTMOST (last inserted) tuple with this distance+PK
+ for (int i = upperBound - 1; i >= 0; i--) {
+ double dist = getDistanceToCentroid(i);
+
+ // Stop when we reach a different distance zone
+ if (dist < distance) {
+ break;
+ }
+
+ // Check primary key match using binary comparison (no type assumption)
+ byte[] pk = getPrimaryKey(i, pkFieldIndex);
+ if (Arrays.equals(pk, primaryKey)) {
+ return i; // Found the rightmost matching tuple
+ }
+ }
+
+ return -1; // Not found
+ }
+
+ /**
+ * Split this data frame into {@code this} (left) and {@code rightFrame} (right) and insert {@code tuple}
+ * into whichever side keeps the sorted-by-distance invariant. Follows the BTreeNSMLeafFrame.split() pattern:
+ * mirror the buffer, shift right-half slots, fix tuple counts, compact, then insert.
+ * The {@code insertIndex} parameter is kept for API compatibility; the real insert position is recomputed
+ * from the new tuple's distance after compaction.
+ */
+ @Override
+ public void split(IVTreeDataFrame rightFrameArg, ITupleReference tuple) throws HyracksDataException {
+ VTreeDataFrame rightFrame = (VTreeDataFrame) rightFrameArg;
+ int tupleCount = getTupleCount();
+ int tuplesToLeft = tupleCount / 2;
+ int tuplesToRight = tupleCount - tuplesToLeft;
+
+ // Mirror entire page buffer into right frame, then shift its right-half slot range left.
+ ByteBuffer rightBuffer = rightFrame.getBuffer();
+ System.arraycopy(buf.array(), 0, rightBuffer.array(), 0, buf.capacity());
+
+ int src = rightFrame.getSlotManager().getSlotEndOff();
+ int dest =
+ rightFrame.getSlotManager().getSlotEndOff() + tuplesToLeft * rightFrame.getSlotManager().getSlotSize();
+ int length = rightFrame.getSlotManager().getSlotSize() * tuplesToRight;
+ System.arraycopy(rightBuffer.array(), src, rightBuffer.array(), dest, length);
+
+ rightBuffer.putInt(Constants.TUPLE_COUNT_OFFSET, tuplesToRight);
+ buf.putInt(Constants.TUPLE_COUNT_OFFSET, tuplesToLeft);
+
+ rightFrame.compact();
+ this.compact();
+
+ // Pick the target frame based on the new tuple's distance vs. the left frame's last (split-point) tuple.
+ double newTupleDistance = extractDistanceFromTuple(tuple);
+ VTreeDataFrame targetFrame;
+ if (tuplesToLeft > 0) {
+ double splitPointDistance = getDistanceToCentroid(tuplesToLeft - 1);
+ targetFrame = (newTupleDistance <= splitPointDistance) ? this : rightFrame;
+ } else {
+ // Edge case: left frame is empty after split.
+ targetFrame = rightFrame;
+ }
+
+ // Recompute insertion index in the target frame to honor RIGHT-boundary semantics.
+ int targetTupleIndex = targetFrame.findInsertPosition(newTupleDistance);
+ targetFrame.insert(tuple, targetTupleIndex);
+ }
+
+ /**
+ * Extract distance from tuple (first field).
+ */
+ private double extractDistanceFromTuple(ITupleReference tuple) {
+ byte[] data = tuple.getFieldData(VTreeDataTupleAccessor.DISTANCE_FIELD);
+ int offset = tuple.getFieldStart(VTreeDataTupleAccessor.DISTANCE_FIELD);
+ return DoublePointable.getDouble(data, offset);
+ }
+
+ /**
+ * Get primary key from data tuple.
+ *
+ * @param tupleIndex the tuple index in this frame
+ * @param pkFieldIndex the field index of the primary key in the data tuple
+ * @return the primary key bytes
+ */
+ public byte[] getPrimaryKey(int tupleIndex, int pkFieldIndex) throws HyracksDataException {
+ frameTuple.resetByTupleIndex(this, tupleIndex);
+
+ byte[] data = frameTuple.getFieldData(pkFieldIndex);
+ int offset = frameTuple.getFieldStart(pkFieldIndex);
+ int length = frameTuple.getFieldLength(pkFieldIndex);
+
+ return Arrays.copyOfRange(data, offset, offset + length);
+ }
+
+ @Override
+ public String printHeader() {
+ StringBuilder strBuilder = new StringBuilder(super.printHeader());
+ strBuilder.append("nextPage: ").append(getNextPage()).append('\n');
+ return strBuilder.toString();
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/frames/VTreeDataFrameFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/frames/VTreeDataFrameFactory.java
new file mode 100644
index 0000000000..791fada9be
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/frames/VTreeDataFrameFactory.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.vector.frames;
+
+import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriterFactory;
+import org.apache.hyracks.storage.am.vector.api.IVTreeDataFrame;
+
+/**
+ * Factory for {@link VTreeDataFrame} instances. Data frames hold tuples sorted by
+ * {@code distance_to_centroid} ascending; the exact tuple shape depends on whether the index is
+ * quantized:
+ * <ul>
+ * <li>Non-quantized: {@code <distance_to_centroid, centroid_id, primary_key, included_fields>}</li>
+ * <li>Quantized: {@code <distance_to_centroid, centroid_id, quantized_distance,
+ * quantized_embedding, primary_key, included_fields>} (the default in this build, since
+ * quantization is enforced at index creation; pkStartField=4 vs 2)</li>
+ * </ul>
+ */
+public class VTreeDataFrameFactory implements ITreeIndexFrameFactory {
+
+ private static final long serialVersionUID = 1L;
+ private final ITreeIndexTupleWriterFactory tupleWriterFactory;
+ private final int vectorDimensions;
+
+ public VTreeDataFrameFactory(ITreeIndexTupleWriterFactory tupleWriterFactory, int vectorDimensions) {
+ this.tupleWriterFactory = tupleWriterFactory;
+ this.vectorDimensions = vectorDimensions;
+ }
+
+ @Override
+ public IVTreeDataFrame createFrame() {
+ return new VTreeDataFrame(tupleWriterFactory.createTupleWriter());
+ }
+
+ @Override
+ public ITreeIndexTupleWriterFactory getTupleWriterFactory() {
+ return tupleWriterFactory;
+ }
+
+ public int getVectorDimensions() {
+ return vectorDimensions;
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/frames/VTreeInteriorFrame.java b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/frames/VTreeInteriorFrame.java
new file mode 100644
index 0000000000..403460a104
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/frames/VTreeInteriorFrame.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.vector.frames;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.primitive.IntegerPointable;
+import org.apache.hyracks.storage.am.btree.frames.OrderedSlotManager;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
+import org.apache.hyracks.storage.am.vector.api.IVTreeInteriorFrame;
+
+/**
+ * VTree interior (root and intermediate) frame.
+ * <p>
+ * Page layout: base header (from {@link VTreeNSMFrame}) followed by a 4-byte next-overflow-page pointer and a
+ * 1-byte overflow flag. Tuples have the format
+ * {@code <cid, full_precision_centroid, child_page_pointer>}; the child pointer is read from the last field.
+ */
+public class VTreeInteriorFrame extends VTreeNSMFrame implements IVTreeInteriorFrame {
+
+ // Offset (bytes from page start) of the 4-byte next-overflow-page pointer.
+ protected static final int NEXT_PAGE_OFFSET = CENTROID_ID_OFFSET + Integer.BYTES;
+ // Offset of the 1-byte overflow flag (0 = no overflow, non-zero = chained).
+ protected static final int OVERFLOW_FLAG_OFFSET = NEXT_PAGE_OFFSET + Integer.BYTES;
+
+ public VTreeInteriorFrame(ITreeIndexTupleWriter tupleWriter) {
+ super(tupleWriter, new OrderedSlotManager());
+ }
+
+ @Override
+ public void initBuffer(byte level) {
+ super.initBuffer(level);
+ buf.putInt(NEXT_PAGE_OFFSET, -1);
+ buf.put(OVERFLOW_FLAG_OFFSET, (byte) 0);
+ }
+
+ @Override
+ public int getPageHeaderSize() {
+ return OVERFLOW_FLAG_OFFSET + Byte.BYTES;
+ }
+
+ @Override
+ public void setOverflowFlagBit(boolean overflowFlag) {
+ buf.put(OVERFLOW_FLAG_OFFSET, (byte) (overflowFlag ? 1 : 0));
+ }
+
+ @Override
+ public boolean getOverflowFlagBit() {
+ return buf.get(OVERFLOW_FLAG_OFFSET) != 0;
+ }
+
+ @Override
+ public void setNextPage(int nextPageId) {
+ buf.putInt(NEXT_PAGE_OFFSET, nextPageId);
+ }
+
+ @Override
+ public int getNextPage() {
+ return buf.getInt(NEXT_PAGE_OFFSET);
+ }
+
+ @Override
+ public int getChildPageId(int tupleIndex) throws HyracksDataException {
+ frameTuple.resetByTupleIndex(this, tupleIndex);
+ // Child page pointer is stored as the last field of the tuple.
+ int childPtrFieldIndex = frameTuple.getFieldCount() - 1;
+ return IntegerPointable.getInteger(frameTuple.getFieldData(childPtrFieldIndex),
+ frameTuple.getFieldStart(childPtrFieldIndex));
+ }
+
+ @Override
+ public void setChildPageId(int tupleIndex, int childPageId) throws HyracksDataException {
+ frameTuple.resetByTupleIndex(this, tupleIndex);
+ int childPtrFieldIndex = frameTuple.getFieldCount() - 1;
+ IntegerPointable.setInteger(frameTuple.getFieldData(childPtrFieldIndex),
+ frameTuple.getFieldStart(childPtrFieldIndex), childPageId);
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/frames/VTreeInteriorFrameFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/frames/VTreeInteriorFrameFactory.java
new file mode 100644
index 0000000000..c8ab044e55
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/frames/VTreeInteriorFrameFactory.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.vector.frames;
+
+import org.apache.hyracks.api.dataflow.value.ITypeTraits;
+import org.apache.hyracks.storage.am.common.api.INullIntrospector;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriterFactory;
+import org.apache.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
+import org.apache.hyracks.storage.am.vector.api.IVTreeInteriorFrame;
+import org.apache.hyracks.storage.am.vector.utils.VTreeStaticTupleAccessor;
+
+/**
+ * Factory for {@link VTreeInteriorFrame} instances. {@code centroidDimensions} is retained for
+ * future per-factory dimension validation.
+ *
+ * <p>Interior tuples are NOT caller-parameterized — every VTree interior frame uses the same
+ * fixed three-field layout {@code <cluster_id, centroid, child_page_pointer>}. The schema and the
+ * tuple writer it implies are owned by this factory so call sites need only supply the dimension
+ * and null-handling knobs.
+ */
+public class VTreeInteriorFrameFactory implements ITreeIndexFrameFactory {
+
+ private static final long serialVersionUID = 1L;
+ private final ITreeIndexTupleWriter tupleWriter;
+ private final int centroidDimensions;
+
+ public VTreeInteriorFrameFactory(int centroidDimensions, ITypeTraits nullTypeTraits,
+ INullIntrospector nullIntrospector) {
+ this.centroidDimensions = centroidDimensions;
+ this.tupleWriter = new TypeAwareTupleWriterFactory(VTreeStaticTupleAccessor.interiorTypeTraits(),
+ nullTypeTraits, nullIntrospector).createTupleWriter();
+ }
+
+ @Override
+ public IVTreeInteriorFrame createFrame() {
+ return new VTreeInteriorFrame(tupleWriter);
+ }
+
+ @Override
+ public ITreeIndexTupleWriterFactory getTupleWriterFactory() {
+ return null;
+ }
+
+ public ITreeIndexTupleWriter getTupleWriter() {
+ return tupleWriter;
+ }
+
+ public int getCentroidDimensions() {
+ return centroidDimensions;
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/frames/VTreeLeafFrame.java b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/frames/VTreeLeafFrame.java
new file mode 100644
index 0000000000..b0edb93981
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/frames/VTreeLeafFrame.java
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.vector.frames;
+
+import java.util.Arrays;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.primitive.ByteArrayPointable;
+import org.apache.hyracks.data.std.primitive.IntegerPointable;
+import org.apache.hyracks.storage.am.btree.frames.OrderedSlotManager;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleReference;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
+import org.apache.hyracks.storage.am.vector.api.IVTreeLeafFrame;
+import org.apache.hyracks.storage.am.vector.api.VTreeStaticTupleConstants;
+
+/**
+ * VTree leaf frame (bottom of the static structure).
+ * <p>
+ * Page layout: base header (from {@link VTreeNSMFrame}) followed by a 4-byte next-leaf pointer and a 1-byte
+ * overflow flag. Tuples come in two shapes:
+ * <ul>
+ * <li>Non-quantized (3 fields): {@code <cid, full_precision_centroid, metadata_page_pointer>}</li>
+ * <li>Quantized (4 fields): {@code <cid, full_precision_centroid, quantized_bytes, metadata_page_pointer>}</li>
+ * </ul>
+ * In both cases the metadata-page pointer is the last field.
+ */
+public class VTreeLeafFrame extends VTreeNSMFrame implements IVTreeLeafFrame {
+ // Offset (bytes from page start) of the 4-byte next-leaf pointer.
+ protected static final int NEXT_PAGE_OFFSET = CENTROID_ID_OFFSET + Integer.BYTES;
+ // Offset of the 1-byte overflow flag (0 = no overflow, non-zero = chained).
+ protected static final int OVERFLOW_FLAG_OFFSET = NEXT_PAGE_OFFSET + Integer.BYTES;
+
+ public VTreeLeafFrame(ITreeIndexTupleWriter tupleWriter) {
+ super(tupleWriter, new OrderedSlotManager());
+ }
+
+ @Override
+ public void initBuffer(byte level) {
+ super.initBuffer(level);
+ buf.putInt(NEXT_PAGE_OFFSET, -1);
+ buf.put(OVERFLOW_FLAG_OFFSET, (byte) 0);
+ }
+
+ @Override
+ public int getPageHeaderSize() {
+ return OVERFLOW_FLAG_OFFSET + Byte.BYTES;
+ }
+
+ @Override
+ public void setNextLeaf(int nextLeafPage) {
+ buf.putInt(NEXT_PAGE_OFFSET, nextLeafPage);
+ }
+
+ @Override
+ public int getNextLeaf() {
+ return buf.getInt(NEXT_PAGE_OFFSET);
+ }
+
+ @Override
+ public void setOverflowFlagBit(boolean overflowFlag) {
+ buf.put(OVERFLOW_FLAG_OFFSET, (byte) (overflowFlag ? 1 : 0));
+ }
+
+ @Override
+ public boolean getOverflowFlagBit() {
+ return buf.get(OVERFLOW_FLAG_OFFSET) != 0;
+ }
+
+ @Override
+ public int getMetadataPagePointer(int tupleIndex) throws HyracksDataException {
+ frameTuple.resetByTupleIndex(this, tupleIndex);
+ // Metadata page pointer is stored as the last field of the leaf tuple.
+ int metadataPtrFieldIndex = frameTuple.getFieldCount() - 1;
+ return IntegerPointable.getInteger(frameTuple.getFieldData(metadataPtrFieldIndex),
+ frameTuple.getFieldStart(metadataPtrFieldIndex));
+ }
+
+ @Override
+ public void setMetadataPagePointer(int tupleIndex, int metadataPageId) throws HyracksDataException {
+ frameTuple.resetByTupleIndex(this, tupleIndex);
+ int metadataPtrFieldIndex = frameTuple.getFieldCount() - 1;
+ IntegerPointable.setInteger(frameTuple.getFieldData(metadataPtrFieldIndex),
+ frameTuple.getFieldStart(metadataPtrFieldIndex), metadataPageId);
+ }
+
+ @Override
+ public int getCentroidId(int tupleIndex) throws HyracksDataException {
+ frameTuple.resetByTupleIndex(this, tupleIndex);
+ // Centroid ID is the first field of the leaf tuple.
+ int cidField = VTreeStaticTupleConstants.CENTROID_ID_FIELD;
+ return IntegerPointable.getInteger(frameTuple.getFieldData(cidField), frameTuple.getFieldStart(cidField));
+ }
+
+ @Override
+ public ITreeIndexTupleReference getFrameTuple() {
+ return frameTuple;
+ }
+
+ @Override
+ public byte[] getQuantizedCentroidBytes(int tupleIndex) throws HyracksDataException {
+ frameTuple.resetByTupleIndex(this, tupleIndex);
+ // A non-quantized leaf is [cid, embedding, metadataPtr] with the pointer at index
+ // LEAF_QUANTIZED_BYTES_FIELD; a quantized leaf inserts the bytes field there and pushes the
+ // pointer out, so it has at least LEAF_QUANTIZED_BYTES_FIELD + 2 fields (bytes + trailing ptr).
+ if (frameTuple.getFieldCount() < VTreeStaticTupleConstants.LEAF_QUANTIZED_BYTES_FIELD + 2) {
+ // Non-quantized tuple has no quantized-bytes field.
+ return null;
+ }
+ // Quantized bytes live in field 2 of the quantized layout (>=4 fields):
+ // [cid, embedding, quantizedBytes, (neighborList,) metadataPtr].
+ int fieldIndex = VTreeStaticTupleConstants.LEAF_QUANTIZED_BYTES_FIELD;
+ byte[] fieldData = frameTuple.getFieldData(fieldIndex);
+ int fieldStart = frameTuple.getFieldStart(fieldIndex);
+ // The field is a ByteArraySerializerDeserializer payload: a VarLen length prefix then the content
+ // bytes. Read the length via ByteArrayPointable and copy the content directly, avoiding the per-tuple
+ // DataInputStream/ByteArrayInputStream allocation on the search path.
+ int contentLength = ByteArrayPointable.getContentLength(fieldData, fieldStart);
+ int metaLength = ByteArrayPointable.getNumberBytesToStoreMeta(contentLength);
+ return Arrays.copyOfRange(fieldData, fieldStart + metaLength, fieldStart + metaLength + contentLength);
+ }
+
+ @Override
+ public String printHeader() {
+ StringBuilder strBuilder = new StringBuilder(super.printHeader());
+ strBuilder.append("nextLeaf: ").append(getNextLeaf()).append('\n');
+ return strBuilder.toString();
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/frames/VTreeLeafFrameFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/frames/VTreeLeafFrameFactory.java
new file mode 100644
index 0000000000..5066a778be
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/frames/VTreeLeafFrameFactory.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.vector.frames;
+
+import org.apache.hyracks.api.dataflow.value.ITypeTraits;
+import org.apache.hyracks.storage.am.common.api.INullIntrospector;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriterFactory;
+import org.apache.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
+import org.apache.hyracks.storage.am.vector.api.IVTreeLeafFrame;
+import org.apache.hyracks.storage.am.vector.utils.VTreeStaticTupleAccessor;
+
+/**
+ * Factory for {@link VTreeLeafFrame} instances. {@code centroidDimensions} is retained for
+ * future per-factory dimension validation.
+ *
+ * <p>Leaf tuples are NOT caller-parameterized — the schema is determined entirely by whether
+ * quantization is enabled:
+ * <ul>
+ * <li>Quantized (production): {@code <cluster_id, centroid, quantized_embedding, child_page_pointer>}</li>
+ * <li>Non-quantized (test fixtures only): {@code <cluster_id, centroid, child_page_pointer>}</li>
+ * </ul>
+ */
+public class VTreeLeafFrameFactory implements ITreeIndexFrameFactory {
+
+ private static final long serialVersionUID = 1L;
+ private final ITreeIndexTupleWriter tupleWriter;
+ private final int centroidDimensions;
+
+ public VTreeLeafFrameFactory(int centroidDimensions, boolean quantized, ITypeTraits nullTypeTraits,
+ INullIntrospector nullIntrospector) {
+ this.centroidDimensions = centroidDimensions;
+ ITypeTraits[] schema = VTreeStaticTupleAccessor.leafTypeTraits(quantized);
+ this.tupleWriter =
+ new TypeAwareTupleWriterFactory(schema, nullTypeTraits, nullIntrospector).createTupleWriter();
+ }
+
+ @Override
+ public IVTreeLeafFrame createFrame() {
+ return new VTreeLeafFrame(tupleWriter);
+ }
+
+ @Override
+ public ITreeIndexTupleWriterFactory getTupleWriterFactory() {
+ return null;
+ }
+
+ public ITreeIndexTupleWriter getTupleWriter() {
+ return tupleWriter;
+ }
+
+ public int getCentroidDimensions() {
+ return centroidDimensions;
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/frames/VTreeMetadataFrame.java b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/frames/VTreeMetadataFrame.java
new file mode 100644
index 0000000000..62e6d98c6d
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/frames/VTreeMetadataFrame.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.vector.frames;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.primitive.DoublePointable;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.dataflow.common.utils.TupleUtils;
+import org.apache.hyracks.storage.am.btree.frames.OrderedSlotManager;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
+import org.apache.hyracks.storage.am.vector.api.IVTreeMetadataFrame;
+import org.apache.hyracks.storage.am.vector.utils.VTreeMetadataTupleAccessor;
+
+/**
+ * VTree metadata frame.
+ * <p>
+ * Page layout: base header (from {@link VTreeNSMFrame}) followed by a 4-byte next-metadata-page pointer
+ * (sentinel {@code -1}). Tuples have the format {@code <max_distance, data_page_pointer>} and are kept sorted
+ * by {@code max_distance} ascending so that {@code findInsertPosition} and bounded scans can use binary search.
+ */
+public class VTreeMetadataFrame extends VTreeNSMFrame implements IVTreeMetadataFrame {
+
+ // Offset (bytes from page start) of the 4-byte next-metadata-page pointer.
+ private static final int NEXT_PAGE_OFFSET = CENTROID_ID_OFFSET + Integer.BYTES;
+
+ public VTreeMetadataFrame(ITreeIndexTupleWriter tupleWriter) {
+ super(tupleWriter, new OrderedSlotManager());
+ }
+
+ @Override
+ public void initBuffer(byte level) {
+ super.initBuffer(level);
+ buf.putInt(NEXT_PAGE_OFFSET, -1);
+ }
+
+ @Override
+ public int getPageHeaderSize() {
+ return NEXT_PAGE_OFFSET + Integer.BYTES;
+ }
+
+ @Override
+ public void setNextPage(int nextPage) {
+ buf.putInt(NEXT_PAGE_OFFSET, nextPage);
+ }
+
+ @Override
+ public int getNextPage() {
+ return buf.getInt(NEXT_PAGE_OFFSET);
+ }
+
+ @Override
+ public double getMaxDistance(int tupleIndex) throws HyracksDataException {
+ frameTuple.resetByTupleIndex(this, tupleIndex);
+ return VTreeMetadataTupleAccessor.getMaxDistance(frameTuple);
+ }
+
+ @Override
+ public int getDataPagePointer(int tupleIndex) throws HyracksDataException {
+ frameTuple.resetByTupleIndex(this, tupleIndex);
+ return VTreeMetadataTupleAccessor.getDataPagePointer(frameTuple);
+ }
+
+ /**
+ * Overwrite the {@code max_distance} field of an existing tuple in place. Caller must preserve the
+ * sort invariant on {@code max_distance}.
+ */
+ public void updateMaxDistance(int tupleIndex, double newMaxDistance) throws HyracksDataException {
+ frameTuple.resetByTupleIndex(this, tupleIndex);
+ int f = VTreeMetadataTupleAccessor.MAX_DISTANCE_FIELD;
+ DoublePointable.setDouble(frameTuple.getFieldData(f), frameTuple.getFieldStart(f), newMaxDistance);
+ }
+
+ /**
+ * Binary-search the leftmost insertion index that keeps {@code max_distance} sorted ascending.
+ */
+ public int findInsertPosition(double maxDistance) throws HyracksDataException {
+ int left = 0;
+ int right = getTupleCount();
+ while (left < right) {
+ int mid = (left + right) / 2;
+ double midMaxDistance = getMaxDistance(mid);
+ if (midMaxDistance < maxDistance) {
+ left = mid + 1;
+ } else {
+ right = mid;
+ }
+ }
+ return left;
+ }
+
+ /**
+ * Split a full metadata frame into two halves and insert {@code tuple} into whichever side preserves the
+ * {@code max_distance}-ascending invariant. Uses a copy-and-reinitialize strategy (rather than in-place
+ * deletes) to avoid page fragmentation.
+ */
+ public void split(IVTreeMetadataFrame rightFrame, ITupleReference tuple) throws HyracksDataException {
+ int tupleCount = getTupleCount();
+ int splitIndex = tupleCount / 2;
+
+ // Snapshot existing tuples (copyTuple returns an ArrayTupleReference compatible with tupleWriter.writeTuple).
+ ITupleReference[] leftTuples = new ITupleReference[splitIndex];
+ for (int i = 0; i < splitIndex; i++) {
+ frameTuple.resetByTupleIndex(this, i);
+ leftTuples[i] = TupleUtils.copyTuple(frameTuple);
+ }
+ ITupleReference[] rightTuples = new ITupleReference[tupleCount - splitIndex];
+ for (int i = splitIndex; i < tupleCount; i++) {
+ frameTuple.resetByTupleIndex(this, i);
+ rightTuples[i - splitIndex] = TupleUtils.copyTuple(frameTuple);
+ }
+
+ initBuffer((byte) 0);
+ rightFrame.initBuffer((byte) 0);
+
+ for (int i = 0; i < leftTuples.length; i++) {
+ insert(leftTuples[i], i);
+ }
+ for (int i = 0; i < rightTuples.length; i++) {
+ rightFrame.insert(rightTuples[i], i);
+ }
+
+ // Route the new tuple to the side whose range covers its max_distance.
+ double newMaxDistance = extractMaxDistanceFromTuple(tuple);
+ if (getTupleCount() == 0 || newMaxDistance <= getMaxDistance(getTupleCount() - 1)) {
+ insert(tuple, findInsertPosition(newMaxDistance));
+ } else {
+ rightFrame.insert(tuple, ((VTreeMetadataFrame) rightFrame).findInsertPosition(newMaxDistance));
+ }
+ }
+
+ private double extractMaxDistanceFromTuple(ITupleReference tuple) {
+ return VTreeMetadataTupleAccessor.getMaxDistance(tuple);
+ }
+
+ @Override
+ public String printHeader() {
+ StringBuilder strBuilder = new StringBuilder(super.printHeader());
+ strBuilder.append("nextPage: ").append(getNextPage()).append('\n');
+ return strBuilder.toString();
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/frames/VTreeMetadataFrameFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/frames/VTreeMetadataFrameFactory.java
new file mode 100644
index 0000000000..c2fb2baa9d
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/frames/VTreeMetadataFrameFactory.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.vector.frames;
+
+import org.apache.hyracks.api.dataflow.value.ITypeTraits;
+import org.apache.hyracks.storage.am.common.api.INullIntrospector;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriterFactory;
+import org.apache.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
+import org.apache.hyracks.storage.am.vector.api.IVTreeMetadataFrame;
+import org.apache.hyracks.storage.am.vector.utils.VTreeMetadataTupleAccessor;
+
+/**
+ * Factory for {@link VTreeMetadataFrame} instances. {@code centroidDimensions} is retained for
+ * future per-factory dimension validation.
+ *
+ * <p>Metadata tuples are NOT caller-parameterized — every metadata frame uses the same fixed
+ * two-field layout {@code <max_distance_to_centroid, data_page_pointer>}, sorted by
+ * {@code max_distance_to_centroid} ascending.
+ */
+public class VTreeMetadataFrameFactory implements ITreeIndexFrameFactory {
+
+ private static final long serialVersionUID = 1L;
+ private final ITreeIndexTupleWriter tupleWriter;
+ private final int centroidDimensions;
+
+ public VTreeMetadataFrameFactory(int centroidDimensions, ITypeTraits nullTypeTraits,
+ INullIntrospector nullIntrospector) {
+ this.centroidDimensions = centroidDimensions;
+ this.tupleWriter = new TypeAwareTupleWriterFactory(VTreeMetadataTupleAccessor.typeTraits(), nullTypeTraits,
+ nullIntrospector).createTupleWriter();
+ }
+
+ @Override
+ public IVTreeMetadataFrame createFrame() {
+ return new VTreeMetadataFrame(tupleWriter);
+ }
+
+ @Override
+ public ITreeIndexTupleWriterFactory getTupleWriterFactory() {
+ return null;
+ }
+
+ public ITreeIndexTupleWriter getTupleWriter() {
+ return tupleWriter;
+ }
+
+ public int getCentroidDimensions() {
+ return centroidDimensions;
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/frames/VTreeNSMFrame.java b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/frames/VTreeNSMFrame.java
new file mode 100644
index 0000000000..6574156cb4
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/frames/VTreeNSMFrame.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.vector.frames;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.storage.am.common.api.ISlotManager;
+import org.apache.hyracks.storage.am.common.api.ISplitKey;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleReference;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
+import org.apache.hyracks.storage.am.common.frames.FrameOpSpaceStatus;
+import org.apache.hyracks.storage.am.common.frames.TreeIndexNSMFrame;
+import org.apache.hyracks.storage.am.vector.api.IVTreeFrame;
+import org.apache.hyracks.storage.common.MultiComparator;
+import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.buffercache.IExtraPageBlockHelper;
+
+/**
+ * Common NSM (N-ary Storage Model) base for VTree page frames.
+ * <p>
+ * Page header layout extends {@link TreeIndexNSMFrame}: after the inherited reserved header bytes come a
+ * 4-byte cluster ID and a 4-byte centroid ID. Subclasses append their own header fields after
+ * {@link #CENTROID_ID_OFFSET} + {@link Integer#BYTES} and override {@link #getPageHeaderSize()} accordingly.
+ * <p>
+ * NOTE: these two <em>per-page header</em> fields are currently <em>reserved</em>: {@code cluster_id}
+ * is written as the {@code -1} sentinel and read back only by the debug {@link #printHeader()}, and
+ * {@code centroid_id} is never written (also read only by {@code printHeader}). They are retained (not
+ * removed) because dropping them would shift every subclass header offset — an on-disk format change
+ * that is out of scope here.
+ * <p>
+ * This header {@code centroid_id} slot is <em>distinct</em> from the per-tuple centroid id carried in
+ * each interior/leaf/data tuple (read via {@code IVTreeLeafFrame#getCentroidId(int)} /
+ * {@code VTreeBulkLoader.extractCentroidId}). That per-tuple id is live and load-bearing — e.g. the
+ * bulk loader compares it against the current cluster to detect a cluster boundary and route records —
+ * and is not affected by the reserved status of this header slot.
+ */
+
+public abstract class VTreeNSMFrame extends TreeIndexNSMFrame implements IVTreeFrame {
+
+ // Offset of the 4-byte cluster ID field (sentinel -1 = unassigned).
+ protected static final int CLUSTER_ID_OFFSET = TreeIndexNSMFrame.RESERVED_HEADER_SIZE;
+ // Offset of the 4-byte centroid ID field. Subclass headers extend from CENTROID_ID_OFFSET + Integer.BYTES.
+ protected static final int CENTROID_ID_OFFSET = CLUSTER_ID_OFFSET + Integer.BYTES;
+
+ protected MultiComparator cmp;
+
+ public VTreeNSMFrame(ITreeIndexTupleWriter tupleWriter, ISlotManager slotManager) {
+ // frameTuple is declared and initialized (tupleWriter.createTupleReference()) by TreeIndexNSMFrame;
+ // no need to redeclare/shadow it here.
+ super(tupleWriter, slotManager);
+ }
+
+ @Override
+ public void initBuffer(byte level) {
+ super.initBuffer(level);
+ buf.putInt(CLUSTER_ID_OFFSET, -1);
+ }
+
+ @Override
+ public int getPageHeaderSize() {
+ return CENTROID_ID_OFFSET + Integer.BYTES;
+ }
+
+ @Override
+ public ITreeIndexTupleReference createTupleReference() {
+ return tupleWriter.createTupleReference();
+ }
+
+ @Override
+ public void setMultiComparator(MultiComparator cmp) {
+ this.cmp = cmp;
+ }
+
+ @Override
+ public int getBytesRequiredToWriteTuple(ITupleReference tuple) {
+ return tupleWriter.bytesRequired(tuple) + slotManager.getSlotSize();
+ }
+
+ @Override
+ public void insertSorted(ITupleReference tuple) {
+ insert(tuple, getTupleCount());
+ }
+
+ @Override
+ public FrameOpSpaceStatus hasSpaceInsert(ITupleReference tuple) throws HyracksDataException {
+ // One correct definition shared by all VTree frames. The former override in VTreeDataFrame was
+ // the only variant ever exercised (it is the sole frame type hasSpaceInsert is called on); this
+ // base version had the checks in the wrong order and mislabeled results — total (possibly
+ // fragmented) free space was reported as SUFFICIENT_CONTIGUOUS_SPACE, which would let a caller
+ // insert without compacting and overrun the contiguous region. Interior/leaf/metadata frames
+ // inherit this, so keep it correct here rather than relying on an override.
+ int bytesRequired = tupleWriter.bytesRequired(tuple);
+ int slotSize = slotManager.getSlotSize();
+ // Contiguous space available without compaction: gap between the tuple-data high-water mark
+ // (free-space offset) and the slot array growing down from the end of the page.
+ int contiguousFree = buf.capacity() - getFreeSpaceOff() - getTupleCount() * slotSize;
+ if (bytesRequired + slotSize <= contiguousFree) {
+ return FrameOpSpaceStatus.SUFFICIENT_CONTIGUOUS_SPACE;
+ }
+ // Space available only after compacting reclaimable (deleted-tuple) free space.
+ if (bytesRequired + slotSize <= buf.getInt(TOTAL_FREE_SPACE_OFFSET)) {
+ return FrameOpSpaceStatus.SUFFICIENT_SPACE;
+ }
+ return FrameOpSpaceStatus.INSUFFICIENT_SPACE;
+ }
+
+ @Override
+ public void split(ITreeIndexFrame rightFrame, ITupleReference tuple, ISplitKey splitKey,
+ IExtraPageBlockHelper extraPageBlockHelper, IBufferCache bufferCache) throws HyracksDataException {
+ // Generic split is not supported on VTree frames; subclasses that split (e.g. data/metadata frames)
+ // expose their own split(...) entry points with the right argument types.
+ throw new HyracksDataException("Split operation not implemented for " + this.getClass().getSimpleName());
+ }
+
+ @Override
+ public String printHeader() {
+ StringBuilder strBuilder = new StringBuilder(super.printHeader());
+ strBuilder.append("clusterId: ").append(buf.getInt(CLUSTER_ID_OFFSET)).append('\n');
+ strBuilder.append("centroidId: ").append(buf.getInt(CENTROID_ID_OFFSET)).append('\n');
+ return strBuilder.toString();
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/impls/ClusterSearchResult.java b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/impls/ClusterSearchResult.java
new file mode 100644
index 0000000000..2c0df0b09e
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/impls/ClusterSearchResult.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.vector.impls;
+
+/**
+ * Result of cluster search operation.
+ * Contains all information needed to open a cluster without re-traversing the tree.
+ */
+public class ClusterSearchResult {
+ public final int leafPageId;
+ public final int clusterIndex;
+ /**
+ * The centroid vector. This array is shared (not defensively copied) and callers must treat it
+ * as read-only; mutating it corrupts the source this result was derived from.
+ */
+ public final double[] centroid;
+
+ public final double distance;
+ public final int centroidId;
+ public final long directoryPageId; // Direct pointer to cluster's directory/metadata page
+ public final double quantizedDistance; // Distance computed on quantized representations (NaN = not computed)
+
+ ClusterSearchResult(int leafPageId, int clusterIndex, double[] centroid, double distance, int centroidId,
+ long directoryPageId, double quantizedDistance) {
+ this.leafPageId = leafPageId;
+ this.clusterIndex = clusterIndex;
+ this.centroid = centroid;
+ this.distance = distance;
+ this.centroidId = centroidId;
+ this.directoryPageId = directoryPageId;
+ this.quantizedDistance = quantizedDistance;
+ }
+
+ /**
+ * Factory method to create ClusterSearchResult instances.
+ *
+ * @param leafPageId The leaf page ID
+ * @param clusterIndex The cluster index within the page
+ * @param centroid The centroid vector
+ * @param distance The distance to the centroid
+ * @param centroidId The centroid ID
+ * @param directoryPageId The directory/metadata page ID for this cluster
+ * @return New ClusterSearchResult instance
+ */
+ public static ClusterSearchResult create(int leafPageId, int clusterIndex, double[] centroid, double distance,
+ int centroidId, long directoryPageId) {
+ return new ClusterSearchResult(leafPageId, clusterIndex, centroid, distance, centroidId, directoryPageId,
+ Double.NaN);
+ }
+
+ /**
+ * Factory method with quantized distance.
+ *
+ * @param leafPageId The leaf page ID
+ * @param clusterIndex The cluster index within the page
+ * @param centroid The centroid vector
+ * @param distance The distance to the centroid
+ * @param centroidId The centroid ID
+ * @param directoryPageId The directory/metadata page ID for this cluster
+ * @param quantizedDistance Distance computed on quantized representations (NaN if not computed)
+ * @return New ClusterSearchResult instance
+ */
+ public static ClusterSearchResult create(int leafPageId, int clusterIndex, double[] centroid, double distance,
+ int centroidId, long directoryPageId, double quantizedDistance) {
+ return new ClusterSearchResult(leafPageId, clusterIndex, centroid, distance, centroidId, directoryPageId,
+ quantizedDistance);
+ }
+
+ /**
+ * Legacy factory method for backwards compatibility.
+ * Creates a result with directoryPageId = -1 (unknown).
+ */
+ public static ClusterSearchResult create(int leafPageId, int clusterIndex, double[] centroid, double distance,
+ int centroidId) {
+ return new ClusterSearchResult(leafPageId, clusterIndex, centroid, distance, centroidId, -1, Double.NaN);
+ }
+
+ /**
+ * Check if this result has a valid quantized distance.
+ * @return true if quantizedDistance was computed (not NaN)
+ */
+ public boolean hasQuantizedDistance() {
+ return !Double.isNaN(quantizedDistance);
+ }
+}
\ No newline at end of file
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/impls/VTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/impls/VTree.java
new file mode 100644
index 0000000000..c3d28e1d96
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/impls/VTree.java
@@ -0,0 +1,1387 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.vector.impls;
+
+import static org.apache.hyracks.storage.common.buffercache.context.read.DefaultBufferCacheReadContextProvider.NEW;
+
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import org.apache.hyracks.api.exceptions.ErrorCode;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.data.std.primitive.LongPointable;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.storage.am.common.api.IPageManager;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexAccessor;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexCursor;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexMetadataFrame;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
+import org.apache.hyracks.storage.am.common.frames.FrameOpSpaceStatus;
+import org.apache.hyracks.storage.am.common.impls.AbstractTreeIndex;
+import org.apache.hyracks.storage.am.common.impls.TreeIndexDiskOrderScanCursor;
+import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
+import org.apache.hyracks.storage.am.vector.api.IVTreeBinaryAccessorFactory;
+import org.apache.hyracks.storage.am.vector.api.IVTreeDataFrame;
+import org.apache.hyracks.storage.am.vector.api.IVTreeDataTupleBuilderFactory;
+import org.apache.hyracks.storage.am.vector.api.IVTreeDistanceFunction;
+import org.apache.hyracks.storage.am.vector.api.IVTreeDistanceFunctionFactory;
+import org.apache.hyracks.storage.am.vector.api.IVTreeMetadataFrame;
+import org.apache.hyracks.storage.am.vector.api.IVTreeQuantizer;
+import org.apache.hyracks.storage.am.vector.api.IVTreeQuantizerFactory;
+import org.apache.hyracks.storage.am.vector.api.VTreeQuantizationParams;
+import org.apache.hyracks.storage.am.vector.frames.VTreeDataFrame;
+import org.apache.hyracks.storage.am.vector.frames.VTreeMetadataFrame;
+import org.apache.hyracks.storage.am.vector.tuples.VTreeTupleUtils;
+import org.apache.hyracks.storage.am.vector.utils.CrossPollinationConfig;
+import org.apache.hyracks.storage.am.vector.utils.RngAcceptanceFilter;
+import org.apache.hyracks.storage.am.vector.utils.VTreeDataTupleAccessor;
+import org.apache.hyracks.storage.am.vector.utils.VTreeMetadataKeys;
+import org.apache.hyracks.storage.am.vector.utils.VTreeMetadataTupleAccessor;
+import org.apache.hyracks.storage.am.vector.utils.VTreeNavigationUtils;
+import org.apache.hyracks.storage.common.IIndexAccessParameters;
+import org.apache.hyracks.storage.common.IIndexBulkLoader;
+import org.apache.hyracks.storage.common.IIndexCursor;
+import org.apache.hyracks.storage.common.ISearchPredicate;
+import org.apache.hyracks.storage.common.ISketchSampler;
+import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.buffercache.ICachedPage;
+import org.apache.hyracks.storage.common.buffercache.IPageWriteCallback;
+import org.apache.hyracks.storage.common.file.BufferedFileHandle;
+import org.apache.hyracks.util.annotations.AiProvenance;
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+/**
+ * Vector Clustering Tree implementation for multi-level k-means vector index.
+ *
+ * This tree supports hierarchical vector clustering with specialized frame types:
+ * - Interior frames: Store cluster
+ * centroids and child page pointers
+ * - Leaf frames: Store cluster centroids and metadata page pointers
+ * - Metadata frames: Store max distances and data page pointers
+ * - Data frames: Store distances, primary keys
+ */
+public class VTree extends AbstractTreeIndex {
+
+ private static final Logger LOGGER = LogManager.getLogger();
+
+ private final int vectorDimensions;
+ private final ITreeIndexFrameFactory metadataFrameFactory;
+ private final ITreeIndexFrameFactory dataFrameFactory;
+ private final IVTreeBinaryAccessorFactory vectorAccessorFactory;
+ private final IVTreeDataTupleBuilderFactory dataTupleBuilderFactory;
+ // Raw quantization params: {minQuantile, maxQuantile, alpha, confidenceInterval, bits, sampleCount}
+ // null = non-quantized index.
+ private final VTreeQuantizationParams quantizationParams;
+ private final IVTreeDistanceFunctionFactory distanceFunctionFactory;
+ // Distance function for this index's metric, used by the write/clustering path; search builds
+ // per-query functions from the factory.
+ private final IVTreeDistanceFunction distanceFunction;
+ // Cross-pollination placement parameters (M=1 = legacy single-closest). Threaded from the index
+ // WITH clause so incremental insert/delete replicate into the same leaf clusters as bulk-load.
+ private final CrossPollinationConfig crossPollination;
+
+ // Static-structure navigation state (memory components only). Threading contract: this group is
+ // written exactly once by setStaticStructure() (idempotent via the `initialized` guard) while a
+ // memory component is being allocated or recycled — i.e. before that component is published for
+ // operations — and is read-only thereafter until resetInitialization() clears it for the next
+ // recycle. Publication to the threads that later run searches/inserts on the component is provided
+ // by the LSM harness's operation-tracker happens-before (a thread must enter the component through
+ // the tracker before touching it), the same mechanism that publishes all other in-memory component
+ // state (e.g. BTree memory frames). These fields are therefore intentionally NOT volatile: adding
+ // volatility would tax the per-navigation read of staticBufferCache without adding a guarantee the
+ // harness does not already give. Do not read this group outside that established happens-before.
+ private boolean initialized = false;
+
+ // For memory components: reference to static structure for navigation
+ private IBufferCache staticBufferCache;
+ private int staticFileId;
+ private int staticRootPage;
+
+ // Centroid-to-directory-page mapping (memory components only)
+ private int[] centroidDirPageMap; // centroidIndex -> VBC directory page ID
+ private int firstLeafCentroidIdMem;
+ private int numLeafCentroidMem;
+
+ public VTree(IBufferCache bufferCache, IPageManager freePageManager, ITreeIndexFrameFactory interiorFrameFactory,
+ ITreeIndexFrameFactory leafFrameFactory, ITreeIndexFrameFactory metadataFrameFactory,
+ ITreeIndexFrameFactory dataFrameFactory, IBinaryComparatorFactory[] cmpFactories, int fieldCount,
+ int vectorDimensions, FileReference file, IVTreeBinaryAccessorFactory vectorAccessorFactory,
+ IVTreeDataTupleBuilderFactory dataTupleBuilderFactory, VTreeQuantizationParams quantizationParams,
+ IVTreeDistanceFunctionFactory distanceFunctionFactory, CrossPollinationConfig crossPollination)
+ throws HyracksDataException {
+ super(bufferCache, freePageManager, interiorFrameFactory, leafFrameFactory, cmpFactories, fieldCount, file);
+ this.vectorDimensions = vectorDimensions;
+ this.metadataFrameFactory = metadataFrameFactory;
+ this.dataFrameFactory = dataFrameFactory;
+ this.vectorAccessorFactory = vectorAccessorFactory;
+ this.dataTupleBuilderFactory = dataTupleBuilderFactory;
+ this.quantizationParams = quantizationParams;
+ this.distanceFunctionFactory = distanceFunctionFactory;
+ this.distanceFunction = distanceFunctionFactory.createDistanceFunction();
+ this.crossPollination = crossPollination != null ? crossPollination : CrossPollinationConfig.LEGACY;
+ }
+
+ /**
+ * Get the data frame factory for creating data frames.
+ *
+ * @return the data frame factory
+ */
+ public ITreeIndexFrameFactory getDataFrameFactory() {
+ return dataFrameFactory;
+ }
+
+ /**
+ * Get the metadata frame factory for creating metadata frames.
+ *
+ * @return the metadata frame factory
+ */
+ public ITreeIndexFrameFactory getMetadataFrameFactory() {
+ return metadataFrameFactory;
+ }
+
+ @Override
+ public ITreeIndexAccessor createAccessor(IIndexAccessParameters iap) {
+ return new VTreeAccessor(this, iap);
+ }
+
+ @Override
+ public int getNumOfFilterFields() {
+ return 0;
+ }
+
+ public IIndexBulkLoader createComponentBulkLoader(IPageWriteCallback callback, ITreeIndexAccessor staticAccessor,
+ ISketchSampler sampler) throws HyracksDataException {
+ return new VTreeBulkLoader(callback, this, staticAccessor, sampler, null);
+ }
+
+ /**
+ * Variant that writes data frames through the supplied frame factory instead of this tree's
+ * default. Used by merges to install a polarity-preserving (copy) tuple writer; matter/delete-marker
+ * semantics are defined in the LSM layer (see hyracks-storage-am-lsm-vtree).
+ */
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_FABLE_5, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.ASSISTED)
+ public IIndexBulkLoader createComponentBulkLoader(IPageWriteCallback callback, ITreeIndexAccessor staticAccessor,
+ ISketchSampler sampler, ITreeIndexFrameFactory dataFrameFactoryOverride) throws HyracksDataException {
+ return new VTreeBulkLoader(callback, this, staticAccessor, sampler, dataFrameFactoryOverride);
+ }
+
+ @Override
+ public IIndexBulkLoader createBulkLoader(float fillFactor, boolean verifyInput, long numElementsHint,
+ boolean checkIfEmptyIndex, ISketchSampler sampler, IPageWriteCallback callback)
+ throws HyracksDataException {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * Creates a bulk loader for building the static hierarchical clustering structure.
+ * This is called during index creation to build the multi-level centroid tree.
+ *
+ * @param numLevels Number of levels in the hierarchy
+ * @param clustersPerLevel List of cluster counts per level
+ * @param centroidsPerCluster List of centroids per cluster per level
+ * @param maxEntriesPerPage Maximum entries per page
+ * @param callback Page write callback
+ * @return IIndexBulkLoader for building static structure
+ */
+ public IIndexBulkLoader createStaticStructureBulkLoader(int numLevels, List<Integer> clustersPerLevel,
+ List<List<Integer>> centroidsPerCluster, int maxEntriesPerPage, IPageWriteCallback callback)
+ throws HyracksDataException {
+ return new VTreeStaticStructureBuilder(callback, this, numLevels, clustersPerLevel, centroidsPerCluster,
+ maxEntriesPerPage);
+ }
+
+ /**
+ * Insert a vector into the clustering tree. The vector tuple contains: <vector_embedding, primary_key,
+ * [additional_fields]>
+ */
+ private void insertVector(ITupleReference tuple, VTreeOpContext ctx) throws HyracksDataException {
+ double[] vector = VTreeTupleUtils.extractVectorFromTuple(tuple, 0, vectorAccessorFactory);
+ if (vector == null) {
+ // A tuple with no extractable vector is corrupt/unexpected; ILLEGAL_STATE carries the message.
+ throw HyracksDataException.create(ErrorCode.ILLEGAL_STATE, "Failed to extract vector from tuple");
+ }
+
+ // Cross-pollination: write one replica per accepted cluster (M=1 = single closest cluster).
+ for (ClusterSearchResult clusterResult : findReplicaClusters(vector)) {
+ try (ClusterAccessResult accessResult = prepareClusterAccess(clusterResult, ctx)) {
+ // Distance is to THIS cluster's centroid so each replica's stored key is self-consistent.
+ double distance = distanceFunction.apply(vector, clusterResult.centroid);
+ insertIntoDataPages(accessResult.metadataPageId(), vector, distance, clusterResult.centroidId, tuple,
+ ctx);
+ }
+ }
+ }
+
+ /**
+ * Insert vector data into data pages via metadata pages. This method traverses through all linked metadata pages to
+ * find the appropriate data page.
+ */
+ private void insertIntoDataPages(long metadataPageId, double[] vector, double distance, int centroidId,
+ ITupleReference originalTuple, VTreeOpContext ctx) throws HyracksDataException {
+
+ // Traverse through all linked directory (metadata) pages to find the appropriate data page.
+ // Guard against a corrupted next-page chain that loops back on itself by tracking the page
+ // ids already visited; a repeat is a genuine cycle, not just a long-but-valid chain.
+ //
+ // Concurrency: this is a forward-only walk that write-latches one directory page at a time and
+ // releases it before pinning the next (it does not hold the whole chain). That is safe because
+ // the directory chain is globally max_distance-ascending and only ever grows by in-place split
+ // (VTreeMetadataFrame keeps entries sorted; handleMetadataPageOverflow moves the upper entries to
+ // a new page linked *after* the current one — pages are never removed or reordered). Each page is
+ // read under its own latch, so the walker sees a consistent snapshot per page. Since this walker
+ // advanced past page N only because the record's distance exceeded every entry on N, a concurrent
+ // split of N (which can only relocate entries <= N's max to a new page inserted between N and its
+ // old successor) cannot hold this record's band — so following N's already-read successor pointer
+ // never skips the correct page. The walker therefore always converges on the right band by moving
+ // forward. tryPhysicalDelete relies on the same invariant.
+ long currentMetadataPageId = metadataPageId;
+ Set<Long> visitedMetadataPageIds = new HashSet<>();
+
+ while (currentMetadataPageId != -1) {
+ if (!visitedMetadataPageIds.add(currentMetadataPageId)) {
+ throw HyracksDataException.create(ErrorCode.ILLEGAL_STATE,
+ "Cycle detected in directory page chain starting at page " + metadataPageId
+ + " (revisited page " + currentMetadataPageId + ")");
+ }
+ ICachedPage metadataPage =
+ bufferCache.pin(BufferedFileHandle.getDiskPageId(getFileId(), (int) currentMetadataPageId));
+ ctx.setMetadataPageId(currentMetadataPageId);
+ boolean latched = false;
+ try {
+ metadataPage.acquireWriteLatch();
+ latched = true;
+ ctx.getMetadataFrame().setPage(metadataPage);
+
+ // Determine if this is the last directory page in the chain
+ int nextMetadataPageId = ctx.getMetadataFrame().getNextPage();
+ boolean isLastInChain = (nextMetadataPageId == VTreeDataTupleAccessor.NO_NEXT_PAGE);
+
+ // Try to find appropriate data page based on distance.
+ // Only uses catch-all (last data page) on the last directory page;
+ // otherwise returns -1 so we traverse to the next directory page.
+ long targetDataPageId = findDataPageInMetadataPage(ctx.getMetadataFrame(), distance, isLastInChain);
+
+ if (targetDataPageId != -1) {
+ // Found appropriate data page - try to insert
+ boolean inserted =
+ tryInsertIntoDataPage(targetDataPageId, vector, distance, centroidId, originalTuple, ctx);
+
+ if (inserted) {
+ return; // Successfully inserted
+ }
+
+ // If insert failed due to space, we need to handle overflow
+ handleDataPageOverflow(currentMetadataPageId, vector, distance, centroidId, originalTuple, ctx);
+ return;
+ }
+
+ // No match on this directory page
+ if (isLastInChain) {
+ // Last page in chain - create new data page
+ handleDataPageOverflow(currentMetadataPageId, vector, distance, centroidId, originalTuple, ctx);
+ return;
+ }
+
+ // Traverse to next directory page
+ currentMetadataPageId = nextMetadataPageId;
+
+ } finally {
+ if (latched) {
+ metadataPage.releaseWriteLatch(true);
+ }
+ bufferCache.unpin(metadataPage);
+ }
+ }
+ }
+
+ /**
+ * Find the appropriate data page in a specific metadata page based on distance. This searches for a data page that
+ * can accommodate the given distance.
+ *
+ * Returns last data page as catch-all when distance > all max_distance values.
+ * This is needed for BOTH insertion and deletion:
+ * - Matter insertion: Vectors with distance > all max values go into last page (catch-all)
+ * - Delete-marker tuple insertion: Same as matter insertion - uses last page as catch-all
+ * - Physical deletion: To find those vectors, we must check the last page (same catch-all)
+ *
+ * The last page dynamically expands and metadata max_distance is updated automatically
+ * via updateMetadataMaxDistanceIfNeeded() in the insertion path.
+ *
+ * @param metadataFrame The metadata frame to search
+ * @param distance The distance to search for
+ * @return Data page ID, or -1 if metadata is empty
+ */
+ private long findDataPageInMetadataPage(IVTreeMetadataFrame metadataFrame, double distance, boolean isLastInChain)
+ throws HyracksDataException {
+
+ int tupleCount = metadataFrame.getTupleCount();
+
+ // Entries are kept sorted by max_distance ascending (VTreeMetadataFrame invariant, maintained
+ // by findInsertPosition on every metadata insert). Binary-search the first entry whose
+ // max_distance >= distance (i.e. the first data page whose range covers this distance) instead
+ // of an O(n) scan. This matches the previous linear "first distance <= maxDistance" result.
+ int lo = 0;
+ int hi = tupleCount; // half-open [lo, hi)
+ while (lo < hi) {
+ int mid = (lo + hi) >>> 1;
+ if (metadataFrame.getMaxDistance(mid) >= distance) {
+ hi = mid;
+ } else {
+ lo = mid + 1;
+ }
+ }
+ if (lo < tupleCount) {
+ return metadataFrame.getDataPagePointer(lo);
+ }
+
+ // Only use catch-all (last data page) on the last directory page in the chain.
+ // For non-last pages, return -1 so the caller traverses to the next directory page.
+ if (isLastInChain && tupleCount > 0) {
+ return metadataFrame.getDataPagePointer(tupleCount - 1);
+ }
+
+ return -1; // No match on this page (or empty)
+
+ }
+
+ /**
+ * Try to insert into a specific data page. Returns true if successful, false if page is full.
+ */
+ private boolean tryInsertIntoDataPage(long dataPageId, double[] vector, double distance, int centroidId,
+ ITupleReference originalTuple, VTreeOpContext ctx) throws HyracksDataException {
+
+ ICachedPage dataPage = bufferCache.pin(BufferedFileHandle.getDiskPageId(getFileId(), (int) dataPageId));
+
+ boolean latched = false;
+ try {
+ dataPage.acquireWriteLatch();
+ latched = true;
+ ctx.getDataFrame().setPage(dataPage);
+
+ // Create data tuple: <distance, centroidId, vector, PK>
+ // Pass context so buildDataTuple can check operation type and encode a delete-polarity tuple
+ // if DELETE (the encoding is decided by the caller-supplied frame's tuple writer)
+ ITupleReference dataTuple =
+ ctx.getDataTupleBuilder().buildDataTuple(vector, distance, centroidId, originalTuple);
+
+ // Check if there's space for the tuple
+ FrameOpSpaceStatus spaceStatus = ctx.getDataFrame().hasSpaceInsert(dataTuple);
+
+ switch (spaceStatus) {
+ case SUFFICIENT_CONTIGUOUS_SPACE:
+ insertSortedIntoDataPage(dataTuple, distance, dataPageId, originalTuple, ctx);
+ return true;
+ case SUFFICIENT_SPACE:
+ // Fix bug-vtree-delete-frame-corruption: reclaimable space exists but is fragmented
+ // (FREE_SPACE_OFFSET has been pushed past the slot region by prior inserts whose
+ // deletes only updated TOTAL_FREE_SPACE_OFFSET). Compact first to reset
+ // FREE_SPACE_OFFSET to a safe high-water mark, then insert. Matches the canonical
+ // BTreeNSMLeafFrame pattern (BTree.java:309-315).
+ ctx.getDataFrame().compact();
+ insertSortedIntoDataPage(dataTuple, distance, dataPageId, originalTuple, ctx);
+ return true;
+ case INSUFFICIENT_SPACE:
+ // Handle overflow by splitting the data page (split recomputes the insertion index
+ // in whichever half the tuple lands, so no position needs to be passed in).
+ splitDataPageMaintainOrder(ctx.getMetadataPageId(), dataPageId, dataTuple, ctx);
+ return true;
+
+ default:
+ throw HyracksDataException.create(ErrorCode.ILLEGAL_STATE, "Unexpected FrameOpSpaceStatus "
+ + spaceStatus + " from VTreeDataFrame.hasSpaceInsert on data page " + dataPageId);
+ }
+
+ } finally {
+ if (latched) {
+ dataPage.releaseWriteLatch(true);
+ }
+ bufferCache.unpin(dataPage);
+ }
+ }
+
+ /**
+ * Insert a data tuple into the currently-latched data frame at its distance-sorted position, fire the
+ * modification callback, bump the page LSN, and grow the catch-all page's metadata max_distance if this
+ * distance is a new maximum. Shared by the contiguous-space and post-compaction insert paths in
+ * {@link #tryInsertIntoDataPage}.
+ */
+ private void insertSortedIntoDataPage(ITupleReference dataTuple, double distance, long dataPageId,
+ ITupleReference originalTuple, VTreeOpContext ctx) throws HyracksDataException {
+ int insertIndex = ((VTreeDataFrame) ctx.getDataFrame()).findInsertPosition(distance);
+ ctx.getDataFrame().insert(dataTuple, insertIndex);
+ ctx.getModificationCallback().found(null, originalTuple);
+ ctx.getDataFrame().setPageLsn(ctx.getDataFrame().getPageLsn() + 1);
+ updateMetadataMaxDistanceIfNeeded(ctx.getMetadataPageId(), dataPageId, distance, ctx);
+ }
+
+ /**
+ * Split data page while maintaining distance-based ordering.
+ */
+ private void splitDataPageMaintainOrder(long metadataPageId, long dataPageId, ITupleReference newTuple,
+ VTreeOpContext ctx) throws HyracksDataException {
+
+ // Create new data page for split
+ int newDataPageId = freePageManager.takePage(ctx.getMetaFrame());
+ ICachedPage newDataPage = bufferCache.pin(BufferedFileHandle.getDiskPageId(getFileId(), newDataPageId), NEW);
+
+ boolean latched = false;
+ try {
+ newDataPage.acquireWriteLatch();
+ latched = true;
+ VTreeDataFrame newFrame = (VTreeDataFrame) ctx.getDataFrameFactory().createFrame();
+ newFrame.setPage(newDataPage);
+ newFrame.initBuffer((byte) 0);
+
+ // Use the frame's split method (following BTree pattern)
+ ctx.getDataFrame().split(newFrame, newTuple);
+
+ // Update page links (maintain linked list structure)
+ int originalNextPage = ctx.getDataFrame().getNextPage();
+ ctx.getDataFrame().setNextPage(newDataPageId);
+ newFrame.setNextPage(originalNextPage);
+
+ // Bump both pages past the source page's prior LSN. Page LSNs are not currently
+ // consulted for recovery on these LSM-component pages (no readers in this codebase),
+ // but keeping the value monotonic mirrors the increment pattern used elsewhere in
+ // this class and avoids the non-monotonicity of System.currentTimeMillis().
+ long currentLsn = ctx.getDataFrame().getPageLsn() + 1;
+ ctx.getDataFrame().setPageLsn(currentLsn);
+ newFrame.setPageLsn(currentLsn);
+
+ // Update metadata to reflect the split
+ updateMetadataAfterDataSplit(metadataPageId, dataPageId, newDataPageId, ctx);
+
+ } finally {
+ if (latched) {
+ newDataPage.releaseWriteLatch(true);
+ }
+ bufferCache.unpin(newDataPage);
+ }
+ }
+
+ /**
+ * Update metadata page after data page split.
+ * Updates BOTH original page's maxDistance and adds new page's entry.
+ */
+ private void updateMetadataAfterDataSplit(long targetMetadataPageId, long originalDataPageId, int newDataPageId,
+ VTreeOpContext ctx) throws HyracksDataException {
+ if (targetMetadataPageId == -1) {
+ throw HyracksDataException.create(ErrorCode.ILLEGAL_STATE,
+ "updateMetadataAfterDataSplit called without a metadata page id (originalDataPageId="
+ + originalDataPageId + ")");
+ }
+
+ // Read each page's post-split max distance (original shrank, new page is the spilled-off tail).
+ double originalPageMaxDistance = readMaxDistanceInDataPage(originalDataPageId, ctx);
+ double newPageMaxDistance = readMaxDistanceInDataPage(newDataPageId, ctx);
+
+ // Update ORIGINAL page's maxDistance in metadata (decreased after split)
+ forceUpdateMetadataMaxDistance(targetMetadataPageId, originalDataPageId, originalPageMaxDistance, ctx);
+
+ // Add NEW page's metadata entry
+ updateMetadataWithNewDataPage(targetMetadataPageId, newDataPageId, newPageMaxDistance, ctx);
+ }
+
+ /**
+ * Read the maximum distance-to-centroid stored in a data page. Data-page tuples are kept sorted by
+ * distance ascending, so the last tuple carries the page's max; an empty page reports {@code 0.0}.
+ * Pins and read-latches the page for the duration.
+ */
+ private double readMaxDistanceInDataPage(long dataPageId, VTreeOpContext ctx) throws HyracksDataException {
+ ICachedPage dataPage = bufferCache.pin(BufferedFileHandle.getDiskPageId(getFileId(), (int) dataPageId));
+ boolean latched = false;
+ try {
+ dataPage.acquireReadLatch();
+ latched = true;
+ IVTreeDataFrame dataFrame = (IVTreeDataFrame) ctx.getDataFrameFactory().createFrame();
+ dataFrame.setPage(dataPage);
+ int tupleCount = dataFrame.getTupleCount();
+ return tupleCount > 0 ? dataFrame.getDistanceToCentroid(tupleCount - 1) : 0.0;
+ } finally {
+ if (latched) {
+ dataPage.releaseReadLatch();
+ }
+ bufferCache.unpin(dataPage);
+ }
+ }
+
+ /**
+ * In-place deletion with three-scenario handling (follows LSMBTree pattern):
+ * 1. Tuple not found → Insert a logical-delete marker tuple (delegates to insertIntoDataPages)
+ * 2. Tuple found (matter only) → Physical DELETE
+ * 3. Tuple found (after a delete marker was replaced by matter during reinsertion) → Physical DELETE
+ *
+ * Note on layering: VTree never reads or sets deletion polarity itself — the injected frame
+ * factory's tuple writer decides how a delete-polarity tuple is encoded, and matter/delete-marker
+ * semantics and reconciliation are defined in the LSM layer (see hyracks-storage-am-lsm-vtree).
+ */
+ private void deleteVector(ITupleReference tuple, VTreeOpContext ctx) throws HyracksDataException {
+
+ // Extract vector and primary key (binary format - no type assumption)
+ double[] vector = VTreeTupleUtils.extractVectorFromTuple(tuple, 0, vectorAccessorFactory);
+ byte[] primaryKey = VTreeTupleUtils.extractPrimaryKeyFromTuple(tuple);
+
+ // Cross-pollination: the record was replicated into every accepted cluster at insert/bulk-load
+ // time, so a delete must reconcile in EVERY one of them — otherwise the non-cancelled replicas
+ // resurface in ANN search. findReplicaClusters() recomputes the identical centroid set (same
+ // eps/M/rng on the immutable static structure), so each replica is matched in its own cluster.
+ for (ClusterSearchResult clusterResult : findReplicaClusters(vector)) {
+ try (ClusterAccessResult accessResult = prepareClusterAccess(clusterResult, ctx)) {
+ double distance = distanceFunction.apply(vector, clusterResult.centroid);
+
+ // Try to find and physically delete tuple from data pages (Scenarios 2 & 3)
+ boolean foundAndDeleted =
+ tryPhysicalDelete(accessResult.metadataPageId(), distance, primaryKey, tuple, ctx);
+
+ if (!foundAndDeleted) {
+ // Scenario 1: Tuple not found in this cluster → Insert a delete-marker tuple via
+ // insertIntoDataPages, which handles empty metadata pages, page splits, and metadata
+ // max-distance updates.
+ insertIntoDataPages(accessResult.metadataPageId(), vector, distance, clusterResult.centroidId,
+ tuple, ctx);
+ }
+ }
+ }
+ }
+
+ /**
+ * Try to physically delete a tuple from data pages. Searches through metadata
+ * pages to find the tuple and delete it. Returns true if found and deleted,
+ * false if not found (caller should insert a delete-marker tuple).
+ *
+ * Uses binary comparison for primary key matching - no type assumption.
+ */
+ private boolean tryPhysicalDelete(long metadataPageId, double distance, byte[] primaryKey,
+ ITupleReference originalTuple, VTreeOpContext ctx) throws HyracksDataException {
+
+ // Traverse through all linked directory (metadata) pages
+ long currentMetadataPageId = metadataPageId;
+
+ while (currentMetadataPageId != -1) {
+ ICachedPage metadataPage =
+ bufferCache.pin(BufferedFileHandle.getDiskPageId(getFileId(), (int) currentMetadataPageId));
+
+ boolean metadataLatched = false;
+ try {
+ metadataPage.acquireReadLatch();
+ metadataLatched = true;
+ ctx.getMetadataFrame().setPage(metadataPage);
+
+ // Determine if this is the last directory page in the chain
+ int nextMetadataPageId = ctx.getMetadataFrame().getNextPage();
+ boolean isLastInChain = (nextMetadataPageId == VTreeDataTupleAccessor.NO_NEXT_PAGE);
+
+ // Find appropriate data page based on distance
+ long targetDataPageId = findDataPageInMetadataPage(ctx.getMetadataFrame(), distance, isLastInChain);
+
+ if (targetDataPageId != -1) {
+ // Try physical deletion in this data page
+ ICachedPage dataPage =
+ bufferCache.pin(BufferedFileHandle.getDiskPageId(getFileId(), (int) targetDataPageId));
+
+ boolean dataLatched = false;
+ try {
+ dataPage.acquireWriteLatch();
+ dataLatched = true;
+ ctx.getDataFrame().setPage(dataPage);
+
+ // Search for tuple by distance + PK (uses binary comparison internally)
+ int pkFieldIndex = VTreeDataTupleAccessor.getPkStartField(quantizationParams != null);
+ int tupleIndex = ((VTreeDataFrame) ctx.getDataFrame())
+ .findTupleByDistanceAndPrimaryKey(distance, primaryKey, pkFieldIndex);
+
+ if (tupleIndex >= 0) {
+ // Found: findTupleByDistanceAndPrimaryKey only returns an index whose PK equals
+ // primaryKey (binary compare), and the data page stays write-latched here, so the
+ // match cannot change under us — no re-check is needed. Physically delete it.
+ ctx.getDataFrame().delete(originalTuple, tupleIndex);
+ return true;
+ }
+
+ // Not found in this data page
+ } finally {
+ if (dataLatched) {
+ dataPage.releaseWriteLatch(true);
+ }
+ bufferCache.unpin(dataPage);
+ }
+ }
+
+ // No match or not found - check next directory page
+ if (isLastInChain) {
+ break; // End of chain
+ }
+ currentMetadataPageId = nextMetadataPageId;
+
+ } finally {
+ if (metadataLatched) {
+ metadataPage.releaseReadLatch();
+ }
+ bufferCache.unpin(metadataPage);
+ }
+ }
+
+ return false; // Not found in any data page
+ }
+
+ private void handleDataPageOverflow(long metadataPageId, double[] vector, double distance, int centroidId,
+ ITupleReference originalTuple, VTreeOpContext ctx) throws HyracksDataException {
+ // Use the frame factories and page manager to handle overflow
+ IVTreeDataFrame dataFrame = (IVTreeDataFrame) ctx.getDataFrameFactory().createFrame();
+ IPageManager pageManager = ctx.getFreePageManager();
+
+ // Create a new data page for overflow
+ int newDataPageId = pageManager.takePage(ctx.getMetaFrame());
+ ICachedPage newPage = bufferCache.pin(BufferedFileHandle.getDiskPageId(getFileId(), newDataPageId), NEW);
+
+ boolean latched = false;
+ try {
+ newPage.acquireWriteLatch();
+ latched = true;
+ // Initialize the new data frame
+ dataFrame.setPage(newPage);
+ dataFrame.initBuffer((byte) 0);
+
+ // Create data tuple for the new vector
+ ITupleReference dataTuple =
+ ctx.getDataTupleBuilder().buildDataTuple(vector, distance, centroidId, originalTuple);
+
+ // Insert the tuple into the new page
+ dataFrame.insert(dataTuple, 0);
+
+ // Update metadata page to include the new data page
+ updateMetadataWithNewDataPage(metadataPageId, newDataPageId, distance, ctx);
+
+ } finally {
+ if (latched) {
+ newPage.releaseWriteLatch(true);
+ }
+ bufferCache.unpin(newPage);
+ }
+ }
+
+ /**
+ * Update metadata maxDistance if the new distance exceeds the current maxDistance.
+ * This is needed when inserting into the last data page with a distance greater than
+ * the current maxDistance - the last page acts as a catch-all that dynamically expands.
+ */
+ private void updateMetadataMaxDistanceIfNeeded(long metadataPageId, long dataPageId, double newDistance,
+ VTreeOpContext ctx) throws HyracksDataException {
+
+ VTreeMetadataFrame metadataFrame = requireLatchedMetadataFrame(metadataPageId, ctx);
+
+ // Find the metadata entry for this data page
+ int tupleCount = metadataFrame.getTupleCount();
+ for (int i = 0; i < tupleCount; i++) {
+ long pagePtr = metadataFrame.getDataPagePointer(i);
+
+ if (pagePtr == dataPageId) {
+ double currentMaxDistance = metadataFrame.getMaxDistance(i);
+
+ // Only update if new distance is larger
+ if (newDistance > currentMaxDistance) {
+ metadataFrame.updateMaxDistance(i, newDistance);
+ }
+ break;
+ }
+ }
+ }
+
+ /**
+ * Force update metadata maxDistance to a specific value (regardless of increase/decrease).
+ * This is needed after data page splits where the original page's maxDistance decreases.
+ */
+ private void forceUpdateMetadataMaxDistance(long metadataPageId, long dataPageId, double newMaxDistance,
+ VTreeOpContext ctx) throws HyracksDataException {
+
+ VTreeMetadataFrame metadataFrame = requireLatchedMetadataFrame(metadataPageId, ctx);
+
+ // Find the metadata entry for this data page
+ int tupleCount = metadataFrame.getTupleCount();
+ for (int i = 0; i < tupleCount; i++) {
+ long pagePtr = metadataFrame.getDataPagePointer(i);
+
+ if (pagePtr == dataPageId) {
+ metadataFrame.updateMaxDistance(i, newMaxDistance);
+ break;
+ }
+ }
+ }
+
+ /**
+ * Update metadata page to include a new data page. Handles metadata page overflow by splitting when necessary.
+ */
+ private void updateMetadataWithNewDataPage(long metadataPageId, int newDataPageId, double maxDistance,
+ VTreeOpContext ctx) throws HyracksDataException {
+
+ VTreeMetadataFrame metadataFrame = requireLatchedMetadataFrame(metadataPageId, ctx);
+
+ // Create metadata tuple for new data page
+ ITupleReference metadataTuple = VTreeMetadataTupleAccessor.createMetadataTuple(maxDistance, newDataPageId);
+ ITreeIndexTupleWriter metadataFrameTupleWriter = metadataFrame.getTupleWriter();
+ int slotSize = metadataFrame.getSlotSize();
+
+ // Check if there's space for the new metadata entry
+ // Check if directory page has space
+ int spaceNeeded = metadataFrameTupleWriter.bytesRequired(metadataTuple) + slotSize;
+ int spaceAvailable = metadataFrame.getTotalFreeSpace();
+
+ if (spaceNeeded > spaceAvailable) {
+ // Insufficient space - need to split metadata page
+ handleMetadataPageOverflow(metadataPageId, metadataTuple, ctx);
+ } else {
+ // Insert the new data-page entry in its sorted position by max_distance, preserving the
+ // directory's max_distance-ascending invariant (see VTreeMetadataFrame javadoc) that
+ // findDataPageInMetadataPage() relies on for correct distance-based routing. Appending at
+ // getTupleCount() here corrupted that invariant after a non-last data-page split (the new
+ // page's max_distance falls between existing entries), which mis-routed subsequent inserts,
+ // produced overlapping data-page distance ranges, and broke the sorted-stream precondition
+ // of the search-side merge and the matter/delete-marker reconciliation done in the LSM layer.
+ int insertPos = metadataFrame.findInsertPosition(maxDistance);
+ metadataFrame.insert(metadataTuple, insertPos);
+ }
+ }
+
+ /**
+ * Return the shared metadata frame that the caller already holds pinned and write-latched for
+ * {@code metadataPageId}.
+ * <p>
+ * All metadata-mutation helpers ({@link #updateMetadataMaxDistanceIfNeeded},
+ * {@link #forceUpdateMetadataMaxDistance}, {@link #updateMetadataWithNewDataPage}) run only from
+ * inside {@link #insertIntoDataPages}, which pins the current directory page, write-latches it, sets
+ * {@code ctx.getMetadataFrame()} to it, and releases the latch (marking the page dirty) in its own
+ * {@code finally}. Operating on that already-latched frame here — instead of re-pinning and
+ * re-latching the same page — removes redundant buffer-cache I/O and the former reliance on latch
+ * reentrancy ({@code ReentrantReadWriteLock}) that a non-reentrant latch would have turned into a
+ * self-deadlock. {@link #handleMetadataPageOverflow} already assumes this same shared-frame contract.
+ */
+ private VTreeMetadataFrame requireLatchedMetadataFrame(long metadataPageId, VTreeOpContext ctx) {
+ assert metadataPageId == ctx.getMetadataPageId() : "metadata mutation on page " + metadataPageId
+ + " but the write-latched page is " + ctx.getMetadataPageId();
+ return (VTreeMetadataFrame) ctx.getMetadataFrame();
+ }
+
+ /**
+ * Handle metadata page overflow by splitting the page and distributing tuples.
+ */
+ private void handleMetadataPageOverflow(long metadataPageId, ITupleReference newTuple, VTreeOpContext ctx)
+ throws HyracksDataException {
+
+ // Allocate a new metadata page
+ int newMetadataPageId = freePageManager.takePage(ctx.getMetaFrame());
+ ICachedPage newMetadataPage =
+ bufferCache.pin(BufferedFileHandle.getDiskPageId(getFileId(), newMetadataPageId), NEW);
+
+ boolean latched = false;
+ try {
+ newMetadataPage.acquireWriteLatch();
+ latched = true;
+
+ // Create new metadata frame for the split page
+ IVTreeMetadataFrame rightFrame = (IVTreeMetadataFrame) metadataFrameFactory.createFrame();
+ rightFrame.setPage(newMetadataPage);
+ rightFrame.initBuffer((byte) 0);
+
+ // Split the current metadata page using the correct method from VTreeMetadataFrame
+ ((VTreeMetadataFrame) ctx.getMetadataFrame()).split(rightFrame, newTuple);
+
+ // Update the next page pointer in the original metadata page
+ ctx.getMetadataFrame().setNextPage(newMetadataPageId);
+
+ // Initialize the next page pointer in the new metadata page to end-of-chain.
+ rightFrame.setNextPage(VTreeDataTupleAccessor.NO_NEXT_PAGE);
+
+ } finally {
+ if (latched) {
+ newMetadataPage.releaseWriteLatch(true);
+ }
+ bufferCache.unpin(newMetadataPage);
+ }
+ }
+
+ @Override
+ public void validate() throws HyracksDataException {
+ // Validation logic specific to vector clustering tree
+ }
+
+ /**
+ * Find the closest cluster starting from root and traversing down to leaf level. Handles overflow pages for both
+ * interior and leaf frames.
+ */
+ public ClusterSearchResult findClosestClusterFromRoot(double[] queryVector, IVTreeDistanceFunction distanceFunction)
+ throws HyracksDataException {
+ return findClosestClusterFromRoot(queryVector, distanceFunction, null, null);
+ }
+
+ /**
+ * Find the closest cluster starting from root and traversing down to leaf level,
+ * optionally computing a quantized distance for the best result.
+ *
+ * @param queryVector Query vector for navigation (full precision)
+ * @param distanceFunction Distance function for centroid comparison
+ * @param quantizedQueryVector Quantized form of queryVector (nullable — pass null to skip quantized distance)
+ * @param quantizer Quantizer for dequantizing leaf centroid bytes (nullable — pass null to skip)
+ * @return ClusterSearchResult with optional quantizedDistance
+ */
+ public ClusterSearchResult findClosestClusterFromRoot(double[] queryVector, IVTreeDistanceFunction distanceFunction,
+ double[] quantizedQueryVector, IVTreeQuantizer quantizer) throws HyracksDataException {
+
+ // For memory components: navigate via static structure reference
+ IBufferCache navBC = (staticBufferCache != null) ? staticBufferCache : bufferCache;
+ int navFileId = (staticBufferCache != null) ? staticFileId : getFileId();
+ int navRoot = (staticBufferCache != null) ? staticRootPage : rootPage;
+
+ LOGGER.log(Level.TRACE, "Starting findClosestClusterFromRoot with navRoot={}, isMemoryComponent={}", navRoot,
+ staticBufferCache != null);
+
+ ClusterSearchResult result =
+ VTreeNavigationUtils.findClosestCentroid(navBC, navFileId, navRoot, getInteriorFrameFactory(),
+ getLeafFrameFactory(), queryVector, distanceFunction, quantizedQueryVector, quantizer);
+
+ // For memory components: replace directoryPageId with VBC mapping
+ if (centroidDirPageMap != null) {
+ int centroidIndex = result.centroidId - firstLeafCentroidIdMem;
+ if (centroidIndex >= 0 && centroidIndex < centroidDirPageMap.length) {
+ result = ClusterSearchResult.create(result.leafPageId, result.clusterIndex, result.centroid,
+ result.distance, result.centroidId, centroidDirPageMap[centroidIndex],
+ result.quantizedDistance);
+ }
+ }
+
+ return result;
+ }
+
+ public int getVectorDimensions() {
+ return vectorDimensions;
+ }
+
+ /**
+ * Get the buffer cache to use for tree navigation.
+ * For memory components, this returns the static structure's buffer cache.
+ * For disk components, this returns the component's own buffer cache.
+ */
+ public IBufferCache getNavigationBufferCache() {
+ return (staticBufferCache != null) ? staticBufferCache : bufferCache;
+ }
+
+ /**
+ * Get the file ID to use for tree navigation.
+ * For memory components, this returns the static structure's file ID.
+ * For disk components, this returns the component's own file ID.
+ */
+ public int getNavigationFileId() {
+ return (staticBufferCache != null) ? staticFileId : getFileId();
+ }
+
+ /**
+ * Get the root page ID to use for tree navigation.
+ * For memory components, this returns the static structure's root page.
+ * For disk components, this returns the component's own root page.
+ */
+ public int getNavigationRootPageId() {
+ return (staticBufferCache != null) ? staticRootPage : rootPage;
+ }
+
+ /**
+ * Find close centroids via level-wise traversal with global sort (delegates to
+ * VTreeNavigationUtils.findCloseCentroidsLevelWiseGlobalSort). Handles overflow pages for both
+ * interior and leaf frames.
+ */
+ public List<ClusterSearchResult> findCloseCentroidsLevelWiseGlobalSortFromRoot(double[] queryVector,
+ IVTreeDistanceFunction distanceFunction, double ep) throws HyracksDataException {
+
+ // For memory components: navigate via static structure reference
+ IBufferCache navBC = (staticBufferCache != null) ? staticBufferCache : bufferCache;
+ int navFileId = (staticBufferCache != null) ? staticFileId : getFileId();
+ int navRoot = (staticBufferCache != null) ? staticRootPage : rootPage;
+
+ LOGGER.log(Level.TRACE, "Starting findCloseCentroidsLevelWiseFromRoot with navRoot={}", navRoot);
+
+ List<ClusterSearchResult> results = VTreeNavigationUtils.findCloseCentroidsLevelWiseGlobalSort(navBC, navFileId,
+ navRoot, getInteriorFrameFactory(), getLeafFrameFactory(), queryVector, distanceFunction, ep);
+
+ // For memory components: replace directoryPageId with VBC mapping
+ if (centroidDirPageMap != null) {
+ for (int r = 0; r < results.size(); r++) {
+ ClusterSearchResult result = results.get(r);
+ int centroidIndex = result.centroidId - firstLeafCentroidIdMem;
+ if (centroidIndex >= 0 && centroidIndex < centroidDirPageMap.length) {
+ results.set(r,
+ ClusterSearchResult.create(result.leafPageId, result.clusterIndex, result.centroid,
+ result.distance, result.centroidId, centroidDirPageMap[centroidIndex],
+ result.quantizedDistance));
+ }
+ }
+ }
+
+ return results;
+ }
+
+ /**
+ * Get the raw quantization parameters, or null if no quantization is configured.
+ * Format: {minQuantile, maxQuantile, alpha, confidenceInterval, bits, sampleCount}
+ */
+ public VTreeQuantizationParams getQuantizationParams() {
+ return quantizationParams;
+ }
+
+ public boolean isInitialized() {
+ return initialized;
+ }
+
+ /**
+ * Reset initialization state so that static structure directory pages
+ * can be re-created after a memory component flush/recycle.
+ */
+ public void resetInitialization() {
+ initialized = false;
+ centroidDirPageMap = null;
+ }
+
+ /**
+ * Initialize this memory component's static-structure navigation state (see the field-group comment
+ * above for the full threading contract). Called by the LSM layer during memory-component allocation
+ * or post-flush recycle, before the component is published for operations; {@code synchronized} plus
+ * the {@code initialized} guard make it a safe, idempotent single write. Reads of the fields it sets
+ * rely on the LSM harness's operation-tracker happens-before for visibility, so callers must not read
+ * them concurrently with this method outside that ordering.
+ */
+ public synchronized void setStaticStructure(VTreeAccessor staticAccessor) throws HyracksDataException {
+ if (initialized) {
+ return; // Already initialized, skip
+ }
+
+ VTree staticStructure = staticAccessor.getIndex();
+ ITreeIndexMetadataFrame metaFrame = staticAccessor.getOpContext().getMetaFrame();
+
+ // Store references to static structure for read-only navigation
+ this.staticBufferCache = staticStructure.getBufferCache();
+ this.staticFileId = staticStructure.getFileId();
+ this.staticRootPage = staticStructure.rootPage;
+
+ // Pin the static structure's metadata page onto metaFrame before reading
+ // (getMaxPageId internally calls metaFrame.setPage() which initializes the frame's buffer)
+ staticStructure.getPageManager().getMaxPageId(metaFrame);
+
+ // Read metadata from static structure
+ LongPointable value1 = LongPointable.FACTORY.createPointable();
+ LongPointable value2 = LongPointable.FACTORY.createPointable();
+ metaFrame.get(VTreeMetadataKeys.NUM_LEAF_CENTROIDS, value1);
+ metaFrame.get(VTreeMetadataKeys.FIRST_LEAF_CENTROID_ID, value2);
+ this.numLeafCentroidMem = value1.intValue();
+ this.firstLeafCentroidIdMem = value2.intValue();
+
+ // Create empty directory pages in VBC (using takePage() directly)
+ ITreeIndexMetadataFrame vbcMetaFrame = freePageManager.createMetadataFrame();
+ ITreeIndexFrame directoryFrame = metadataFrameFactory.createFrame();
+ centroidDirPageMap = new int[numLeafCentroidMem];
+
+ for (int i = 0; i < numLeafCentroidMem; i++) {
+ int dirPageId = freePageManager.takePage(vbcMetaFrame);
+ centroidDirPageMap[i] = dirPageId;
+
+ ICachedPage targetPage = bufferCache.pin(BufferedFileHandle.getDiskPageId(getFileId(), dirPageId), NEW);
+ try {
+ // No latch is taken here: the page is freshly allocated (NEW) and not yet visible to
+ // other threads, so a failing setPage/initBuffer only needs the pin unwound.
+ directoryFrame.setPage(targetPage);
+ directoryFrame.initBuffer((byte) 0);
+ } finally {
+ bufferCache.unpin(targetPage);
+ }
+
+ LOGGER.log(Level.TRACE, "Created directory page {} for leaf centroid {}", dirPageId, i);
+ }
+
+ initialized = true;
+ }
+
+ public void setRootPageId(int rootPageId) {
+ rootPage = rootPageId;
+ }
+
+ public int[] getCentroidDirPageMap() {
+ return centroidDirPageMap;
+ }
+
+ public int getFirstLeafCentroidIdMem() {
+ return firstLeafCentroidIdMem;
+ }
+
+ public int getNumLeafCentroidMem() {
+ return numLeafCentroidMem;
+ }
+
+ /**
+ * Result of {@link #prepareClusterAccess}: carries the cluster lookup, the (still
+ * write-latched) leaf page, and the metadata page id. Implements {@link AutoCloseable} so
+ * callers can release the latch and unpin via try-with-resources. {@code leafPage} is
+ * {@code null} for memory components — in that case {@code close()} is a no-op.
+ */
+ public record ClusterAccessResult(ClusterSearchResult clusterResult, ICachedPage leafPage, long metadataPageId,
+ IBufferCache bufferCache) implements AutoCloseable {
+
+ @Override
+ public void close() {
+ if (leafPage != null) {
+ leafPage.releaseWriteLatch(true);
+ bufferCache.unpin(leafPage);
+ }
+ }
+
+ }
+
+ /**
+ * Resolve the leaf clusters a record must be written to / deleted from.
+ * <p>
+ * For ALL {@code M} (including the legacy {@code M == 1}, where the top-1 is returned) this reproduces
+ * the bulk-load placement exactly: the eps-filtered, RNG-thinned top-{@code M} accepted centroids
+ * ({@link #findCloseCentroidsLevelWiseGlobalSortFromRoot} + {@link RngAcceptanceFilter#accept}). Because
+ * the static structure (centroid set) is immutable and the eps/M/rng parameters are fixed for the index,
+ * this resolves to the same set at bulk-load, insert AND delete time — the property delete reconciliation
+ * relies on to cancel every replica. (Using greedy {@code findClosestClusterFromRoot} for {@code M == 1}
+ * broke this: it diverged from bulk-load's level-wise choice and leaked deletes — see the method body.)
+ */
+ private List<ClusterSearchResult> findReplicaClusters(double[] vector) throws HyracksDataException {
+ // Route via the SAME level-wise global-sort navigation + RNG thinning that BULK-LOAD uses, for ALL
+ // M — including the legacy M==1 case, where RngAcceptanceFilter returns the single globally-closest
+ // centroid. The old M==1 shortcut to greedy findClosestClusterFromRoot picked a *locally* closest
+ // leaf that, for boundary records, differed from bulk-load's globally-closest choice; the delete then
+ // wrote its antimatter into a different cluster (and a different field-0 distance) than the bulk-loaded
+ // matter, so search-time reconciliation could never cancel it and the deleted record leaked in ANN
+ // results (~66% of bulk-loaded deletes at some k-means seeds). One routing function across bulk-load,
+ // insert and delete guarantees matter and antimatter always agree on cluster and distance-to-centroid.
+ List<ClusterSearchResult> candidates =
+ findCloseCentroidsLevelWiseGlobalSortFromRoot(vector, distanceFunction, crossPollination.epsilon());
+ List<ClusterSearchResult> accepted = RngAcceptanceFilter.accept(candidates, distanceFunction,
+ crossPollination.rngFactor(), crossPollination.m(), null);
+ if (accepted.isEmpty()) {
+ // Defensive: navigation found nothing within eps — never silently drop the record; fall back to
+ // the single closest cluster.
+ return Collections.singletonList(requireCluster(findClosestClusterFromRoot(vector, distanceFunction)));
+ }
+ return accepted;
+ }
+
+ private static ClusterSearchResult requireCluster(ClusterSearchResult clusterResult) throws HyracksDataException {
+ if (clusterResult == null) {
+ throw HyracksDataException.create(ErrorCode.ILLEGAL_STATE, "No cluster found for vector");
+ }
+ return clusterResult;
+ }
+
+ /**
+ * Prepare data-page access for a single (already resolved) cluster: pin/latch the leaf page when needed
+ * and resolve the metadata page id. Used by insert, delete, and update once per replica cluster.
+ * <ul>
+ * <li>Memory components ({@code centroidDirPageMap != null}): the metadata page is taken straight from
+ * the VBC centroid→directory mapping carried on {@code clusterResult}; no leaf page is pinned.</li>
+ * <li>Disk components: the leaf page is write-latched and the metadata pointer read from its frame.</li>
+ * </ul>
+ */
+ private ClusterAccessResult prepareClusterAccess(ClusterSearchResult clusterResult, VTreeOpContext ctx)
+ throws HyracksDataException {
+ if (centroidDirPageMap != null) {
+ return new ClusterAccessResult(clusterResult, null, clusterResult.directoryPageId, bufferCache);
+ }
+
+ ICachedPage leafPage = bufferCache.pin(BufferedFileHandle.getDiskPageId(getFileId(), clusterResult.leafPageId));
+ try {
+ leafPage.acquireWriteLatch();
+ ctx.getLeafFrame().setPage(leafPage);
+ long metadataPageId = ctx.getLeafFrame().getMetadataPagePointer(clusterResult.clusterIndex);
+ return new ClusterAccessResult(clusterResult, leafPage, metadataPageId, bufferCache);
+ } catch (Exception e) {
+ try {
+ leafPage.releaseWriteLatch(false);
+ } finally {
+ bufferCache.unpin(leafPage);
+ }
+ throw e;
+ }
+ }
+
+ public class VTreeAccessor implements ITreeIndexAccessor {
+
+ private VTree tree;
+ private VTreeOpContext ctx;
+ private IIndexAccessParameters iap;
+ private boolean destroyed = false;
+ // The IAP map and the tree's quantization params are fixed for the accessor's lifetime, so these are
+ // resolved once at construction and read as fixed fields during search. queryDistanceFunctionFactory
+ // is the optional query-time factory (null → fall back to the index's own); binaryAccessorFactory
+ // decodes the query tuple; quantizerFactory / injectedQuantizer are the production and test quantizer
+ // seams. Only the quantizer instance is built per search (it depends on the predicate's distance metric).
+ private final IVTreeDistanceFunctionFactory queryDistanceFunctionFactory;
+ private final IVTreeBinaryAccessorFactory binaryAccessorFactory;
+ private final IVTreeQuantizerFactory quantizerFactory;
+ private final IVTreeQuantizer injectedQuantizer;
+ private final VTreeQuantizationParams quantizationParams;
+
+ public VTreeAccessor(VTree tree, IIndexAccessParameters iap) {
+ this.tree = tree;
+ this.iap = iap;
+ this.ctx = new VTreeOpContext(this, tree.interiorFrameFactory, tree.leafFrameFactory,
+ tree.metadataFrameFactory, tree.dataFrameFactory, tree.freePageManager, tree.cmpFactories,
+ tree.vectorDimensions, iap.getModificationCallback(), iap.getSearchOperationCallback(),
+ tree.dataTupleBuilderFactory, tree.quantizationParams);
+ this.queryDistanceFunctionFactory =
+ (IVTreeDistanceFunctionFactory) iap.getParameters().get(IVTreeDistanceFunctionFactory.IAP_KEY);
+ this.binaryAccessorFactory =
+ (IVTreeBinaryAccessorFactory) iap.getParameters().get(IVTreeBinaryAccessorFactory.IAP_KEY);
+ this.quantizerFactory = (IVTreeQuantizerFactory) iap.getParameters().get(IVTreeQuantizerFactory.IAP_KEY);
+ this.injectedQuantizer = (IVTreeQuantizer) iap.getParameters().get(IVTreeQuantizer.IAP_KEY);
+ this.quantizationParams = tree.getQuantizationParams();
+ }
+
+ @Override
+ public void insert(ITupleReference tuple) throws HyracksDataException {
+ ctx.setOperation(IndexOperation.INSERT);
+ insertVector(tuple, ctx);
+ }
+
+ @Override
+ public void update(ITupleReference tuple) throws HyracksDataException {
+ ctx.setOperation(IndexOperation.UPDATE);
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void delete(ITupleReference tuple) throws HyracksDataException {
+ ctx.setOperation(IndexOperation.DELETE);
+ // Use in-place deletion instead of always inserting a delete-marker tuple
+ tree.deleteVector(tuple, ctx);
+ }
+
+ @Override
+ public void upsert(ITupleReference tuple) throws HyracksDataException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public IIndexCursor createSearchCursor(boolean exclusive) throws HyracksDataException {
+ return createSearchCursor(exclusive, false);
+ }
+
+ /**
+ * Create a search cursor with explicit full-scan mode control.
+ *
+ * @param exclusive Whether to create an exclusive cursor
+ * @param fullScanMode true for full-scan (merge) mode, false for query mode
+ * @return Configured search cursor
+ */
+ public IIndexCursor createSearchCursor(boolean exclusive, boolean fullScanMode) throws HyracksDataException {
+ VTreeSearchCursor cursor = new VTreeSearchCursor();
+ configureCursor(cursor);
+ cursor.setFullScanMode(fullScanMode);
+ return cursor;
+ }
+
+ private void configureCursor(VTreeSearchCursor cursor) {
+ if (tree.staticBufferCache != null) {
+ cursor.setBufferCache(tree.staticBufferCache);
+ cursor.setFileId(tree.staticFileId);
+ cursor.setRootPageId(tree.staticRootPage);
+ cursor.setDataBufferCache(tree.bufferCache, tree.getFileId());
+ cursor.setCentroidDirPageMap(tree.centroidDirPageMap, tree.firstLeafCentroidIdMem);
+ } else {
+ cursor.setBufferCache(tree.bufferCache);
+ cursor.setFileId(tree.getFileId());
+ cursor.setRootPageId(tree.rootPage);
+ cursor.setDataBufferCache(tree.bufferCache, tree.getFileId());
+ }
+ cursor.setFrameFactories(tree.interiorFrameFactory, tree.leafFrameFactory, tree.metadataFrameFactory,
+ tree.dataFrameFactory);
+ }
+
+ public VTree getIndex() {
+ return tree;
+ }
+
+ @Override
+ public void search(IIndexCursor cursor, ISearchPredicate searchPred) throws HyracksDataException {
+ ctx.setOperation(IndexOperation.SEARCH);
+
+ VTreeSearchCursor vectorCursor = (VTreeSearchCursor) cursor;
+ configureCursor(vectorCursor);
+
+ // Extract query vector and distance metric from predicate using the accessor factory
+ // The predicate holds the tuple reference (updated per-tuple in resetSearchPredicate)
+ double[] queryVector = extractQueryVector(searchPred);
+ IVTreeDistanceFunction distanceFunction = resolveDistanceFunctionFactory().createDistanceFunction();
+
+ VTreeCursorInitialState initialState = buildInitialState(queryVector, distanceFunction);
+ resolveAndSetQuantizer(initialState, queryVector);
+
+ // Open the cursor - it will perform centroid finding and position on data pages
+ vectorCursor.open(initialState, searchPred);
+ }
+
+ /** Decode the query vector from the predicate's tuple via the IAP accessor factory (null if none). */
+ private double[] extractQueryVector(ISearchPredicate searchPred) throws HyracksDataException {
+ if (searchPred instanceof VTreeSearchPredicate vectorPred && vectorPred.getQueryTuple() != null) {
+ // binaryAccessorFactory was resolved from the IAP at accessor construction.
+ return VTreeTupleUtils.extractVectorFromTuple(vectorPred.getQueryTuple(),
+ vectorPred.getQueryFieldIndex(), binaryAccessorFactory);
+ }
+ return null;
+ }
+
+ /**
+ * Prefer a query-time factory from IAP (see
+ * VTreeSearchOperatorNodePushable#addAdditionalIndexAccessorParams), else the factory the index
+ * was built with.
+ */
+ private IVTreeDistanceFunctionFactory resolveDistanceFunctionFactory() {
+ // queryDistanceFunctionFactory was resolved from the IAP at accessor construction.
+ return queryDistanceFunctionFactory != null ? queryDistanceFunctionFactory : tree.distanceFunctionFactory;
+ }
+
+ /** Build the cursor initial state: root page (static for memory components), query vector, metric fn. */
+ private VTreeCursorInitialState buildInitialState(double[] queryVector,
+ IVTreeDistanceFunction distanceFunction) {
+ VTreeCursorInitialState initialState = new VTreeCursorInitialState(ctx.getAccessor());
+ // For memory components, use staticRootPage (the static structure's root);
+ // for disk components, use the tree's own rootPage
+ initialState.setRootPageId(tree.staticBufferCache != null ? tree.staticRootPage : tree.rootPage);
+ if (queryVector != null) {
+ initialState.setQueryVector(queryVector);
+ }
+ initialState.setDistanceFunction(distanceFunction);
+ return initialState;
+ }
+
+ /**
+ * Resolve the query-time quantizer (if any) and publish its quantized query vector onto
+ * {@code initialState}. Production path: an {@link IVTreeQuantizerFactory} supplied via IAP
+ * by VTreeSearchOperatorNodePushable builds a quantizer from the float[6] quantization
+ * params persisted on the tree. Test fallback: a pre-built {@link IVTreeQuantizer} injected
+ * directly under {@link IVTreeQuantizer#IAP_KEY} (e.g. NoOpVectorQuantizer.INSTANCE in
+ * VectorTreeTestUtils). Leaves {@code initialState} unquantized when neither applies.
+ */
+ private void resolveAndSetQuantizer(VTreeCursorInitialState initialState, double[] queryVector)
+ throws HyracksDataException {
+ // quantizationParams / quantizerFactory / injectedQuantizer were resolved from the tree and IAP at
+ // accessor construction; the quantizer's distance metric is baked into the factory.
+ if (quantizationParams != null && queryVector != null && quantizerFactory != null) {
+ IVTreeQuantizer quantizer = quantizerFactory.createQuantizer(tree.vectorDimensions, quantizationParams);
+ initialState.setQuantizedQueryVector(quantizer.quantize(queryVector));
+ initialState.setQuantizer(quantizer);
+ }
+
+ // Fallback: a pre-built IVTreeQuantizer injected directly under IVTreeQuantizer.IAP_KEY.
+ if (initialState.getQuantizer() == null && queryVector != null && injectedQuantizer != null) {
+ initialState.setQuantizedQueryVector(injectedQuantizer.quantize(queryVector));
+ initialState.setQuantizer(injectedQuantizer);
+ }
+ }
+
+ /**
+ * Find the closest leaf centroid for a given query vector.
+ * This method delegates to the tree's findClosestClusterFromRoot implementation.
+ *
+ * @param queryVector The query vector to find the closest centroid for
+ * @param distanceFunction The distance function to use for centroid finding
+ * @return ClusterSearchResult containing information about the closest leaf centroid
+ * @throws HyracksDataException if any error occurs during the search
+ */
+ public ClusterSearchResult findClosestLeafCentroid(double[] queryVector,
+ IVTreeDistanceFunction distanceFunction) throws HyracksDataException {
+ return findClosestLeafCentroid(queryVector, distanceFunction, null, null);
+ }
+
+ /**
+ * Find the closest leaf centroid, optionally computing quantized distance.
+ *
+ * @param queryVector The query vector to find the closest centroid for
+ * @param distanceFunction The distance function to use for centroid finding
+ * @param quantizedQueryVector Quantized form of queryVector (nullable)
+ * @param quantizer Quantizer for dequantizing leaf centroid bytes (nullable)
+ * @return ClusterSearchResult with optional quantizedDistance
+ * @throws HyracksDataException if any error occurs during the search
+ */
+ public ClusterSearchResult findClosestLeafCentroid(double[] queryVector,
+ IVTreeDistanceFunction distanceFunction, double[] quantizedQueryVector, IVTreeQuantizer quantizer)
+ throws HyracksDataException {
+ validateQueryVector(queryVector);
+ return tree.findClosestClusterFromRoot(queryVector, distanceFunction, quantizedQueryVector, quantizer);
+ }
+
+ /** Reject a query on a destroyed accessor, a null query vector, or a dimension mismatch. */
+ private void validateQueryVector(double[] queryVector) throws HyracksDataException {
+ if (destroyed) {
+ throw HyracksDataException.create(ErrorCode.ILLEGAL_STATE, "Accessor has been destroyed");
+ }
+ if (queryVector == null) {
+ throw HyracksDataException.create(ErrorCode.ILLEGAL_STATE, "Query vector cannot be null");
+ }
+ if (queryVector.length != tree.vectorDimensions) {
+ throw HyracksDataException.create(ErrorCode.ILLEGAL_STATE, "Query vector dimension ("
+ + queryVector.length + ") does not match tree dimension (" + tree.vectorDimensions + ")");
+ }
+ }
+
+ @Override
+ public void destroy() throws HyracksDataException {
+ if (destroyed) {
+ return;
+ }
+ destroyed = true;
+ ctx.destroy();
+ }
+
+ @Override
+ public ITreeIndexCursor createDiskOrderScanCursor() {
+ return new TreeIndexDiskOrderScanCursor(leafFrameFactory.createFrame());
+ }
+
+ @Override
+ public void diskOrderScan(ITreeIndexCursor cursor) throws HyracksDataException {
+ ctx.setOperation(IndexOperation.DISKORDERSCAN);
+ throw new UnsupportedOperationException("Disk order scan not yet implemented");
+ }
+
+ public VTreeOpContext getOpContext() {
+ return ctx;
+ }
+
+ public ICachedPage getCachedPage(int pageId) throws HyracksDataException {
+ return bufferCache.pin(BufferedFileHandle.getDiskPageId(getFileId(), pageId));
+ }
+
+ public void releasePage(ICachedPage page) {
+ bufferCache.unpin(page);
+ }
+
+ public List<ClusterSearchResult> findCloseCentroidsLevelWiseGlobalSort(double[] queryVector,
+ IVTreeDistanceFunction hyracksDistanceFunction, double epi) throws HyracksDataException {
+ validateQueryVector(queryVector);
+ return tree.findCloseCentroidsLevelWiseGlobalSortFromRoot(queryVector, hyracksDistanceFunction, epi);
+ }
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/impls/VTreeBulkLoader.java b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/impls/VTreeBulkLoader.java
new file mode 100644
index 0000000000..2c749d03e7
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/impls/VTreeBulkLoader.java
@@ -0,0 +1,662 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.vector.impls;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.primitive.IntegerPointable;
+import org.apache.hyracks.data.std.primitive.LongPointable;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.dataflow.common.data.marshalling.DoubleSerializerDeserializer;
+import org.apache.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import org.apache.hyracks.dataflow.common.utils.TupleUtils;
+import org.apache.hyracks.storage.am.common.api.IPageManager;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexAccessor;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexMetadataFrame;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
+import org.apache.hyracks.storage.am.common.impls.AbstractTreeIndex;
+import org.apache.hyracks.storage.am.vector.api.IVTreeDataFrame;
+import org.apache.hyracks.storage.am.vector.api.IVTreeFrame;
+import org.apache.hyracks.storage.am.vector.api.IVTreeInteriorFrame;
+import org.apache.hyracks.storage.am.vector.api.IVTreeLeafFrame;
+import org.apache.hyracks.storage.am.vector.api.IVTreeMetadataFrame;
+import org.apache.hyracks.storage.am.vector.utils.VTreeDataTupleAccessor;
+import org.apache.hyracks.storage.am.vector.utils.VTreeLeafNeighborList;
+import org.apache.hyracks.storage.am.vector.utils.VTreeMetadataKeys;
+import org.apache.hyracks.storage.common.IIndexBulkLoader;
+import org.apache.hyracks.storage.common.ISketchSampler;
+import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.buffercache.ICachedPage;
+import org.apache.hyracks.storage.common.buffercache.IFIFOPageWriter;
+import org.apache.hyracks.storage.common.buffercache.IPageWriteCallback;
+import org.apache.hyracks.storage.common.buffercache.PageWriteFailureCallback;
+import org.apache.hyracks.storage.common.buffercache.context.write.DefaultBufferCacheWriteContext;
+import org.apache.hyracks.storage.common.compression.file.ICompressedPageWriter;
+import org.apache.hyracks.storage.common.file.BufferedFileHandle;
+import org.apache.hyracks.util.annotations.AiProvenance;
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+public class VTreeBulkLoader extends PageWriteFailureCallback implements IIndexBulkLoader {
+ private static final Logger LOGGER = LogManager.getLogger();
+
+ private final IBufferCache bufferCache;
+ private final IPageManager freePageManager;
+ private final ITreeIndexMetadataFrame metaFrame;
+ private final AbstractTreeIndex treeIndex;
+ private final int fileId;
+ private final int slotSize;
+ private final IFIFOPageWriter pageWriter;
+ private final ICompressedPageWriter compressedPageWriter;
+
+ // Accessor to the source static structure, which is immutable and stays open for the
+ // lifetime of this load. Its pages are re-read one at a time in end() rather than being
+ // snapshotted up front, so the loader holds O(1) pages at any moment.
+ private final VTree.VTreeAccessor staticAccessor;
+ private final int numStaticPages;
+ // Root page id within the static structure's page id space. After the static
+ // pages are copied into this data component starting at staticBasePageId, the
+ // root of the copied structure sits at (staticBasePageId + staticStructureRootPage).
+ private final int staticStructureRootPage;
+
+ private final int firstLeafCentroidId;
+ private final int numLeafCentroid;
+
+ // Per-cluster directory page tracking: clusterIndex -> first dir page ID
+ private final int[] clusterFirstDirPageId;
+
+ // Current cluster state
+ private int currentLeafClusterIndex;
+ private int currentCentroidId;
+
+ // Current data page (only one in memory at a time — written immediately when full)
+ private ICachedPage currentDataPage;
+ private int currentDataPageId;
+ private final ITreeIndexFrame currentDataFrame;
+ private final ITreeIndexTupleWriter dataFrameTupleWriter;
+ private int entriesInCurrentDataPage;
+
+ // Directory pages for current cluster. Confiscated with INVALID_DPID and kept in memory
+ // until the cluster is finalized, at which point they receive real page IDs, get chained
+ // via nextPage pointers, and are written to disk. Typically only 1 page per cluster in
+ // production (one 32KB directory page holds ~2000 entries).
+ private final ITreeIndexFrame currentDirectoryFrame;
+ private final ITreeIndexTupleWriter directoryFrameTupleWriter;
+ private final List<ICachedPage> pendingDirectoryPages = new ArrayList<>();
+ private ICachedPage currentDirectoryPage;
+ private final ISketchSampler sampler;
+
+ public VTreeBulkLoader(IPageWriteCallback callback, VTree vectorTree, ITreeIndexAccessor staticAccessor,
+ ISketchSampler sampler) throws HyracksDataException {
+ this(callback, vectorTree, staticAccessor, sampler, null);
+ }
+
+ /**
+ * @param dataFrameFactoryOverride data-frame factory to write with instead of the tree's
+ * default, or {@code null} for the default. Merges pass a polarity-preserving
+ * (copy) factory so preserved delete-marker tuples are not re-encoded as matter;
+ * marker semantics are defined in the LSM layer (see hyracks-storage-am-lsm-vtree).
+ */
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_FABLE_5, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.ASSISTED)
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_FABLE_5, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.REFACTORED)
+ public VTreeBulkLoader(IPageWriteCallback callback, VTree vectorTree, ITreeIndexAccessor staticAccessor,
+ ISketchSampler sampler, ITreeIndexFrameFactory dataFrameFactoryOverride) throws HyracksDataException {
+
+ this.sampler = sampler;
+ this.bufferCache = vectorTree.getBufferCache();
+ this.freePageManager = vectorTree.getPageManager();
+ this.fileId = vectorTree.getFileId();
+ this.treeIndex = vectorTree;
+ this.metaFrame = freePageManager.createMetadataFrame();
+
+ // Initialize frames
+ this.currentDirectoryFrame = vectorTree.getMetadataFrameFactory().createFrame();
+ this.currentDataFrame = dataFrameFactoryOverride != null ? dataFrameFactoryOverride.createFrame()
+ : vectorTree.getDataFrameFactory().createFrame();
+ this.dataFrameTupleWriter = currentDataFrame.getTupleWriter();
+ this.directoryFrameTupleWriter = currentDirectoryFrame.getTupleWriter();
+ this.slotSize = currentDataFrame.getSlotSize();
+ this.currentLeafClusterIndex = -1;
+ this.currentCentroidId = -1;
+
+ this.pageWriter = bufferCache.createFIFOWriter(callback, this, DefaultBufferCacheWriteContext.INSTANCE);
+ this.compressedPageWriter = bufferCache.getCompressedPageWriter(fileId);
+
+ this.staticAccessor = (VTree.VTreeAccessor) staticAccessor;
+ VTree vtree = this.staticAccessor.getIndex();
+ ITreeIndexMetadataFrame staticMetaFrame = this.staticAccessor.getOpContext().getMetaFrame();
+ int maxPageId = vtree.getPageManager().getMaxPageId(staticMetaFrame);
+
+ LongPointable value1 = LongPointable.FACTORY.createPointable();
+ LongPointable value2 = LongPointable.FACTORY.createPointable();
+ staticMetaFrame.get(VTreeMetadataKeys.NUM_LEAF_CENTROIDS, value1);
+ staticMetaFrame.get(VTreeMetadataKeys.FIRST_LEAF_CENTROID_ID, value2);
+ this.numLeafCentroid = value1.intValue();
+ this.firstLeafCentroidId = value2.intValue();
+
+ // Only static-structure METADATA is read here. The page contents themselves are copied
+ // one page at a time in end() — the source component is immutable and remains open for
+ // the whole load, so no up-front snapshot is needed.
+ numStaticPages = maxPageId + 1;
+ // VTreeStaticStructureBuilder writes the root at the highest page id (bottom-up).
+ // Capture it now so we can translate when the static pages are copied into this
+ // data component with an offset.
+ this.staticStructureRootPage = vtree.getRootPageId();
+
+ // Initialize per-cluster directory page tracking
+ clusterFirstDirPageId = new int[numLeafCentroid];
+ for (int i = 0; i < numLeafCentroid; i++) {
+ clusterFirstDirPageId[i] = VTreeDataTupleAccessor.UNASSIGNED_DIR_PAGE;
+ }
+
+ LOGGER.log(Level.TRACE,
+ "VTreeBulkLoader initialized: numLeafCentroid={}, firstLeafCentroidId={}, numStaticPages={}",
+ numLeafCentroid, firstLeafCentroidId, numStaticPages);
+ }
+
+ /**
+ * Create a directory page confiscated with INVALID_DPID.
+ * Directory pages are kept in memory until the cluster is finalized,
+ * at which point they receive real page IDs.
+ */
+ private void createDirectoryPage() throws HyracksDataException {
+ currentDirectoryPage = bufferCache.confiscatePage(IBufferCache.INVALID_DPID);
+ currentDirectoryFrame.setPage(currentDirectoryPage);
+ currentDirectoryFrame.initBuffer((byte) 0);
+
+ LOGGER.log(Level.TRACE, "Created directory page (in-memory) for cluster {}", currentLeafClusterIndex);
+ }
+
+ private int extractCentroidId(ITupleReference tuple) {
+ // Centroid id lives at field index 1 in both the non-quantized and quantized data-tuple
+ // layouts (NQ_CENTROID_ID_FIELD == Q_CENTROID_ID_FIELD).
+ final int cidField = VTreeDataTupleAccessor.NQ_CENTROID_ID_FIELD;
+ return IntegerPointable.getInteger(tuple.getFieldData(cidField), tuple.getFieldStart(cidField));
+ }
+
+ @Override
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_4_8, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.ASSISTED)
+ public void add(ITupleReference tuple) throws HyracksDataException {
+ sampler.addTuple(tuple);
+ int tupleCentroidId = extractCentroidId(tuple);
+ if (currentCentroidId == -1) {
+ // First tuple being added - initialize for first cluster
+ LOGGER.log(Level.TRACE, "Starting bulk load with first centroid cluster: {}", tupleCentroidId);
+ currentCentroidId = tupleCentroidId;
+ currentLeafClusterIndex = tupleCentroidId - firstLeafCentroidId;
+ createDirectoryPage();
+ createNewDataPage();
+ } else if (currentCentroidId != tupleCentroidId) {
+ // Moved to a new centroid cluster
+ LOGGER.log(Level.TRACE, "Switching from centroid {} to centroid {}", currentCentroidId, tupleCentroidId);
+ currentCentroidId = tupleCentroidId;
+ int targetClusterIndex = tupleCentroidId - firstLeafCentroidId;
+ loadToNextLeafCluster(targetClusterIndex);
+ }
+ try {
+ int spaceNeeded = dataFrameTupleWriter.bytesRequired(tuple) + slotSize;
+ int spaceAvailable = currentDataFrame.getTotalFreeSpace();
+
+ // A tuple larger than a fresh empty data page's usable space (pageSize - pageHeaderSize -
+ // slotSize) can never fit; reject it here instead of looping/overrunning at insertSorted().
+ int maxUsableTupleBytes = bufferCache.getPageSize() - currentDataFrame.getPageHeaderSize() - slotSize;
+ int tupleBytes = dataFrameTupleWriter.bytesRequired(tuple);
+ if (tupleBytes > maxUsableTupleBytes) {
+ throw HyracksDataException.create(org.apache.hyracks.api.exceptions.ErrorCode.RECORD_IS_TOO_LARGE,
+ tupleBytes, maxUsableTupleBytes);
+ }
+
+ if (spaceNeeded > spaceAvailable) {
+ // The current data page cannot hold this tuple, so flush it and start a fresh one.
+ // (The empty-page case is unreachable here: the oversized-tuple guard above already
+ // rejected any tuple that would not fit in a fresh empty page, so an empty current
+ // page always has spaceAvailable >= spaceNeeded and never enters this branch. The
+ // former "if tupleCount==0 returnPage(currentDataPage)" sub-branch returned the page
+ // and then fell through to insertSorted() on it — a dead use-after-return.)
+ // Data page full - write it to disk immediately
+ finishCurrentDataPage(false);
+ }
+ ((IVTreeDataFrame) currentDataFrame).insertSorted(tuple);
+ entriesInCurrentDataPage++;
+
+ LOGGER.log(Level.TRACE, "Added tuple to leaf cluster {}, data page entries: {}", currentLeafClusterIndex,
+ entriesInCurrentDataPage);
+ } catch (HyracksDataException | RuntimeException e) {
+ logDataPageState(tuple, e);
+ handleException();
+ throw e;
+ }
+ }
+
+ /**
+ * Switch to a specific leaf cluster. Finishes the current data page,
+ * finalizes the current cluster's directory pages, then starts the new cluster.
+ */
+ public void loadToNextLeafCluster(int targetClusterIndex) throws HyracksDataException {
+ if (targetClusterIndex < 0 || targetClusterIndex >= numLeafCentroid) {
+ throw HyracksDataException.create(org.apache.hyracks.api.exceptions.ErrorCode.ILLEGAL_STATE,
+ "Target cluster index out of bounds: " + targetClusterIndex + " (valid range: 0-"
+ + (numLeafCentroid - 1) + ")");
+ }
+
+ if (currentLeafClusterIndex == targetClusterIndex) {
+ return;
+ }
+
+ // Finish current data page if it has data
+ if (currentDataPage != null && entriesInCurrentDataPage > 0) {
+ finishCurrentDataPage(true);
+ }
+
+ // Finalize directory pages for current cluster (assign IDs, chain, write)
+ finalizeClusterDirectory();
+
+ // Start new cluster
+ currentLeafClusterIndex = targetClusterIndex;
+ createDirectoryPage();
+ createNewDataPage();
+
+ LOGGER.log(Level.TRACE, "Moved to leaf cluster {} (centroid ID: {})", currentLeafClusterIndex,
+ firstLeafCentroidId + currentLeafClusterIndex);
+ }
+
+ /**
+ * Create a new data page with a real page ID.
+ * Data pages get real IDs immediately so they can be written to disk right away.
+ */
+ private void createNewDataPage() throws HyracksDataException {
+ currentDataPageId = freePageManager.takePage(metaFrame);
+ long dpid = BufferedFileHandle.getDiskPageId(fileId, currentDataPageId);
+ currentDataPage = bufferCache.confiscatePage(dpid);
+ currentDataFrame.setPage(currentDataPage);
+ currentDataFrame.initBuffer((byte) 0);
+ entriesInCurrentDataPage = 0;
+
+ LOGGER.log(Level.TRACE, "Created new data page {} for leaf cluster {}", currentDataPageId,
+ currentLeafClusterIndex);
+ }
+
+ /**
+ * Finish the current data page: set the next-page pointer, write to disk immediately,
+ * and add a directory entry for it.
+ *
+ * @param lastPage true if this is the last data page for the current cluster
+ */
+ private void finishCurrentDataPage(boolean lastPage) throws HyracksDataException {
+ int tupleCount = currentDataFrame.getTupleCount();
+ if (tupleCount == 0) {
+ return;
+ }
+
+ double maxDistance = ((IVTreeDataFrame) currentDataFrame).getDistanceToCentroid(tupleCount - 1);
+ int writtenDataPageId = currentDataPageId;
+
+ if (lastPage) {
+ // Last data page in cluster - no next page
+ ((IVTreeDataFrame) currentDataFrame).setNextPage(VTreeDataTupleAccessor.NO_NEXT_PAGE);
+
+ write(currentDataPage);
+ currentDataPage = null;
+ entriesInCurrentDataPage = 0;
+ } else {
+ // Allocate next data page ID and set forward pointer before writing
+ int nextDataPageId = freePageManager.takePage(metaFrame);
+ ((IVTreeDataFrame) currentDataFrame).setNextPage(nextDataPageId);
+
+ // Write current data page to disk immediately
+ write(currentDataPage);
+
+ // Create new data page with the pre-allocated ID
+ currentDataPageId = nextDataPageId;
+ long dpid = BufferedFileHandle.getDiskPageId(fileId, currentDataPageId);
+ currentDataPage = bufferCache.confiscatePage(dpid);
+ currentDataFrame.setPage(currentDataPage);
+ currentDataFrame.initBuffer((byte) 0);
+ entriesInCurrentDataPage = 0;
+
+ LOGGER.log(Level.TRACE, "Created new data page {} for leaf cluster {}", currentDataPageId,
+ currentLeafClusterIndex);
+ }
+
+ // Add directory entry for the written data page
+ addDirectoryEntry(maxDistance, writtenDataPageId);
+ }
+
+ /**
+ * Add a directory entry <maxDistance, dataPageId> to the current directory page.
+ * If the directory page is full, move it to the pending list and create a new overflow.
+ */
+ private void addDirectoryEntry(double maxDistance, int dataPageId) throws HyracksDataException {
+ try {
+ ITupleReference directoryEntry =
+ TupleUtils.createTuple(new ISerializerDeserializer[] { DoubleSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE }, maxDistance, dataPageId);
+
+ // Check if directory page has space
+ int spaceNeeded = directoryFrameTupleWriter.bytesRequired(directoryEntry) + slotSize;
+ int spaceAvailable = currentDirectoryFrame.getTotalFreeSpace();
+
+ if (spaceNeeded > spaceAvailable) {
+ // Directory page full - keep in pending list and create overflow
+ pendingDirectoryPages.add(currentDirectoryPage);
+ createDirectoryPage();
+
+ LOGGER.log(Level.TRACE, "Directory page full for cluster {}, created overflow",
+ currentLeafClusterIndex);
+ }
+
+ ((IVTreeFrame) currentDirectoryFrame).insertSorted(directoryEntry);
+
+ LOGGER.log(Level.TRACE, "Added directory entry for data page {} (maxDist={}) to directory, cluster {}",
+ dataPageId, maxDistance, currentLeafClusterIndex);
+
+ } catch (HyracksDataException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new HyracksDataException("Failed to create directory entry", e);
+ }
+ }
+
+ /**
+ * Finalize directory pages for the current cluster:
+ * 1. Assign real sequential page IDs to all pending directory pages
+ * 2. Set nextPage chain (dir0 -> dir1 -> ... -> -1)
+ * 3. Write all directory pages in ascending ID order
+ * 4. Record clusterFirstDirPageId for leaf frame pointer assignment
+ *
+ * Since directory page IDs are allocated after all data pages have been written,
+ * the overall write order is: data pages (lower IDs) then directory pages (higher IDs),
+ * which naturally maintains strict FIFO ordering.
+ */
+ private void finalizeClusterDirectory() throws HyracksDataException {
+ // Add current directory page to the pending list
+ if (currentDirectoryPage != null) {
+ pendingDirectoryPages.add(currentDirectoryPage);
+ currentDirectoryPage = null;
+ }
+
+ if (pendingDirectoryPages.isEmpty()) {
+ return;
+ }
+
+ // Allocate real page IDs for all directory pages
+ int numDirPages = pendingDirectoryPages.size();
+ int[] dirPageIds = new int[numDirPages];
+ for (int i = 0; i < numDirPages; i++) {
+ dirPageIds[i] = freePageManager.takePage(metaFrame);
+ }
+
+ // Set disk page IDs, nextPage chain, and write
+ for (int i = 0; i < numDirPages; i++) {
+ ICachedPage dirPage = pendingDirectoryPages.get(i);
+
+ // Assign real disk page ID
+ dirPage.setDiskPageId(BufferedFileHandle.getDiskPageId(fileId, dirPageIds[i]));
+
+ // Set nextPage chain
+ currentDirectoryFrame.setPage(dirPage);
+ if (i < numDirPages - 1) {
+ ((IVTreeMetadataFrame) currentDirectoryFrame).setNextPage(dirPageIds[i + 1]);
+ } else {
+ ((IVTreeMetadataFrame) currentDirectoryFrame).setNextPage(VTreeDataTupleAccessor.NO_NEXT_PAGE);
+
+ }
+
+ write(dirPage);
+ }
+
+ // Record first directory page ID for this cluster
+ clusterFirstDirPageId[currentLeafClusterIndex] = dirPageIds[0];
+ pendingDirectoryPages.clear();
+
+ LOGGER.log(Level.TRACE, "Finalized directory for cluster {}: {} pages, first dir page = {}",
+ currentLeafClusterIndex, numDirPages, dirPageIds[0]);
+ }
+
+ private void logDataPageState(ITupleReference tuple, Exception e) {
+ try {
+ if (currentDataFrame != null) {
+ int tupleSize = currentDataFrame.getBytesRequiredToWriteTuple(tuple);
+ int spaceNeeded = dataFrameTupleWriter.bytesRequired(tuple) + slotSize;
+ int spaceUsed = currentDataFrame.getBuffer().capacity() - currentDataFrame.getTotalFreeSpace();
+
+ LOGGER.log(Level.TRACE,
+ "Data page state - tupleSize: {}, spaceNeeded: {}, spaceUsed: {}, entriesInCurrentDataPage: {}",
+ tupleSize, spaceNeeded, spaceUsed, entriesInCurrentDataPage);
+ }
+ } catch (Throwable t) {
+ e.addSuppressed(t);
+ }
+ }
+
+ private void handleException() {
+ compressedPageWriter.abort();
+ // Return pending directory pages (confiscated with INVALID_DPID or real IDs)
+ for (ICachedPage page : pendingDirectoryPages) {
+ if (page != null && page.confiscated()) {
+ bufferCache.returnPage(page, false);
+ }
+ }
+ pendingDirectoryPages.clear();
+ if (currentDirectoryPage != null && currentDirectoryPage.confiscated()) {
+ bufferCache.returnPage(currentDirectoryPage, false);
+ currentDirectoryPage = null;
+ }
+ if (currentDataPage != null && currentDataPage.confiscated()) {
+ bufferCache.returnPage(currentDataPage, false);
+ currentDataPage = null;
+ }
+ freePageManager.returnAllPages();
+ }
+
+ @Override
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_FABLE_5, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.REFACTORED)
+ public void end() throws HyracksDataException {
+ // Finish last cluster's remaining data page
+ if (currentDataPage != null && entriesInCurrentDataPage > 0) {
+ finishCurrentDataPage(true);
+ }
+
+ // Finalize last cluster's directory pages
+ finalizeClusterDirectory();
+
+ // --- Copy static pages to end of file ---
+ // Destination page ids are deterministic: sequential from staticBasePageId, in source
+ // page-id order. This lets both passes below run with a bounded number of pages in
+ // memory (one source pin plus at most one confiscated destination page at a time).
+ int staticBasePageId = freePageManager.takePage(metaFrame);
+ // Allocate remaining S-1 pages
+ for (int i = 1; i < numStaticPages; i++) {
+ freePageManager.takePage(metaFrame);
+ }
+
+ // Create frames for pointer adjustment
+ IVTreeInteriorFrame intFrame = (IVTreeInteriorFrame) treeIndex.getInteriorFrameFactory().createFrame();
+ IVTreeLeafFrame lfFrame = (IVTreeLeafFrame) treeIndex.getLeafFrameFactory().createFrame();
+
+ // PASS 1: walk the source leaf pages one at a time to map each leaf centroid id to its
+ // final placement in this data component: {pageId, slot}. This O(K)-entry map is what
+ // allows graph-neighbor entries to be resolved during the single copy pass below without
+ // holding every copied page in memory at once.
+ Map<Integer, int[]> cidToFinalLoc = new HashMap<>();
+ for (int i = 0; i < numStaticPages; i++) {
+ ICachedPage sourcePage = staticAccessor.getCachedPage(i);
+ try {
+ lfFrame.setPage(sourcePage);
+ if (lfFrame.getLevel() == 0) {
+ int leafTupleCount = lfFrame.getTupleCount();
+ for (int t = 0; t < leafTupleCount; t++) {
+ cidToFinalLoc.put(lfFrame.getCentroidId(t), new int[] { staticBasePageId + i, t });
+ }
+ }
+ } finally {
+ staticAccessor.releasePage(sourcePage);
+ }
+ }
+
+ // PASS 2: copy each source page into one confiscated destination page, patch its
+ // pointers, and write it out before moving to the next page. A failure mid-copy would
+ // otherwise leak the in-flight confiscated destination page and leave a half-written
+ // component; track it so the catch can return it and route through handleException()
+ // (which also returns any pending directory pages and all taken pages), matching add().
+ ICachedPage inFlightPage = null;
+ try {
+ for (int i = 0; i < numStaticPages; i++) {
+ int newPageId = staticBasePageId + i;
+ long dpid = BufferedFileHandle.getDiskPageId(fileId, newPageId);
+ ICachedPage page = bufferCache.confiscatePage(dpid);
+ inFlightPage = page;
+
+ // Copy content directly from the pinned source page (released before patching)
+ ICachedPage sourcePage = staticAccessor.getCachedPage(i);
+ try {
+ System.arraycopy(sourcePage.getBuffer().array(), 0, page.getBuffer().array(), 0,
+ sourcePage.getBuffer().capacity());
+ } finally {
+ staticAccessor.releasePage(sourcePage);
+ }
+
+ // Determine page type via level field and adjust pointers
+ intFrame.setPage(page);
+ byte level = intFrame.getLevel();
+ int tupleCount = intFrame.getTupleCount();
+
+ if (level > 0) {
+ // Interior page: offset child pointers by staticBasePageId
+ for (int t = 0; t < tupleCount; t++) {
+ int oldChildId = intFrame.getChildPageId(t);
+ intFrame.setChildPageId(t, oldChildId + staticBasePageId);
+ }
+ // Offset next-page (overflow) pointer if present
+ if (intFrame.getOverflowFlagBit()) {
+ intFrame.setNextPage(intFrame.getNextPage() + staticBasePageId);
+ }
+ } else {
+ // Leaf page: set metadata pointers to actual dir page IDs
+ // Use centroidId from each tuple to compute correct cluster index,
+ // because page-ID order does NOT match centroid BFS order when
+ // overflow pages exist (overflow pages have higher IDs than
+ // subsequent clusters' main pages).
+ lfFrame.setPage(page);
+ int leafTupleCount = lfFrame.getTupleCount();
+ for (int t = 0; t < leafTupleCount; t++) {
+ int centroidId = lfFrame.getCentroidId(t);
+ int clusterIndex = centroidId - firstLeafCentroidId;
+ if (clusterIndex >= 0 && clusterIndex < numLeafCentroid) {
+ lfFrame.setMetadataPagePointer(t, clusterFirstDirPageId[clusterIndex]);
+ }
+ }
+ // Offset next-leaf pointer (overflow or sibling chain)
+ int oldNextLeaf = lfFrame.getNextLeaf();
+ if (oldNextLeaf >= 0) {
+ lfFrame.setNextLeaf(oldNextLeaf + staticBasePageId);
+ }
+ // Translate graph-neighbor pointers into this component's page space. The static
+ // structure carries resolved pointers (resolved by the builder in the scaffold's
+ // own page space), so each is offset by staticBasePageId — the same shift applied
+ // to child/next-leaf pointers above. Any still-provisional entry (e.g. from an
+ // older scaffold) is resolved from the pass-1 directory instead.
+ resolveLeafNeighborPointers(lfFrame, cidToFinalLoc, staticBasePageId);
+ }
+
+ write(page);
+ inFlightPage = null;
+ }
+ } catch (HyracksDataException | RuntimeException e) {
+ if (inFlightPage != null && inFlightPage.confiscated()) {
+ bufferCache.returnPage(inFlightPage, false);
+ }
+ handleException();
+ throw e;
+ }
+
+ // Set root page and metadata. In bottom-up static structures the root sits at
+ // the highest page id; staticStructureRootPage captures that offset within the
+ // static page id space, and we translate it into this data component's space
+ // by adding staticBasePageId.
+ int rootPageId = staticBasePageId + staticStructureRootPage;
+ ((VTree) treeIndex).setRootPageId(rootPageId);
+ freePageManager.setRootPageId(rootPageId);
+
+ metaFrame.put(VTreeMetadataKeys.NUM_LEAF_CENTROIDS, LongPointable.FACTORY.createPointable(numLeafCentroid));
+ metaFrame.put(VTreeMetadataKeys.FIRST_LEAF_CENTROID_ID,
+ LongPointable.FACTORY.createPointable(firstLeafCentroidId));
+
+ if (hasFailed()) {
+ throw HyracksDataException.create(getFailure());
+ }
+ }
+
+ private void write(ICachedPage cPage) throws HyracksDataException {
+ compressedPageWriter.prepareWrite(cPage);
+ pageWriter.write(cPage);
+ }
+
+ /**
+ * Resolve the given leaf page's graph-neighbor lists from provisional form (neighbor centroid ids)
+ * to resolved physical pointers ({pageId, slot} in this data component), using the directory built
+ * in pass 1 of {@link #end()}. The leaf frame is already set on the destination page. Quantized
+ * leaf tuples carry the neighbor list at field index 3 (a length-prefixed byte array of
+ * fixed-width entries); leaf layouts without a neighbor field are skipped, as are entries whose
+ * neighbor centroid id is not in the directory (dangling).
+ * <p>
+ * The page is rewritten in place (same byte width, so slots do not move) before it is written
+ * exactly once by the caller — no page is re-uploaded, honoring the cloud append-only rule.
+ */
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_FABLE_5, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.REFACTORED)
+ private void resolveLeafNeighborPointers(IVTreeLeafFrame lfFrame, Map<Integer, int[]> cidToFinalLoc,
+ int staticBasePageId) throws HyracksDataException {
+ VTreeLeafNeighborList.forEachLeafNeighborEntry(lfFrame, (neighborList, start, e) -> {
+ if (VTreeLeafNeighborList.isResolved(neighborList, start, e)) {
+ // Scaffold already resolved this to (pageId, slot) in its own 0-based page space;
+ // shift the page id into this component's space (slot copied byte-for-byte).
+ int pageId = VTreeLeafNeighborList.readPageId(neighborList, start, e);
+ int slot = VTreeLeafNeighborList.readSlot(neighborList, start, e);
+ VTreeLeafNeighborList.writeResolved(neighborList, start, e, pageId + staticBasePageId, slot);
+ } else {
+ // Fallback for a still-provisional entry: resolve from this copy's directory,
+ // which already maps to component-space (page, slot).
+ int neighborCid = VTreeLeafNeighborList.readCentroidId(neighborList, start, e);
+ int[] loc = cidToFinalLoc.get(neighborCid);
+ if (loc != null) {
+ VTreeLeafNeighborList.writeResolved(neighborList, start, e, loc[0], loc[1]);
+ }
+ }
+ });
+ }
+
+ @Override
+ public void abort() throws HyracksDataException {
+ LOGGER.log(Level.TRACE, "VTreeBulkLoader aborted");
+ handleException();
+ }
+
+ @Override
+ public void force() throws HyracksDataException {
+ bufferCache.force(fileId, false);
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/impls/VTreeCursorInitialState.java b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/impls/VTreeCursorInitialState.java
new file mode 100644
index 0000000000..4480b1bb14
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/impls/VTreeCursorInitialState.java
@@ -0,0 +1,167 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.vector.impls;
+
+import org.apache.hyracks.storage.am.vector.api.IVTreeDistanceFunction;
+import org.apache.hyracks.storage.am.vector.api.IVTreeQuantizer;
+import org.apache.hyracks.storage.common.ICursorInitialState;
+import org.apache.hyracks.storage.common.IIndexAccessor;
+import org.apache.hyracks.storage.common.ISearchOperationCallback;
+import org.apache.hyracks.storage.common.MultiComparator;
+import org.apache.hyracks.storage.common.buffercache.ICachedPage;
+
+/**
+ * Initial state for vector clustering tree cursors.
+ * Contains information needed to position the cursor at the appropriate location.
+ */
+public class VTreeCursorInitialState implements ICursorInitialState {
+
+ private long metadataPageId;
+ private int rootPageId;
+ private double[] queryVector;
+ private ICachedPage page;
+ private ISearchOperationCallback searchCallback;
+ private MultiComparator originalKeyCmp;
+ private final IIndexAccessor accessor;
+ private IVTreeDistanceFunction distanceFunction;
+ private double[] quantizedQueryVector;
+ private IVTreeQuantizer quantizer;
+
+ public VTreeCursorInitialState(IIndexAccessor accessor) {
+ this.accessor = accessor;
+ this.metadataPageId = -1;
+ }
+
+ public VTreeCursorInitialState(long metadataPageId, double[] queryVector, IIndexAccessor accessor) {
+ this.metadataPageId = metadataPageId;
+ // No defensive copy: the query vector is effectively immutable for the duration of a search, and the
+ // setQueryVector() path stores it by reference too — keep the two consistent.
+ this.queryVector = queryVector;
+ this.accessor = accessor;
+ }
+
+ public IIndexAccessor getIndexAccessor() {
+ return accessor;
+ }
+
+ public void setMetadataPageId(long metadataPageId) {
+ this.metadataPageId = metadataPageId;
+ }
+
+ public long getMetadataPageId() {
+ return metadataPageId;
+ }
+
+ public void setQueryVector(double[] queryVector) {
+ this.queryVector = queryVector;
+ }
+
+ public double[] getQueryVector() {
+ return queryVector;
+ }
+
+ public void setRootPageId(int rootPageId) {
+ this.rootPageId = rootPageId;
+ }
+
+ public int getRootPageId() {
+ return rootPageId;
+ }
+
+ @Override
+ public ICachedPage getPage() {
+ return page;
+ }
+
+ @Override
+ public void setPage(ICachedPage page) {
+ this.page = page;
+ }
+
+ @Override
+ public ISearchOperationCallback getSearchOperationCallback() {
+ return searchCallback;
+ }
+
+ @Override
+ public void setSearchOperationCallback(ISearchOperationCallback searchCallback) {
+ this.searchCallback = searchCallback;
+ }
+
+ @Override
+ public MultiComparator getOriginalKeyComparator() {
+ return originalKeyCmp;
+ }
+
+ @Override
+ public void setOriginialKeyComparator(MultiComparator originalCmp) {
+ this.originalKeyCmp = originalCmp;
+ }
+
+ @Override
+ public String toString() {
+ return "VTreeCursorInitialState[metadataPageId=" + metadataPageId + "]";
+ }
+
+ public IIndexAccessor getAccessor() {
+ return accessor;
+ }
+
+ /**
+ * Set the distance function for vector distance calculations.
+ */
+ public void setDistanceFunction(IVTreeDistanceFunction distanceFunction) {
+ this.distanceFunction = distanceFunction;
+ }
+
+ /**
+ * Get the distance function for vector distance calculations.
+ */
+ public IVTreeDistanceFunction getDistanceFunction() {
+ return distanceFunction;
+ }
+
+ /**
+ * Set the quantized query vector (precomputed once per search).
+ */
+ public void setQuantizedQueryVector(double[] quantizedQueryVector) {
+ this.quantizedQueryVector = quantizedQueryVector;
+ }
+
+ /**
+ * Get the quantized query vector, or null if quantization is not configured.
+ */
+ public double[] getQuantizedQueryVector() {
+ return quantizedQueryVector;
+ }
+
+ /**
+ * Set the vector quantizer.
+ */
+ public void setQuantizer(IVTreeQuantizer quantizer) {
+ this.quantizer = quantizer;
+ }
+
+ /**
+ * Get the vector quantizer, or null if quantization is not configured.
+ */
+ public IVTreeQuantizer getQuantizer() {
+ return quantizer;
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/impls/VTreeDataTupleBuilder.java b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/impls/VTreeDataTupleBuilder.java
new file mode 100644
index 0000000000..72d9ea8b48
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/impls/VTreeDataTupleBuilder.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.vector.impls;
+
+import java.io.DataOutput;
+import java.nio.ByteBuffer;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleReference;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.storage.am.vector.api.IVTreeDataTupleBuilder;
+import org.apache.hyracks.storage.am.vector.api.VTreeQuantizationParams;
+import org.apache.hyracks.storage.am.vector.utils.VTreeDataTupleAccessor;
+import org.apache.hyracks.util.encoding.VarLenIntEncoderDecoder;
+
+/**
+ * Transforms input tuples from operator format to VTree data page storage format.
+ *
+ * Input tuple format: [vector, include_fields..., pk]
+ *
+ * Non-quantized output: [distance, centroidId, pk, include_fields...]
+ * Quantized output: [distance, centroidId, quantized_distance, quantized_embedding, pk, include_fields...]
+ *
+ * The returned ITupleReference is valid until the next call to {@link #buildDataTuple}.
+ */
+public class VTreeDataTupleBuilder implements IVTreeDataTupleBuilder {
+
+ private final int numIncludeFields;
+
+ private final boolean isQuantized;
+ private final VTreeQuantizationParams quantizationParams;
+ private final ArrayTupleBuilder tupleBuilder;
+ private final ArrayTupleReference tupleRef;
+ // Reused per-tuple scratch (insert/bulk-load hot path): a 5-byte VarLen length-prefix buffer (max varlen
+ // width of a 32-bit int), the quantized-embedding buffer, and the test-only double-serialization buffer.
+ // Each produced value is copied into the tuple before the next call, so the buffers are safe to reuse.
+ private final byte[] varlenMeta = new byte[5];
+ private byte[] quantizeScratch;
+ private ByteBuffer fallbackBuf;
+
+ public VTreeDataTupleBuilder(int numIncludeFields, boolean isQuantized,
+ VTreeQuantizationParams quantizationParams) {
+ this.numIncludeFields = numIncludeFields;
+ this.isQuantized = isQuantized;
+ this.quantizationParams = quantizationParams;
+ // Field order and count are owned by VTreeDataTupleAccessor: [secondary fields][1 PK][includes].
+ int fieldCount = new VTreeDataTupleAccessor(isQuantized).numSecondaryFields() + 1 + numIncludeFields;
+
+ this.tupleBuilder = new ArrayTupleBuilder(fieldCount);
+ this.tupleRef = new ArrayTupleReference();
+ }
+
+ @Override
+ public ITupleReference buildDataTuple(double[] vector, double distance, int centroidId,
+ ITupleReference originalTuple) throws HyracksDataException {
+ try {
+ tupleBuilder.reset();
+ DataOutput dos = tupleBuilder.getDataOutput();
+
+ // Field 0: distance (raw double)
+ dos.writeDouble(distance);
+ tupleBuilder.addFieldEndOffset();
+
+ // Field 1: centroidId (raw int)
+ dos.writeInt(centroidId);
+ tupleBuilder.addFieldEndOffset();
+
+ if (isQuantized) {
+ writeQuantizedFields(dos, vector, distance);
+ }
+
+ // PK field (at position 1 + numIncludeFields in input)
+ int pkFieldIndex = 1 + numIncludeFields;
+ tupleBuilder.addField(originalTuple.getFieldData(pkFieldIndex), originalTuple.getFieldStart(pkFieldIndex),
+ originalTuple.getFieldLength(pkFieldIndex));
+
+ // Include fields (from input fields 1..numIncludeFields)
+ for (int i = 0; i < numIncludeFields; i++) {
+ int srcFieldIndex = 1 + i;
+ tupleBuilder.addField(originalTuple.getFieldData(srcFieldIndex),
+ originalTuple.getFieldStart(srcFieldIndex), originalTuple.getFieldLength(srcFieldIndex));
+ }
+
+ tupleRef.reset(tupleBuilder.getFieldEndOffsets(), tupleBuilder.getByteArray());
+ return tupleRef;
+
+ } catch (Exception e) {
+ throw HyracksDataException.create(e);
+ }
+ }
+
+ private void writeQuantizedFields(DataOutput dos, double[] vector, double distance) throws Exception {
+ // Field 2: quantized distance (raw double)
+ dos.writeDouble(distance);
+ tupleBuilder.addFieldEndOffset();
+
+ // Field 3: quantized embedding (VarLen length prefix + content bytes). Encode the prefix into the
+ // reused varlenMeta buffer and write it straight to the output instead of allocating a byte[] per call.
+ byte[] quantizedEmbedding = quantizeVector(vector);
+ int metaLength = VarLenIntEncoderDecoder.encode(quantizedEmbedding.length, varlenMeta, 0);
+ dos.write(varlenMeta, 0, metaLength);
+ dos.write(quantizedEmbedding);
+ tupleBuilder.addFieldEndOffset();
+ }
+
+ private byte[] quantizeVector(double[] vector) {
+ if (quantizationParams != null) {
+ float minQ = quantizationParams.minQuantile();
+ float maxQ = quantizationParams.maxQuantile();
+ float alpha = quantizationParams.alpha();
+ int bits = quantizationParams.bits();
+
+ int levels = 1 << bits;
+ if (quantizeScratch == null || quantizeScratch.length != vector.length) {
+ quantizeScratch = new byte[vector.length];
+ }
+ byte[] result = quantizeScratch;
+ for (int i = 0; i < vector.length; i++) {
+ double value = Math.max(minQ, Math.min(maxQ, vector[i]));
+ int quantizedValue = Math.toIntExact(Math.round((value - minQ) * alpha));
+ quantizedValue = Math.max(0, Math.min(levels - 1, quantizedValue));
+ result[i] = (byte) quantizedValue;
+ }
+ return result;
+ }
+ // Fallback: serialize full-precision vector as raw big-endian doubles (tests), reusing the buffer.
+ int byteLen = vector.length * Double.BYTES;
+ if (fallbackBuf == null || fallbackBuf.capacity() != byteLen) {
+ fallbackBuf = ByteBuffer.allocate(byteLen);
+ }
+ fallbackBuf.clear();
+ for (double d : vector) {
+ fallbackBuf.putDouble(d);
+ }
+ return fallbackBuf.array();
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/impls/VTreeDataTupleBuilderFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/impls/VTreeDataTupleBuilderFactory.java
new file mode 100644
index 0000000000..e7c2702d3a
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/impls/VTreeDataTupleBuilderFactory.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.vector.impls;
+
+import org.apache.hyracks.storage.am.vector.api.IVTreeDataTupleBuilder;
+import org.apache.hyracks.storage.am.vector.api.IVTreeDataTupleBuilderFactory;
+import org.apache.hyracks.storage.am.vector.api.VTreeQuantizationParams;
+
+/**
+ * Factory for creating {@link VTreeDataTupleBuilder} instances.
+ *
+ * Carries the tuple layout context (numIncludeFields, isQuantized) determined
+ * at index creation time. Quantization parameters are supplied at creator
+ * construction time since they are loaded from LSMVTreeLocalResource metadata at index
+ * activation, after this factory has been persisted.
+ */
+public class VTreeDataTupleBuilderFactory implements IVTreeDataTupleBuilderFactory {
+
+ private static final long serialVersionUID = 2L;
+
+ private final int numIncludeFields;
+ private final boolean isQuantized;
+
+ public VTreeDataTupleBuilderFactory(int numIncludeFields, boolean isQuantized) {
+ this.numIncludeFields = numIncludeFields;
+ this.isQuantized = isQuantized;
+ }
+
+ @Override
+ public IVTreeDataTupleBuilder createDataTupleBuilder(VTreeQuantizationParams quantizationParams) {
+ return new VTreeDataTupleBuilder(numIncludeFields, isQuantized, quantizationParams);
+ }
+
+ @Override
+ public boolean isQuantized() {
+ return isQuantized;
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/impls/VTreeFlushLoader.java b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/impls/VTreeFlushLoader.java
new file mode 100644
index 0000000000..b8840c43f3
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/impls/VTreeFlushLoader.java
@@ -0,0 +1,249 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.vector.impls;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.primitive.LongPointable;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.storage.am.common.api.IPageManager;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexMetadataFrame;
+import org.apache.hyracks.storage.am.common.impls.AbstractTreeIndex;
+import org.apache.hyracks.storage.am.vector.api.IVTreeInteriorFrame;
+import org.apache.hyracks.storage.am.vector.api.IVTreeLeafFrame;
+import org.apache.hyracks.storage.am.vector.utils.VTreeLeafNeighborList;
+import org.apache.hyracks.storage.am.vector.utils.VTreeMetadataKeys;
+import org.apache.hyracks.storage.common.IIndexBulkLoader;
+import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.buffercache.ICachedPage;
+import org.apache.hyracks.storage.common.buffercache.IFIFOPageWriter;
+import org.apache.hyracks.storage.common.buffercache.IPageWriteCallback;
+import org.apache.hyracks.storage.common.buffercache.PageWriteFailureCallback;
+import org.apache.hyracks.storage.common.buffercache.context.write.DefaultBufferCacheWriteContext;
+import org.apache.hyracks.storage.common.compression.file.ICompressedPageWriter;
+import org.apache.hyracks.storage.common.file.BufferedFileHandle;
+import org.apache.hyracks.util.annotations.AiProvenance;
+
+/**
+ * Flush loader for VTree that writes memory component (VBC) pages to a disk component,
+ * then appends the static structure pages at the end.
+ * <p>
+ * The identity mapping (VBC page N -> disk page N) applies only to {@link #copyPage} — the
+ * data/directory pages copied verbatim. {@link #copyStaticStructure} is NOT identity: it appends
+ * the static pages at an offset and rewrites their pointers accordingly.
+ */
+public class VTreeFlushLoader extends PageWriteFailureCallback implements IIndexBulkLoader {
+
+ private final IBufferCache bufferCache;
+ private final IPageManager freePageManager;
+ private final ITreeIndexMetadataFrame metaFrame;
+ private final AbstractTreeIndex treeIndex;
+ private final int fileId;
+ private final IFIFOPageWriter pageWriter;
+ private final ICompressedPageWriter compressedPageWriter;
+
+ // Source memory component info (for directory page identification during static structure copy)
+ private final VTree sourceMemoryTree;
+
+ public VTreeFlushLoader(IPageWriteCallback callback, VTree diskTree, VTree sourceMemoryTree) {
+ this.bufferCache = diskTree.getBufferCache();
+ this.freePageManager = diskTree.getPageManager();
+ this.metaFrame = freePageManager.createMetadataFrame();
+ this.fileId = diskTree.getFileId();
+ this.treeIndex = diskTree;
+ this.sourceMemoryTree = sourceMemoryTree;
+ this.pageWriter = bufferCache.createFIFOWriter(callback, this, DefaultBufferCacheWriteContext.INSTANCE);
+ this.compressedPageWriter = bufferCache.getCompressedPageWriter(fileId);
+ }
+
+ @Override
+ public void add(ITupleReference tuple) throws HyracksDataException {
+ throw new UnsupportedOperationException("Use copyPage() instead");
+ }
+
+ /**
+ * Copy a VBC page to disk with identity mapping (VBC page N -> disk page N).
+ */
+ public void copyPage(ICachedPage sourcePage) throws HyracksDataException {
+ int diskPageId = freePageManager.takePage(metaFrame);
+ long dpid = BufferedFileHandle.getDiskPageId(fileId, diskPageId);
+ ICachedPage targetPage = bufferCache.confiscatePage(dpid);
+ System.arraycopy(sourcePage.getBuffer().array(), 0, targetPage.getBuffer().array(), 0,
+ sourcePage.getBuffer().capacity());
+ write(targetPage);
+ }
+
+ /**
+ * Copy static structure pages to end of file with pointer adjustment.
+ * Interior child pointers are offset by staticBasePageId.
+ * Leaf metadata pointers are set from the source memory tree's centroidDirPageMap
+ * (identity mapping: VBC page IDs = disk page IDs).
+ * Leaf next-page pointers are offset by staticBasePageId.
+ * <p>
+ * Bounded memory: each source page is pinned, copied into one confiscated destination page,
+ * patched, written, and released before the next page is touched — no up-front snapshot of
+ * the whole structure and no simultaneous confiscation of every destination page.
+ *
+ * @param staticAccessor accessor to the static structure disk component
+ * @return the root page ID of the flushed component: staticBasePageId offset by the static
+ * tree's own root page id (bottom-up layout puts leaves first and the root at the
+ * highest static page id, so the first copied page is a LEAF, not the root)
+ */
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_FABLE_5, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.ASSISTED)
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_FABLE_5, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.REFACTORED)
+ public int copyStaticStructure(VTree.VTreeAccessor staticAccessor) throws HyracksDataException {
+
+ VTree staticTree = staticAccessor.getIndex();
+ ITreeIndexMetadataFrame staticMeta = staticAccessor.getOpContext().getMetaFrame();
+ int maxStaticPageId = staticTree.getPageManager().getMaxPageId(staticMeta);
+ int numStaticPages = maxStaticPageId + 1;
+
+ // Allocate a CONTIGUOUS block of disk pages for the static structure. takeBlock reserves
+ // numStaticPages consecutive pages and returns the first id, so the destination-id math below
+ // (staticBasePageId + i) is guaranteed. A per-page takePage() loop is NOT safe here: on a free-list
+ // page manager takePage() can hand back non-contiguous ids, which would break the +i offset.
+ int staticBasePageId = freePageManager.takeBlock(metaFrame, numStaticPages);
+
+ // Create frames for pointer adjustment
+ IVTreeInteriorFrame intFrame = (IVTreeInteriorFrame) treeIndex.getInteriorFrameFactory().createFrame();
+ IVTreeLeafFrame lfFrame = (IVTreeLeafFrame) treeIndex.getLeafFrameFactory().createFrame();
+
+ int[] centroidDirPageMap = sourceMemoryTree.getCentroidDirPageMap();
+ int numLeafCentroid = sourceMemoryTree.getNumLeafCentroidMem();
+ int firstLeafCid = sourceMemoryTree.getFirstLeafCentroidIdMem();
+
+ // Copy one source/destination page pair at a time: pin source, copy into the confiscated
+ // destination page, release the source, patch pointers, write, then move on.
+ for (int i = 0; i < numStaticPages; i++) {
+ int newPageId = staticBasePageId + i;
+ long dpid = BufferedFileHandle.getDiskPageId(fileId, newPageId);
+ ICachedPage page = bufferCache.confiscatePage(dpid);
+
+ ICachedPage sourcePage = staticAccessor.getCachedPage(i);
+ try {
+ System.arraycopy(sourcePage.getBuffer().array(), 0, page.getBuffer().array(), 0,
+ sourcePage.getBuffer().capacity());
+ } finally {
+ staticAccessor.releasePage(sourcePage);
+ }
+
+ // Determine page type via level field and adjust pointers
+ intFrame.setPage(page);
+ byte level = intFrame.getLevel();
+
+ if (level > 0) {
+ // Interior page: offset child pointers by staticBasePageId
+ for (int t = 0; t < intFrame.getTupleCount(); t++) {
+ int oldChildId = intFrame.getChildPageId(t);
+ intFrame.setChildPageId(t, oldChildId + staticBasePageId);
+ }
+ // Offset next-page (overflow) pointer if present
+ if (intFrame.getOverflowFlagBit()) {
+ intFrame.setNextPage(intFrame.getNextPage() + staticBasePageId);
+ }
+ } else {
+ // Leaf page: set metadata pointers to VBC directory page IDs
+ // (identity mapping means VBC page IDs = disk page IDs). Index
+ // centroidDirPageMap by the slot's centroid_id (cid - firstLeafCid), not by
+ // traversal order, since physical page-id order need not match the nextLeaf chain.
+ lfFrame.setPage(page);
+ for (int t = 0; t < lfFrame.getTupleCount(); t++) {
+ int cid = lfFrame.getCentroidId(t);
+ int idx = cid - firstLeafCid;
+ if (idx >= 0 && idx < numLeafCentroid) {
+ lfFrame.setMetadataPagePointer(t, centroidDirPageMap[idx]);
+ }
+ }
+ // Offset the next-leaf pointer. The next-leaf field is dual-purpose: with the overflow flag
+ // set it is an intra-cluster overflow chain; with the flag clear it is the sibling link to the
+ // next cluster's first leaf. We deliberately gate on nextLeaf >= 0 (not on the overflow flag,
+ // unlike the interior branch) because BOTH kinds are static-range page ids that need the same
+ // +staticBasePageId shift — only navigation cares which kind it is (to stop at cluster
+ // boundaries).
+ int oldNextLeaf = lfFrame.getNextLeaf();
+ if (oldNextLeaf >= 0) {
+ lfFrame.setNextLeaf(oldNextLeaf + staticBasePageId);
+ }
+ // Offset resolved graph-neighbor pointers by staticBasePageId (slot unchanged), the same
+ // shift applied to next-leaf/metadata pointers — the scaffold carries resolved pointers.
+ offsetLeafNeighborPointers(lfFrame, staticBasePageId);
+ }
+
+ write(page);
+ }
+
+ // The static structure's root sits at its own root page id within the copied range
+ // (mirrors VTreeBulkLoader#end: rootPageId = staticBasePageId + staticStructureRootPage).
+ // Returning staticBasePageId itself would persist the first copied LEAF page as root.
+ return staticBasePageId + staticTree.getRootPageId();
+ }
+
+ /**
+ * Shift each resolved graph-neighbor pointer on the given leaf page (already set on {@code lfFrame})
+ * by {@code staticBasePageId}, translating scaffold-space page ids into this flushed component's
+ * space. The slot is unchanged (pages are copied byte-for-byte). Leaf layouts without a neighbor
+ * field, and any still-provisional entry, are left untouched.
+ */
+ private void offsetLeafNeighborPointers(IVTreeLeafFrame lfFrame, int staticBasePageId) throws HyracksDataException {
+ VTreeLeafNeighborList.forEachLeafNeighborEntry(lfFrame, (neighborList, start, e) -> {
+ if (!VTreeLeafNeighborList.isResolved(neighborList, start, e)) {
+ return;
+ }
+ int pageId = VTreeLeafNeighborList.readPageId(neighborList, start, e);
+ int slot = VTreeLeafNeighborList.readSlot(neighborList, start, e);
+ VTreeLeafNeighborList.writeResolved(neighborList, start, e, pageId + staticBasePageId, slot);
+ });
+ }
+
+ /**
+ * Finalize the flushed disk component with correct metadata.
+ */
+ public void end(int numLeafCentroid, int firstLeafCentroidId, int rootPageId) throws HyracksDataException {
+ ((VTree) treeIndex).setRootPageId(rootPageId);
+ freePageManager.setRootPageId(rootPageId);
+
+ metaFrame.put(VTreeMetadataKeys.NUM_LEAF_CENTROIDS, LongPointable.FACTORY.createPointable(numLeafCentroid));
+ metaFrame.put(VTreeMetadataKeys.FIRST_LEAF_CENTROID_ID,
+ LongPointable.FACTORY.createPointable(firstLeafCentroidId));
+
+ if (hasFailed()) {
+ throw HyracksDataException.create(getFailure());
+ }
+ }
+
+ @Override
+ public void end() throws HyracksDataException {
+ throw new UnsupportedOperationException("Use end(numLeafCentroid, firstLeafCentroidId, rootPageId) instead");
+ }
+
+ @Override
+ public void abort() throws HyracksDataException {
+ compressedPageWriter.abort();
+ freePageManager.returnAllPages();
+ }
+
+ @Override
+ public void force() throws HyracksDataException {
+ bufferCache.force(fileId, false);
+ }
+
+ private void write(ICachedPage cPage) throws HyracksDataException {
+ compressedPageWriter.prepareWrite(cPage);
+ pageWriter.write(cPage);
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/impls/VTreeOpContext.java b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/impls/VTreeOpContext.java
new file mode 100644
index 0000000000..75505ab5b2
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/impls/VTreeOpContext.java
@@ -0,0 +1,257 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.vector.impls;
+
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Deque;
+import java.util.List;
+
+import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.common.api.IIndexOperationContext;
+import org.apache.hyracks.storage.am.common.api.IPageManager;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexMetadataFrame;
+import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
+import org.apache.hyracks.storage.am.vector.api.IVTreeDataFrame;
+import org.apache.hyracks.storage.am.vector.api.IVTreeDataTupleBuilder;
+import org.apache.hyracks.storage.am.vector.api.IVTreeDataTupleBuilderFactory;
+import org.apache.hyracks.storage.am.vector.api.IVTreeInteriorFrame;
+import org.apache.hyracks.storage.am.vector.api.IVTreeLeafFrame;
+import org.apache.hyracks.storage.am.vector.api.IVTreeMetadataFrame;
+import org.apache.hyracks.storage.am.vector.api.VTreeQuantizationParams;
+import org.apache.hyracks.storage.common.IIndexAccessor;
+import org.apache.hyracks.storage.common.IModificationOperationCallback;
+import org.apache.hyracks.storage.common.ISearchOperationCallback;
+import org.apache.hyracks.storage.common.MultiComparator;
+import org.apache.hyracks.storage.common.buffercache.IExtraPageBlockHelper;
+
+/**
+ * Operation context for vector clustering tree operations.
+ * <p>
+ * Not thread-safe: holds mutable per-operation frames/state and is confined to a single operation
+ * context (one accessor / one thread at a time).
+ */
+public class VTreeOpContext implements IIndexOperationContext, IExtraPageBlockHelper {
+
+ private static final int INIT_ARRAYLIST_SIZE = 6;
+
+ private final IIndexAccessor accessor;
+ private final MultiComparator cmp;
+ private final ITreeIndexFrameFactory interiorFrameFactory;
+ private final ITreeIndexFrameFactory leafFrameFactory;
+ private final ITreeIndexFrameFactory metadataFrameFactory;
+ private ITreeIndexFrameFactory dataFrameFactory; // Mutable for LSMBTree-style frame switching
+ private final IPageManager freePageManager;
+ private final ITreeIndexMetadataFrame metaFrame;
+ private final int vectorDimensions;
+ private final IVTreeDataTupleBuilder dataTupleBuilder;
+
+ private IVTreeInteriorFrame interiorFrame;
+ private IVTreeLeafFrame leafFrame;
+ private IVTreeMetadataFrame metadataFrame;
+ private IVTreeDataFrame dataFrame;
+
+ private IndexOperation op;
+ private IModificationOperationCallback modificationCallback;
+ private ISearchOperationCallback searchCallback;
+
+ private Deque<Long> pageLsns;
+ private List<Integer> smPages;
+ private int opRestarts = 0;
+ private boolean destroyed = false;
+ private long metadataPageId = -1;
+
+ public VTreeOpContext(IIndexAccessor accessor, ITreeIndexFrameFactory interiorFrameFactory,
+ ITreeIndexFrameFactory leafFrameFactory, ITreeIndexFrameFactory metadataFrameFactory,
+ ITreeIndexFrameFactory dataFrameFactory, IPageManager freePageManager,
+ IBinaryComparatorFactory[] cmpFactories, int vectorDimensions,
+ IModificationOperationCallback modificationCallback, ISearchOperationCallback searchCallback,
+ IVTreeDataTupleBuilderFactory dataTupleBuilderFactory, VTreeQuantizationParams quantizationParams) {
+ this.accessor = accessor;
+ this.interiorFrameFactory = interiorFrameFactory;
+ this.leafFrameFactory = leafFrameFactory;
+ this.metadataFrameFactory = metadataFrameFactory;
+ this.dataFrameFactory = dataFrameFactory;
+ this.freePageManager = freePageManager;
+ this.vectorDimensions = vectorDimensions;
+ this.modificationCallback = modificationCallback;
+ this.searchCallback = searchCallback;
+ this.dataTupleBuilder = dataTupleBuilderFactory.createDataTupleBuilder(quantizationParams);
+
+ // A null / empty cmpFactories array (or a null first entry) means "no comparator" for this
+ // context; guard the [0] access so a missing array does not surface as an obscure NPE/AIOOBE.
+ if (cmpFactories != null && cmpFactories.length > 0 && cmpFactories[0] != null) {
+ this.cmp = MultiComparator.create(cmpFactories);
+ } else {
+ this.cmp = null;
+ }
+
+ this.interiorFrame = (IVTreeInteriorFrame) interiorFrameFactory.createFrame();
+ this.leafFrame = (IVTreeLeafFrame) leafFrameFactory.createFrame();
+ this.metadataFrame = (IVTreeMetadataFrame) metadataFrameFactory.createFrame();
+ this.dataFrame = (IVTreeDataFrame) dataFrameFactory.createFrame();
+ this.metaFrame = freePageManager.createMetadataFrame();
+
+ this.pageLsns = new ArrayDeque<>(INIT_ARRAYLIST_SIZE);
+ this.smPages = new ArrayList<>(INIT_ARRAYLIST_SIZE);
+ }
+
+ @Override
+ public void reset() {
+ pageLsns.clear();
+ smPages.clear();
+ opRestarts = 0;
+ }
+
+ @Override
+ public void setOperation(IndexOperation newOp) {
+ this.op = newOp;
+ }
+
+ @Override
+ public IndexOperation getOperation() {
+ return op;
+ }
+
+ @Override
+ public void destroy() throws HyracksDataException {
+ if (destroyed) {
+ return;
+ }
+ destroyed = true;
+ // Cleanup resources if needed
+ }
+
+ // Getters
+ public IIndexAccessor getAccessor() {
+ return accessor;
+ }
+
+ public ITreeIndexFrameFactory getInteriorFrameFactory() {
+ return interiorFrameFactory;
+ }
+
+ public ITreeIndexFrameFactory getLeafFrameFactory() {
+ return leafFrameFactory;
+ }
+
+ public ITreeIndexFrameFactory getMetadataFrameFactory() {
+ return metadataFrameFactory;
+ }
+
+ public ITreeIndexFrameFactory getDataFrameFactory() {
+ return dataFrameFactory;
+ }
+
+ public IPageManager getFreePageManager() {
+ return freePageManager;
+ }
+
+ public IVTreeInteriorFrame getInteriorFrame() {
+ return interiorFrame;
+ }
+
+ public IVTreeLeafFrame getLeafFrame() {
+ return leafFrame;
+ }
+
+ public IVTreeMetadataFrame getMetadataFrame() {
+ return metadataFrame;
+ }
+
+ public IVTreeDataFrame getDataFrame() {
+ return dataFrame;
+ }
+
+ public ITreeIndexMetadataFrame getMetaFrame() {
+ return metaFrame;
+ }
+
+ public MultiComparator getCmp() {
+ return cmp;
+ }
+
+ public int getVectorDimensions() {
+ return vectorDimensions;
+ }
+
+ public IVTreeDataTupleBuilder getDataTupleBuilder() {
+ return dataTupleBuilder;
+ }
+
+ public IModificationOperationCallback getModificationCallback() {
+ return modificationCallback;
+ }
+
+ public ISearchOperationCallback getSearchCallback() {
+ return searchCallback;
+ }
+
+ public Deque<Long> getPageLsns() {
+ return pageLsns;
+ }
+
+ public List<Integer> getSmPages() {
+ return smPages;
+ }
+
+ public int getOpRestarts() {
+ return opRestarts;
+ }
+
+ public void setOpRestarts(int opRestarts) {
+ this.opRestarts = opRestarts;
+ }
+
+ // IExtraPageBlockHelper methods
+ @Override
+ public int getFreeBlock(int size) throws HyracksDataException {
+ return freePageManager.takeBlock(metaFrame, size);
+ }
+
+ @Override
+ public void returnFreePageBlock(int blockPageId, int size) throws HyracksDataException {
+ freePageManager.releaseBlock(metaFrame, blockPageId, size);
+ }
+
+ public void setCallbacks(IModificationOperationCallback modificationCallback,
+ ISearchOperationCallback searchCallback) {
+ this.modificationCallback = modificationCallback;
+ this.searchCallback = searchCallback;
+ }
+
+ public long getMetadataPageId() {
+ return metadataPageId;
+ }
+
+ public void setMetadataPageId(long metadataPageId) {
+ this.metadataPageId = metadataPageId;
+ }
+
+ public void setDataFrame(IVTreeDataFrame dataFrame) {
+ this.dataFrame = dataFrame;
+ }
+
+ public void setDataFrameFactory(ITreeIndexFrameFactory dataFrameFactory) {
+ this.dataFrameFactory = dataFrameFactory;
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/impls/VTreeSearchCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/impls/VTreeSearchCursor.java
new file mode 100644
index 0000000000..56c9a274f0
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/impls/VTreeSearchCursor.java
@@ -0,0 +1,1021 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.vector.impls;
+
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Queue;
+import java.util.Set;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleReference;
+import org.apache.hyracks.storage.am.vector.api.IVTreeDataFrame;
+import org.apache.hyracks.storage.am.vector.api.IVTreeDistanceFunction;
+import org.apache.hyracks.storage.am.vector.api.IVTreeInteriorFrame;
+import org.apache.hyracks.storage.am.vector.api.IVTreeLeafFrame;
+import org.apache.hyracks.storage.am.vector.api.IVTreeMetadataFrame;
+import org.apache.hyracks.storage.am.vector.api.IVTreeQuantizer;
+import org.apache.hyracks.storage.am.vector.utils.VTreeNavigationState;
+import org.apache.hyracks.storage.am.vector.utils.VTreeNavigationUtils;
+import org.apache.hyracks.storage.common.EnforcedIndexCursor;
+import org.apache.hyracks.storage.common.ICursorInitialState;
+import org.apache.hyracks.storage.common.IIndexAccessor;
+import org.apache.hyracks.storage.common.ISearchPredicate;
+import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.buffercache.ICachedPage;
+import org.apache.hyracks.storage.common.file.BufferedFileHandle;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+/**
+ * Search cursor for vector clustering tree operations.
+ * Performs centroid finding via tree traversal and then iterates through data pages of the selected cluster.
+ */
+public class VTreeSearchCursor extends EnforcedIndexCursor {
+
+ private static final Logger LOGGER = LogManager.getLogger();
+
+ // Tree navigation fields (used for static structure traversal)
+ private IBufferCache bufferCache;
+ private int fileId;
+ private int rootPageId;
+ private ITreeIndexFrameFactory interiorFrameFactory;
+ private ITreeIndexFrameFactory leafFrameFactory;
+ private ITreeIndexFrameFactory metadataFrameFactory;
+ private ITreeIndexFrameFactory dataFrameFactory;
+
+ // Data access fields (for memory components: VBC; for disk: same as bufferCache)
+ private IBufferCache dataBufferCache;
+ private int dataFileId;
+ // Centroid-to-directory-page mapping (memory components only, null for disk)
+ private int[] centroidDirPageMap;
+ private int firstLeafCentroidIdForMap;
+ // Lazily-built centroid→directoryPageId map for disk components.
+ // When centroidDirPageMap is null and openClusterByResult() is called,
+ // the ClusterSearchResult's directoryPageId may be from a DIFFERENT component
+ // (e.g., static structure with predicted IDs). This map resolves the correct
+ // directory page ID by scanning this component's own leaf pages.
+ private Map<Integer, Long> localCentroidDirPageMap;
+
+ // Cursor state fields
+ /* Metadata page for current cluster */
+ private long targetMetadataPageId;
+ /* Currently opened data page */
+ private long currentDataPageId;
+ private double[] queryVector;
+ private boolean isOpen;
+ private ITupleReference currentTuple;
+ private ICachedPage currentPage;
+ private IVTreeDataFrame dataFrame;
+ private ITreeIndexTupleReference frameTuple;
+ /* Position in current data page (0-based, next tuple to read) */
+ private int currentTupleIndex;
+ /* Total tuples in current data page */
+ private int tupleCount;
+ private IIndexAccessor accessor;
+ private IVTreeDistanceFunction distanceFunction;
+
+ // Multi-cluster support fields
+ /* Total records iterated (before any LSM-layer filtering) */
+ private int recordsIterated;
+ /* Current cluster being scanned */
+ private ClusterSearchResult currentClusterResult;
+ /* Flag when no more clusters available */
+ private boolean exhaustedAllClusters;
+ /* DFS navigation state for query mode */
+ private VTreeNavigationState iteratorState;
+ /* Count of clusters scanned */
+ private int clustersProbed;
+
+ // Count of clusters that resolved to an empty result in query mode — a directory page that was
+ // either unresolvable (-1 sentinel) or resolved but carried no metadata entries. In query mode an
+ // empty cluster is a recall hole (missing candidates), unlike merge/full-scan mode where an empty
+ // cluster is legitimate. Aggregated and logged once at close() so the degradation is observable
+ // rather than lost in per-event noise.
+ private int emptyClustersInQueryMode;
+
+ // Two modes:
+ // - fullScanMode = false: Query mode - distance-based cluster iteration (closest clusters first)
+ // - fullScanMode = true: Merge mode - sequential cluster iteration (0→1→2→...)
+ private boolean fullScanMode;
+ /* For full-scan: which cluster index we're at (0, 1, 2, ...) */
+ private int currentSequentialClusterIndex;
+ /* Total number of leaf clusters */
+ private int totalLeafClusters;
+ /* First directory page ID (cluster 0) */
+ private long firstDirectoryPageId;
+ /* All directory page IDs collected from all leaf pages */
+ private List<Long> allDirectoryPageIds;
+
+ // Shared state from LSM layer (for DFS visited tracking)
+ private Set<Integer> sharedVisitedSet; // Shared visited set from LSM layer
+
+ // Quantization state (propagated from VTreeCursorInitialState)
+ private double[] quantizedQueryVector; // Quantized query vector (null = non-quantized)
+ private IVTreeQuantizer quantizer; // Quantizer instance (null = non-quantized)
+
+ public VTreeSearchCursor() {
+ this.isOpen = false;
+ this.currentTupleIndex = 0;
+ this.tupleCount = 0;
+ this.currentDataPageId = -1;
+ this.targetMetadataPageId = -1;
+ }
+
+ /**
+ * Check if the cursor is open and ready for use.
+ */
+ public boolean isOpen() {
+ return isOpen;
+ }
+
+ public void setBufferCache(IBufferCache bufferCache) {
+ this.bufferCache = bufferCache;
+ // Default: data uses same buffer cache as navigation
+ if (this.dataBufferCache == null) {
+ this.dataBufferCache = bufferCache;
+ }
+ }
+
+ public void setFileId(int fileId) {
+ this.fileId = fileId;
+ // Default: data uses same file ID as navigation
+ if (this.dataFileId == 0) {
+ this.dataFileId = fileId;
+ }
+ }
+
+ /**
+ * Set separate buffer cache for data page access (memory components).
+ * Navigation uses the main bufferCache/fileId (static structure).
+ * Data pages use dataBufferCache/dataFileId (VBC).
+ */
+ public void setDataBufferCache(IBufferCache dataBufferCache, int dataFileId) {
+ this.dataBufferCache = dataBufferCache;
+ this.dataFileId = dataFileId;
+ }
+
+ /**
+ * Set centroid-to-directory-page mapping for memory components.
+ * When set, the cursor resolves directory pages from the map instead of reading leaf pages.
+ */
+ public void setCentroidDirPageMap(int[] map, int firstLeafCentroidId) {
+ this.centroidDirPageMap = map;
+ this.firstLeafCentroidIdForMap = firstLeafCentroidId;
+ }
+
+ public void setRootPageId(int rootPageId) {
+ this.rootPageId = rootPageId;
+ }
+
+ public void setFrameFactories(ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory leafFrameFactory,
+ ITreeIndexFrameFactory metadataFrameFactory, ITreeIndexFrameFactory dataFrameFactory) {
+ this.interiorFrameFactory = interiorFrameFactory;
+ this.leafFrameFactory = leafFrameFactory;
+ this.metadataFrameFactory = metadataFrameFactory;
+ this.dataFrameFactory = dataFrameFactory;
+ }
+
+ public void setQueryVector(double[] queryVector) {
+ this.queryVector = queryVector;
+ }
+
+ public void setFullScanMode(boolean fullScanMode) {
+ this.fullScanMode = fullScanMode;
+ }
+
+ /**
+ * Set the shared visited set from LSM layer.
+ * This allows tracking visited centroids across all LSM components.
+ */
+ public void setSharedVisitedSet(Set<Integer> visitedSet) {
+ this.sharedVisitedSet = visitedSet;
+ if (this.iteratorState != null) {
+ this.iteratorState.setVisitedSet(visitedSet);
+ }
+ }
+
+ /**
+ * Get the number of clusters probed so far.
+ */
+ public int getClustersProbed() {
+ return clustersProbed;
+ }
+
+ /**
+ * Decrement the clusters probed counter.
+ * Used when a cluster contributed 0 valid tuples (structural or reconciliation empty).
+ */
+ public void decrementClustersProbed() {
+ this.clustersProbed = Math.max(0, this.clustersProbed - 1);
+ }
+
+ /**
+ * Reset the clusters probed counter.
+ * Used when re-opening cursor to a different first cluster (e.g., level-wise[0] instead of DFS result).
+ */
+ public void resetClustersProbed() {
+ this.clustersProbed = 0;
+ }
+
+ /**
+ * Get the distance function.
+ */
+ public IVTreeDistanceFunction getDistanceFunction() {
+ return this.distanceFunction;
+ }
+
+ /**
+ * Get the quantized query vector, or null if quantization is not configured.
+ */
+ public double[] getQuantizedQueryVector() {
+ return this.quantizedQueryVector;
+ }
+
+ /**
+ * Get the vector quantizer, or null if quantization is not configured.
+ */
+ public IVTreeQuantizer getQuantizer() {
+ return this.quantizer;
+ }
+
+ @Override
+ protected void doOpen(ICursorInitialState initialState, ISearchPredicate searchPred) throws HyracksDataException {
+ this.isOpen = true;
+ this.recordsIterated = 0;
+ this.exhaustedAllClusters = false;
+ this.clustersProbed = 0;
+ this.emptyClustersInQueryMode = 0;
+
+ // Get query vector and other parameters from initial state
+ // The query vector is passed via IIndexAccessParameters from the operator layer
+ VTreeCursorInitialState vectorState = (VTreeCursorInitialState) initialState;
+ if (vectorState.getQueryVector() != null) {
+ this.queryVector = vectorState.getQueryVector();
+ }
+ // If targetMetadataPageId is already set in the state, use it directly
+ if (vectorState.getMetadataPageId() != -1) {
+ this.targetMetadataPageId = vectorState.getMetadataPageId();
+ }
+ // Use rootPageId from initial state if provided
+ if (vectorState.getRootPageId() != 0) {
+ this.rootPageId = vectorState.getRootPageId();
+ }
+
+ this.accessor = vectorState.getIndexAccessor();
+
+ // Distance function is set by VTree.search() before opening the cursor; a missing
+ // value means the caller bypassed that path and the search cannot proceed safely.
+ this.distanceFunction = vectorState.getDistanceFunction();
+ if (this.distanceFunction == null) {
+ throw new IllegalStateException(
+ "VTreeSearchCursor opened without a distance function on the initial state");
+ }
+
+ // Extract quantized state from initial state (null = non-quantized path)
+ this.quantizedQueryVector = vectorState.getQuantizedQueryVector();
+ this.quantizer = vectorState.getQuantizer();
+
+ if (fullScanMode) {
+ // Full-scan mode: Navigate to cluster 0 and iterate sequentially
+ navigateToFirstCluster();
+ } else {
+ // Query mode: Find closest cluster using DFS
+ // NOTE: Level-wise exploration is handled by LSM layer via openClusterById()
+ if (this.queryVector == null) {
+ throw HyracksDataException
+ .create(new IllegalArgumentException("Query vector must be provided for centroid finding"));
+ }
+
+ // Create navigation state for iterative DFS with shared visited set
+ if (sharedVisitedSet != null) {
+ this.iteratorState = new VTreeNavigationState(bufferCache, fileId, rootPageId, interiorFrameFactory,
+ leafFrameFactory, queryVector, sharedVisitedSet);
+ } else {
+ this.iteratorState = new VTreeNavigationState(bufferCache, fileId, rootPageId, interiorFrameFactory,
+ leafFrameFactory, queryVector);
+ }
+
+ // Initialize DFS iterator and get first (closest) cluster
+ this.currentClusterResult = VTreeNavigationUtils.initializeClusterIterator(iteratorState, distanceFunction);
+
+ if (this.currentClusterResult == null) {
+ // Empty tree
+ this.tupleCount = 0;
+ this.currentTupleIndex = 0;
+ this.exhaustedAllClusters = true;
+ return;
+ }
+
+ // Get metadata page pointer for first cluster
+ this.targetMetadataPageId = getMetadataPageIdFromCluster(currentClusterResult);
+
+ // Start from the first data page of the first cluster
+ this.currentDataPageId = getFirstDataPageFromMetadata();
+
+ if (this.currentDataPageId != -1) {
+ openDataPage(this.currentDataPageId);
+ } else {
+ // No data pages in this cluster
+ this.tupleCount = 0;
+ }
+ this.currentTupleIndex = 0;
+ this.clustersProbed = 1; // First cluster opened
+ }
+ }
+
+ @Override
+ protected boolean doHasNext() throws HyracksDataException {
+ if (!isOpen) {
+ return false;
+ }
+
+ // Check if there are more tuples in current data page
+ if (currentTupleIndex < tupleCount) {
+ return true;
+ }
+
+ // Current page exhausted, try to move to next data page in same cluster
+ if (moveToNextDataPage()) {
+ return true;
+ }
+
+ // Current cluster exhausted - let the LSM layer decide whether to advance to next cluster
+ return false;
+ }
+
+ @Override
+ protected void doNext() throws HyracksDataException {
+ if (!isOpen) {
+ throw HyracksDataException.create(new IllegalStateException("Cursor is not open"));
+ }
+ if (!doHasNext()) {
+ throw HyracksDataException.create(new IllegalStateException("No more tuples"));
+ }
+
+ // Position on next tuple using frameTuple
+ if (this.dataFrame != null && this.frameTuple != null) {
+ this.frameTuple.resetByTupleIndex(this.dataFrame, currentTupleIndex);
+ this.currentTuple = this.frameTuple;
+ }
+ currentTupleIndex++;
+ recordsIterated++;
+ }
+
+ @Override
+ protected ITupleReference doGetTuple() {
+ return currentTuple;
+ }
+
+ /**
+ * Get the query vector used for this search.
+ */
+ public double[] getQueryVector() {
+ return queryVector;
+ }
+
+ /**
+ * Get the current cluster result (the cluster this cursor is currently scanning).
+ * Used by LSM layer to mark the first cluster as visited.
+ */
+ public ClusterSearchResult getCurrentClusterResult() {
+ return currentClusterResult;
+ }
+
+ /**
+ * Navigate to the first cluster and collect ALL directory page IDs for full-scan mode.
+ * Scans all leaf pages in overflow chain to build complete directory list.
+ * Follows the path: root → first child → interior → first child → leftmost leaf
+ * Then: leftmost leaf → next leaf → ... → last leaf (following nextLeaf pointers)
+ */
+ private void navigateToFirstCluster() throws HyracksDataException {
+ // Step 1: Navigate to leftmost leaf page.
+ int currentPageId = descendToLeftmostLeafPage();
+
+ // Step 2: Scan ALL leaf pages and collect ALL directory page IDs
+ // (Full-scan mode is only used for merge operations on disk components)
+ int totalClusters = collectAllDirectoryPageIds(currentPageId);
+
+ // Step 3: Store collected information
+ this.totalLeafClusters = totalClusters;
+
+ if (this.totalLeafClusters == 0) {
+ throw HyracksDataException.create(new IllegalStateException("No leaf centroids found - empty index"));
+ }
+
+ this.firstDirectoryPageId = allDirectoryPageIds.get(0);
+
+ // Step 4: Open cluster 0
+ this.currentSequentialClusterIndex = 0;
+ openClusterByDirectoryPage(this.firstDirectoryPageId);
+ this.clustersProbed = 1;
+
+ // Create ClusterSearchResult for first cluster (for LSM layer to access)
+ this.currentClusterResult = new ClusterSearchResult(-1, // No leaf page ID in full-scan mode
+ 0, // Cluster index
+ null, // No centroid vector
+ 0.0, // No distance in full-scan mode
+ 0, // Cluster index as centroid ID
+ this.firstDirectoryPageId, // Directory page ID for O(1) access
+ Double.NaN // No quantized distance in full-scan mode
+ );
+
+ }
+
+ /**
+ * Descend from the root to the leftmost leaf page and return its page id. Single pin/latch per hop:
+ * read the level to decide leaf-vs-interior and (if interior) descend to child 0 within the same
+ * latch, instead of an extra isLeafPage() pin per hop.
+ */
+ private int descendToLeftmostLeafPage() throws HyracksDataException {
+ int currentPageId = rootPageId;
+ while (true) {
+ ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, currentPageId));
+ try {
+ page.acquireReadLatch();
+ IVTreeInteriorFrame interiorFrame = createInteriorFrame();
+ interiorFrame.setPage(page);
+
+ if (interiorFrame.getLevel() == 0) {
+ return currentPageId; // Reached the leftmost leaf page
+ }
+
+ if (interiorFrame.getTupleCount() > 0) {
+ currentPageId = interiorFrame.getChildPageId(0);
+ } else {
+ throw HyracksDataException
+ .create(new IllegalStateException("Empty interior page encountered during navigation"));
+ }
+ } finally {
+ page.releaseReadLatch();
+ bufferCache.unpin(page);
+ }
+ }
+ }
+
+ /**
+ * Walk the leaf-page chain starting at {@code firstLeafPageId} (following each page's next-leaf
+ * pointer) and append every cluster's directory page pointer to {@link #allDirectoryPageIds}.
+ * Resets {@code allDirectoryPageIds} to a fresh list first. Returns the total number of clusters
+ * collected. Used by full-scan (merge) navigation.
+ */
+ private int collectAllDirectoryPageIds(int firstLeafPageId) throws HyracksDataException {
+ this.allDirectoryPageIds = new ArrayList<>();
+ int leafPageId = firstLeafPageId;
+ int totalClusters = 0;
+
+ while (leafPageId != -1) {
+ ICachedPage leafPage = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, leafPageId));
+ try {
+ leafPage.acquireReadLatch();
+ IVTreeLeafFrame leafFrame = createLeafFrame();
+ leafFrame.setPage(leafPage);
+
+ int tupleCount = leafFrame.getTupleCount();
+ int nextLeafPageId = leafFrame.getNextLeaf();
+
+ // Collect directory page IDs from THIS leaf page
+ for (int i = 0; i < tupleCount; i++) {
+ long dirPageId = leafFrame.getMetadataPagePointer(i);
+ allDirectoryPageIds.add(dirPageId);
+ totalClusters++;
+ }
+
+ // Move to next leaf page (follow nextLeaf pointer)
+ leafPageId = nextLeafPageId;
+
+ } finally {
+ leafPage.releaseReadLatch();
+ bufferCache.unpin(leafPage);
+ }
+ }
+ return totalClusters;
+ }
+
+ /**
+
+ * Open a cluster by its directory (metadata) page ID.
+ * Used by full-scan mode for sequential cluster iteration.
+ */
+ private void openClusterByDirectoryPage(long directoryPageId) throws HyracksDataException {
+ // Log previous cluster summary if we have data
+ this.targetMetadataPageId = directoryPageId;
+
+ // Guard: directoryPageId=-1 means empty cluster (no data assigned during bulk loading)
+ if (directoryPageId == -1) {
+ if (!fullScanMode) {
+ // In query mode a -1 sentinel usually means the directory page could not be
+ // resolved in this component (e.g., a corrupt/wrong component root made the
+ // cluster unreachable) — flag it instead of silently yielding an empty cluster.
+ // Merge (full-scan) mode legitimately sees -1 for empty clusters, so stay quiet there.
+ emptyClustersInQueryMode++;
+ LOGGER.warn("VTreeSearchCursor: -1 directory page sentinel in query mode (fileId={}, rootPageId={}); "
+ + "treating cluster as empty", fileId, rootPageId);
+ }
+ this.currentDataPageId = -1;
+ this.tupleCount = 0;
+ this.currentTupleIndex = 0;
+ return;
+ }
+
+ // Pin metadata/directory page (use data buffer cache for directory pages)
+ ICachedPage dirPage = dataBufferCache.pin(BufferedFileHandle.getDiskPageId(dataFileId, (int) directoryPageId));
+ try {
+ dirPage.acquireReadLatch();
+ IVTreeMetadataFrame metadataFrame = createMetadataFrame();
+ metadataFrame.setPage(dirPage);
+
+ int metadataTupleCount = metadataFrame.getTupleCount();
+ if (metadataTupleCount == 0) {
+ if (!fullScanMode) {
+ // The directory page resolved but has no data-page entries. In query mode that is a
+ // recall hole (an expected-non-empty cluster yields nothing) and a sign of an
+ // inconsistent component; surface it consistently with the -1 case above instead of
+ // returning empty silently.
+ emptyClustersInQueryMode++;
+ LOGGER.warn(
+ "VTreeSearchCursor: directory page {} has no metadata entries in query mode "
+ + "(fileId={}, rootPageId={}); treating cluster as empty",
+ directoryPageId, fileId, rootPageId);
+ }
+ this.currentDataPageId = -1;
+ this.tupleCount = 0;
+ this.currentTupleIndex = 0;
+ return;
+ }
+
+ // Get first data page from metadata
+ long firstDataPageId = metadataFrame.getDataPagePointer(0);
+ this.currentDataPageId = firstDataPageId;
+
+ } finally {
+ dirPage.releaseReadLatch();
+ dataBufferCache.unpin(dirPage);
+ }
+
+ // Open first data page
+ if (this.currentDataPageId != -1) {
+ openDataPage(this.currentDataPageId);
+ } else {
+ this.tupleCount = 0;
+ }
+
+ this.currentTupleIndex = 0;
+ }
+
+ /**
+ * Advance to the next closest cluster.
+ * This method is called by the LSM layer when it needs more data.
+ * <p>
+ * Supports two modes:
+ * - Full-scan mode: Sequential iteration (cluster 0 → 1 → 2 → ...)
+ * - Query mode: Distance-based iteration (closest clusters first)
+ *
+ * @return true if successfully moved to next cluster, false if no more clusters available
+ */
+ public boolean advanceToNextCluster() throws HyracksDataException {
+ if (fullScanMode) {
+ // Full-scan mode: Sequential iteration through clusters
+ currentSequentialClusterIndex++;
+
+ if (currentSequentialClusterIndex >= totalLeafClusters) {
+ exhaustedAllClusters = true;
+ return false;
+ }
+
+ long nextDirectoryPageId = allDirectoryPageIds.get(currentSequentialClusterIndex);
+ openClusterByDirectoryPage(nextDirectoryPageId);
+
+ // Create ClusterSearchResult for this sequential cluster
+ // In full-scan mode, we don't have centroid info, but we have the directory page
+ this.currentClusterResult = new ClusterSearchResult(-1, // No leaf page ID in full-scan mode
+ currentSequentialClusterIndex, // Cluster index
+ null, // No centroid vector
+ 0.0, // No distance in full-scan mode
+ currentSequentialClusterIndex, // Use cluster index as centroid ID
+ nextDirectoryPageId, // Directory page ID for O(1) access
+ Double.NaN // No quantized distance in full-scan mode
+ );
+ this.clustersProbed++;
+
+ return true;
+
+ } else {
+ // Query mode: Distance-based iteration using DFS
+ // Open next cluster and return immediately (even if empty)
+ // Let LSMVTreeSearchCursor handle cluster synchronization
+ ClusterSearchResult nextCluster =
+ VTreeNavigationUtils.findNextClosestCluster(iteratorState, distanceFunction);
+
+ if (nextCluster == null) {
+ exhaustedAllClusters = true;
+ return false;
+ }
+
+ openCluster(nextCluster);
+
+ // Return true even if cluster is empty - let LSMVTreeSearchCursor handle it
+ // This ensures cluster synchronization across all LSM components
+ return true;
+ }
+ }
+
+ /**
+ * Check if this cursor has more clusters to scan.
+ *
+ * @return true if more clusters are available, false if all clusters exhausted
+ */
+ public boolean hasMoreClusters() {
+ return !exhaustedAllClusters;
+ }
+
+ /**
+ * Open a specific cluster using a ClusterSearchResult that already has directoryPageId.
+ * This is O(1) - no tree traversal needed since directoryPageId is already known.
+ * Used by LSM layer for efficient cluster advancement.
+ *
+ * @param cluster the ClusterSearchResult containing directoryPageId
+ * @return true if cluster was opened successfully and has data, false otherwise
+ */
+ public boolean openClusterByResult(ClusterSearchResult cluster) throws HyracksDataException {
+ if (cluster == null) {
+ return false;
+ }
+
+ // Always resolve directoryPageId locally for this component.
+ // The cluster's directoryPageId may come from a different LSM component
+ // (e.g., memory component VBC page IDs vs disk component page IDs).
+ // getMetadataPageIdFromCluster handles both memory (centroidDirPageMap)
+ // and disk (leaf page traversal) correctly.
+ long localDirPageId = getMetadataPageIdFromCluster(cluster);
+ openClusterByDirectoryPage(localDirPageId);
+ this.currentClusterResult = cluster;
+ this.clustersProbed++;
+
+ // Check if cluster has data
+ boolean hasData = currentTupleIndex < tupleCount;
+ return hasData;
+ }
+
+ /**
+ * Find next cluster using DFS (for LSM layer to get cluster ID).
+ * Does NOT open the cluster, just returns the result.
+ * Skips clusters already in the shared visited set.
+ */
+ public ClusterSearchResult findNextClusterDFS() throws HyracksDataException {
+ if (iteratorState == null) {
+ return null;
+ }
+
+ // Initialize if needed
+ if (!iteratorState.initialized) {
+ ClusterSearchResult first = VTreeNavigationUtils.initializeClusterIterator(iteratorState, distanceFunction);
+ if (first != null) {
+ return first;
+ }
+ }
+
+ // Get next from DFS (automatically skips visited via NavigationState)
+ return VTreeNavigationUtils.findNextClosestCluster(iteratorState, distanceFunction);
+ }
+
+ /**
+ * Get the first data page ID from the target metadata page.
+ */
+ private long getFirstDataPageFromMetadata() throws HyracksDataException {
+ if (targetMetadataPageId == -1) {
+ return -1;
+ }
+
+ // Use data buffer cache for directory/metadata pages
+ ICachedPage metadataPage =
+ dataBufferCache.pin(BufferedFileHandle.getDiskPageId(dataFileId, (int) targetMetadataPageId));
+ try {
+ metadataPage.acquireReadLatch();
+ IVTreeMetadataFrame metadataFrame = createMetadataFrame();
+ metadataFrame.setPage(metadataPage);
+
+ int tupleCount = metadataFrame.getTupleCount();
+ if (tupleCount > 0) {
+ // Get the first data page from the metadata page
+ return metadataFrame.getDataPagePointer(0);
+ }
+ return -1;
+ } finally {
+ metadataPage.releaseReadLatch();
+ dataBufferCache.unpin(metadataPage);
+ }
+ }
+
+ /**
+ * Open a specific data page and initialize the cursor on it.
+ */
+ private void openDataPage(long dataPageId) throws HyracksDataException {
+ // Close current page if open
+ closeCurrentPage();
+
+ // Pin and acquire the new data page (use data buffer cache for directory/data pages).
+ // Keep the page in a local until fully initialized so that if the latch-acquire or any
+ // frame setup throws, we release (only if latched) and unpin here rather than leaving a
+ // half-initialized currentPage for close()/closeCurrentPage() to mis-release.
+ ICachedPage page = dataBufferCache.pin(BufferedFileHandle.getDiskPageId(dataFileId, (int) dataPageId));
+ boolean latched = false;
+ try {
+ page.acquireReadLatch();
+ latched = true;
+
+ // Initialize data frame
+ this.dataFrame = createDataFrame();
+ this.dataFrame.setPage(page);
+ this.frameTuple = this.dataFrame.createTupleReference();
+ this.tupleCount = this.dataFrame.getTupleCount();
+ this.currentTupleIndex = 0;
+ this.currentPage = page;
+ } catch (Exception e) {
+ if (latched) {
+ page.releaseReadLatch();
+ }
+ dataBufferCache.unpin(page);
+ throw HyracksDataException.create(e);
+ }
+ }
+
+ /**
+ * Move to the next data page using the linked list structure.
+ * This is more efficient than going back to the metadata page each time.
+ *
+ * @return true if successfully moved to next page, false if no more pages
+ */
+ private boolean moveToNextDataPage() throws HyracksDataException {
+ if (dataFrame == null) {
+ return false;
+ }
+
+ // After deletion, data pages can become empty but there might
+ // be more non-empty pages later in the chain. We must skip empty pages instead
+ // of stopping at the first empty page.
+
+ while (true) {
+ // Get the next page ID from the current data frame's linked list pointer
+ int nextDataPageId = dataFrame.getNextPage();
+ if (nextDataPageId == -1) {
+ return false; // Reached end of chain
+ }
+
+ // Move to the next data page
+ this.currentDataPageId = nextDataPageId;
+ openDataPage(nextDataPageId);
+
+ // Check if this page has tuples
+ if (this.tupleCount > 0) {
+ return true; // Found non-empty page
+ }
+
+ // Page is empty after deletion - continue to next page
+ }
+ }
+
+ /**
+ * Close current data page if open.
+ */
+ private void closeCurrentPage() throws HyracksDataException {
+ if (currentPage != null) {
+ currentPage.releaseReadLatch();
+ dataBufferCache.unpin(currentPage);
+ currentPage = null;
+ }
+ }
+
+ /**
+ * Open a specific cluster for scanning.
+ * Gets metadata page, then opens first data page.
+ */
+ private void openCluster(ClusterSearchResult cluster) throws HyracksDataException {
+ if (cluster == null) {
+ this.currentDataPageId = -1;
+ this.tupleCount = 0;
+ return;
+ }
+
+ this.currentClusterResult = cluster;
+ this.clustersProbed++; // Increment cluster counter
+
+ // Get metadata page pointer from leaf frame
+ this.targetMetadataPageId = getMetadataPageIdFromCluster(cluster);
+
+ // Get first data page from metadata
+ this.currentDataPageId = getFirstDataPageFromMetadata();
+
+ if (this.currentDataPageId != -1) {
+ openDataPage(this.currentDataPageId);
+ } else {
+ // Empty cluster
+ this.tupleCount = 0;
+ }
+
+ this.currentTupleIndex = 0;
+ }
+
+ /**
+ * Create a metadata frame instance using the frame factory.
+ */
+ private IVTreeMetadataFrame createMetadataFrame() {
+ if (metadataFrameFactory == null) {
+ throw new IllegalStateException("Metadata frame factory not set");
+ }
+ return (IVTreeMetadataFrame) metadataFrameFactory.createFrame();
+ }
+
+ /**
+ * Create a data frame instance using the frame factory.
+ */
+ private IVTreeDataFrame createDataFrame() {
+ if (dataFrameFactory == null) {
+ throw new IllegalStateException("Data frame factory not set");
+ }
+ return (IVTreeDataFrame) dataFrameFactory.createFrame();
+ }
+
+ /**
+ * Get metadata page ID from cluster search result.
+ *
+ * For memory components: uses centroidDirPageMap for O(1) lookup (the map
+ * translates centroid IDs to VBC directory page IDs).
+ *
+ * For disk components: builds a lazy local map by scanning this component's
+ * own leaf pages. This is necessary because the ClusterSearchResult's
+ * directoryPageId may come from a DIFFERENT component (e.g., level-wise
+ * computation reads the static structure's leaf pages, which have predicted
+ * directory page IDs that don't match this disk component's actual IDs).
+ */
+ private long getMetadataPageIdFromCluster(ClusterSearchResult clusterResult) throws HyracksDataException {
+ // Memory components: use centroidDirPageMap for O(1) lookup
+ if (centroidDirPageMap != null) {
+ int centroidIndex = clusterResult.centroidId - firstLeafCentroidIdForMap;
+ if (centroidIndex >= 0 && centroidIndex < centroidDirPageMap.length) {
+ return centroidDirPageMap[centroidIndex];
+ }
+ }
+
+ // Disk components: resolve directoryPageId locally by scanning this
+ // component's own leaf pages (which have correct IDs from VTreeBulkLoader).
+ if (localCentroidDirPageMap == null) {
+ buildLocalCentroidDirPageMap();
+ }
+ if (localCentroidDirPageMap != null) {
+ Long dirPageId = localCentroidDirPageMap.get(clusterResult.centroidId);
+ if (dirPageId != null) {
+ return dirPageId;
+ }
+ }
+
+ // Fallback: use directoryPageId from ClusterSearchResult
+ if (clusterResult.directoryPageId == -1) {
+ // The cluster is not in this component's leaf pages AND the caller-provided result
+ // carries the -1 sentinel: the cluster will be silently treated as empty. This is
+ // expected for genuinely empty clusters, but it is also the signature of an
+ // unreachable static structure (e.g., a wrong persisted component root).
+ LOGGER.warn(
+ "VTreeSearchCursor: unresolved directory page for centroidId={} (clusterIndex={}) in "
+ + "component fileId={} (rootPageId={}); cluster will be treated as empty",
+ clusterResult.centroidId, clusterResult.clusterIndex, fileId, rootPageId);
+ }
+ return clusterResult.directoryPageId;
+ }
+
+ /**
+ * Build a centroidId → directoryPageId map by scanning this component's own leaf pages.
+ * Uses BFS from root to find all leaf pages, then reads each leaf tuple's
+ * centroidId and metadataPagePointer to build the map.
+ *
+ * This is needed because ClusterSearchResults from level-wise computation
+ * carry directoryPageIds from the static structure (predicted sequential IDs),
+ * which don't match this disk component's actual directory page IDs
+ * (set correctly by VTreeBulkLoader.end()).
+ */
+ private void buildLocalCentroidDirPageMap() throws HyracksDataException {
+ localCentroidDirPageMap = new HashMap<>();
+
+ // BFS from root through the tree to find all leaf pages
+ Queue<Integer> queue = new ArrayDeque<>();
+ queue.add(rootPageId);
+ Set<Integer> visitedPages = new HashSet<>();
+
+ while (!queue.isEmpty()) {
+ int pageId = queue.poll();
+ if (!visitedPages.add(pageId)) {
+ continue;
+ }
+
+ ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId));
+ try {
+ page.acquireReadLatch();
+
+ // Use leaf frame to check page level (isLeaf works on any page type)
+ IVTreeLeafFrame lf = createLeafFrame();
+ lf.setPage(page);
+
+ if (lf.isLeaf()) {
+ // Leaf page: collect centroidId → metadataPagePointer mappings
+ int tc = lf.getTupleCount();
+ for (int i = 0; i < tc; i++) {
+ int centroidId = lf.getCentroidId(i);
+ long dirPageId = lf.getMetadataPagePointer(i);
+ localCentroidDirPageMap.put(centroidId, dirPageId);
+ }
+ // Follow overflow chain
+ if (lf.getOverflowFlagBit()) {
+ int nextLeafPage = lf.getNextLeaf();
+ if (nextLeafPage >= 0) {
+ queue.add(nextLeafPage);
+ }
+ }
+ } else {
+ // Interior page: enqueue all children
+ IVTreeInteriorFrame intFrame = createInteriorFrame();
+ intFrame.setPage(page);
+
+ int tc = intFrame.getTupleCount();
+ for (int i = 0; i < tc; i++) {
+ queue.add(intFrame.getChildPageId(i));
+ }
+ // Follow overflow chain for interior pages
+ if (intFrame.getOverflowFlagBit()) {
+ int nextPage = intFrame.getNextPage();
+ if (nextPage >= 0) {
+ queue.add(nextPage);
+ }
+ }
+ }
+ } finally {
+ page.releaseReadLatch();
+ bufferCache.unpin(page);
+ }
+ }
+ }
+
+ /**
+ * Create frame instances.
+ */
+ private IVTreeInteriorFrame createInteriorFrame() {
+ if (interiorFrameFactory == null) {
+ throw new IllegalStateException("Interior frame factory not set");
+ }
+ return (IVTreeInteriorFrame) interiorFrameFactory.createFrame();
+ }
+
+ private IVTreeLeafFrame createLeafFrame() {
+ if (leafFrameFactory == null) {
+ throw new IllegalStateException("Leaf frame factory not set");
+ }
+ return (IVTreeLeafFrame) leafFrameFactory.createFrame();
+ }
+
+ @Override
+ protected void doClose() throws HyracksDataException {
+ if (isOpen && emptyClustersInQueryMode > 0) {
+ // Aggregate recall-hole summary for this search: how many probed clusters yielded nothing in
+ // query mode. Non-zero here means candidates were missed (usually an inconsistent component).
+ LOGGER.warn(
+ "VTreeSearchCursor: {} of {} probed clusters were empty in query mode "
+ + "(fileId={}, rootPageId={})",
+ emptyClustersInQueryMode, clustersProbed, fileId, rootPageId);
+ }
+ if (isOpen) {
+ closeCurrentPage();
+ }
+ this.isOpen = false;
+ this.currentTuple = null;
+ this.currentTupleIndex = 0;
+ this.tupleCount = 0;
+ this.currentDataPageId = -1;
+ }
+
+ @Override
+ protected void doDestroy() throws HyracksDataException {
+ // Resource release lives in doClose(); EnforcedIndexCursor guarantees close() runs before
+ // destroy(), so there is nothing left to tear down here.
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/impls/VTreeSearchPredicate.java b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/impls/VTreeSearchPredicate.java
new file mode 100644
index 0000000000..e792beb602
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/impls/VTreeSearchPredicate.java
@@ -0,0 +1,200 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.vector.impls;
+
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.storage.am.common.api.ITupleFilter;
+import org.apache.hyracks.storage.common.ISearchPredicate;
+import org.apache.hyracks.storage.common.MultiComparator;
+
+/**
+ * Search predicate for vector ANN (Approximate Nearest Neighbor) queries.
+ * Holds a reference to the query tuple for per-tuple vector searches.
+ *
+ * Following the Reference Pattern (like RTree): The predicate holds an ITupleReference
+ * that is updated per-tuple by resetSearchPredicate(). The storage layer extracts
+ * the query vector when needed during search.
+ *
+ * For top-K ANN queries, also includes the k parameter (number of nearest neighbors).
+ */
+public class VTreeSearchPredicate implements ISearchPredicate {
+ private static final long serialVersionUID = 1L;
+
+ private ITupleReference queryTuple;
+ private int queryFieldIndex;
+ private int k; // Number of nearest neighbors to return (for ANN queries)
+ private ITupleFilter tupleFilter; // Filter for INCLUDE field predicates (e.g., year > 2000)
+ private double minProbeFraction; // Fraction of leaf clusters to probe (0.0-1.0, default 0.1)
+ private double epsilon; // Distance threshold for level-wise cross-pollination (internal, not user-facing)
+ private int kMultiplier; // Multiplier for candidate limit: K * kMultiplier sent to PK for reranking (default 1)
+
+ private static final double DEFAULT_MIN_PROBE_FRACTION = 0.1;
+ private static final double DEFAULT_EPSILON = 0.3;
+ private static final int DEFAULT_K_MULTIPLIER = 1;
+
+ public VTreeSearchPredicate() {
+ this.k = Integer.MAX_VALUE;
+ this.minProbeFraction = DEFAULT_MIN_PROBE_FRACTION;
+ this.epsilon = DEFAULT_EPSILON;
+ this.kMultiplier = DEFAULT_K_MULTIPLIER;
+ }
+
+ public VTreeSearchPredicate(int k) {
+ this.k = k;
+ this.minProbeFraction = DEFAULT_MIN_PROBE_FRACTION;
+ this.epsilon = DEFAULT_EPSILON;
+ this.kMultiplier = DEFAULT_K_MULTIPLIER;
+ }
+
+ /**
+ * Set the query tuple containing the vector field.
+ * Called by resetSearchPredicate() for each input tuple.
+ */
+ public void setQueryTuple(ITupleReference queryTuple) {
+ this.queryTuple = queryTuple;
+ }
+
+ /**
+ * Set the field index of the vector in the query tuple.
+ */
+ public void setQueryFieldIndex(int queryFieldIndex) {
+ this.queryFieldIndex = queryFieldIndex;
+ }
+
+ /**
+ * Get the query tuple reference.
+ */
+ public ITupleReference getQueryTuple() {
+ return queryTuple;
+ }
+
+ /**
+ * Get the query field index.
+ */
+ public int getQueryFieldIndex() {
+ return queryFieldIndex;
+ }
+
+ /**
+ * Set the K parameter (number of nearest neighbors to return).
+ */
+ public void setK(int k) {
+ this.k = k;
+ }
+
+ /**
+ * Get the K parameter (number of nearest neighbors to return).
+ */
+ public int getK() {
+ return k;
+ }
+
+ /**
+ * Set the min_probe_fraction parameter (fraction of leaf clusters to probe).
+ * Converted to nprobe = max(1, floor(totalLeafClusters * fraction)) at runtime.
+ * Value of 0 means use default (0.1).
+ */
+ public void setMinProbeFraction(double minProbeFraction) {
+ if (minProbeFraction <= 0.0) {
+ this.minProbeFraction = DEFAULT_MIN_PROBE_FRACTION;
+ } else if (minProbeFraction > 1.0) {
+ this.minProbeFraction = 1.0;
+ } else {
+ this.minProbeFraction = minProbeFraction;
+ }
+ }
+
+ /**
+ * Get the min_probe_fraction parameter.
+ */
+ public double getMinProbeFraction() {
+ return minProbeFraction;
+ }
+
+ /**
+ * Set the epsilon parameter (relative distance threshold for level-wise cross-pollination).
+ * Clusters within {@code closestDistance + |closestDistance| * epsilon} — i.e. (1+epsilon)*d
+ * for positive distances and (1-epsilon)*d for negative ones — will be explored via level-wise.
+ */
+ public void setEpsilon(double epsilon) {
+ this.epsilon = epsilon;
+ }
+
+ /**
+ * Get the epsilon parameter (distance threshold for level-wise cross-pollination).
+ */
+ public double getEpsilon() {
+ return epsilon;
+ }
+
+ /**
+ * Set the tuple filter for INCLUDE field predicates.
+ * When set, the cursor will only return tuples that pass this filter,
+ * and only count passing tuples toward K.
+ */
+ public void setTupleFilter(ITupleFilter tupleFilter) {
+ this.tupleFilter = tupleFilter;
+ }
+
+ /**
+ * Get the tuple filter for INCLUDE field predicates.
+ */
+ public ITupleFilter getTupleFilter() {
+ return tupleFilter;
+ }
+
+ /**
+ * Set the kMultiplier for candidate limit (K * kMultiplier sent to PK for reranking).
+ */
+ public void setKMultiplier(int kMultiplier) {
+ this.kMultiplier = kMultiplier;
+ }
+
+ /**
+ * Get the kMultiplier for candidate limit.
+ */
+ public int getKMultiplier() {
+ return kMultiplier;
+ }
+
+ @Override
+ public MultiComparator getLowKeyComparator() {
+ // Vector clustering tree doesn't use traditional key comparisons
+ return null;
+ }
+
+ @Override
+ public MultiComparator getHighKeyComparator() {
+ // Vector clustering tree doesn't use traditional key comparisons
+ return null;
+ }
+
+ @Override
+ public ITupleReference getLowKey() {
+ // Vector clustering tree doesn't use traditional key searches
+ return null;
+ }
+
+ @Override
+ public String toString() {
+ return "VTreeSearchPredicate[queryTuple=" + (queryTuple != null ? "set" : "null") + ", k=" + k
+ + ", minProbeFraction=" + minProbeFraction + ", epsilon=" + epsilon + ", kMultiplier=" + kMultiplier
+ + "]";
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/impls/VTreeStaticStructureBuilder.java b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/impls/VTreeStaticStructureBuilder.java
new file mode 100644
index 0000000000..f52f92adee
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/impls/VTreeStaticStructureBuilder.java
@@ -0,0 +1,569 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.vector.impls;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeSet;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.primitive.IntegerPointable;
+import org.apache.hyracks.data.std.primitive.LongPointable;
+import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleReference;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.storage.am.common.api.IPageManager;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexMetadataFrame;
+import org.apache.hyracks.storage.am.vector.api.IVTreeFrame;
+import org.apache.hyracks.storage.am.vector.api.IVTreeInteriorFrame;
+import org.apache.hyracks.storage.am.vector.api.IVTreeLeafFrame;
+import org.apache.hyracks.storage.am.vector.api.VTreeStaticTupleConstants;
+import org.apache.hyracks.storage.am.vector.utils.VTreeLeafNeighborList;
+import org.apache.hyracks.storage.am.vector.utils.VTreeMetadataKeys;
+import org.apache.hyracks.storage.common.IIndexBulkLoader;
+import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.buffercache.ICachedPage;
+import org.apache.hyracks.storage.common.buffercache.IFIFOPageWriter;
+import org.apache.hyracks.storage.common.buffercache.IPageWriteCallback;
+import org.apache.hyracks.storage.common.buffercache.PageWriteFailureCallback;
+import org.apache.hyracks.storage.common.buffercache.context.write.DefaultBufferCacheWriteContext;
+import org.apache.hyracks.storage.common.buffercache.context.write.LocalOnlyWriteContext;
+import org.apache.hyracks.storage.common.compression.file.ICompressedPageWriter;
+import org.apache.hyracks.storage.common.file.BufferedFileHandle;
+import org.apache.hyracks.util.annotations.AiProvenance;
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+/**
+ * Builds the static structure of a VTree (interior + leaf pages) in bottom-up,
+ * append-only fashion. Tuples must arrive leaf-first (level numLevels-1), one
+ * level at a time, ending with the root level (level 0). Within each level,
+ * clusters arrive in ascending order; within each cluster, centroids arrive in
+ * insertion order.
+ *
+ * Page IDs are assigned via {@link IPageManager#takePage(ITreeIndexMetadataFrame)}
+ * as pages are filled, so the root (allocated last) ends up at the highest page
+ * ID. Each page is written to disk as soon as its forward pointers (overflow
+ * chain within a cluster, sibling chain across leaf clusters, child pointers
+ * for interior tuples) are known, so the builder keeps at most one page
+ * confiscated at a time.
+ *
+ * Centroid IDs follow the BFS-from-root convention (root = 0..N_root-1, leaves
+ * at highest IDs), independent of arrival order. The builder simply records
+ * whatever centroid id is in each tuple and writes it back unchanged.
+ *
+ * Leaf tuples carry a sentinel metadata-page pointer of -1; {@link VTreeBulkLoader}
+ * overwrites the pointer for each leaf tuple based on its centroid id when the
+ * data component is loaded.
+ */
+public class VTreeStaticStructureBuilder extends PageWriteFailureCallback implements IIndexBulkLoader {
+
+ private static final Logger LOGGER = LogManager.getLogger();
+
+ /** Sentinel metadata-page pointer written into leaf tuples; overwritten by VTreeBulkLoader. */
+ private static final int LEAF_METADATA_PTR_SENTINEL = -1;
+
+ // Storage infrastructure
+ private final IBufferCache bufferCache;
+ private final IPageManager freePageManager;
+ private final ITreeIndexMetadataFrame metaFrame;
+ private final int fileId;
+ private final int slotSize;
+ private final IFIFOPageWriter pageWriter;
+ // Leaf pages are published local-only during placement (their neighbor lists are still provisional)
+ // and uploaded later, once, by the leaf resolution pass — see resolveAndUploadLeafNeighbors().
+ private final IFIFOPageWriter leafPlacementWriter;
+ private final ICompressedPageWriter compressedPageWriter;
+
+ // Maps each leaf centroid id to its placement in this static structure: {pageId, slot}. Built during
+ // leaf placement; used by the resolution pass to turn provisional neighbor entries into pointers.
+ private final Map<Integer, int[]> leafDirectory = new HashMap<>();
+ // Leaf page ids in ascending order (the order leaves must be re-uploaded in, for the cloud writer).
+ private final TreeSet<Integer> leafPageIds = new TreeSet<>();
+
+ // Structure shape
+ private final int numLevels;
+ private final List<Integer> clustersPerLevel;
+ private final List<List<Integer>> centroidsPerCluster;
+ private final int maxEntriesPerPage;
+
+ // Frames
+ private final IVTreeInteriorFrame interiorFrame;
+ private final IVTreeLeafFrame leafFrame;
+ // Reused per-entry scratch: createEntryTuple builds one entry at a time (single-threaded build) and
+ // insertSorted copies it into the page before the next entry, so one builder/reference is reused rather
+ // than allocated per centroid. The builder is rebuilt only when the entry field count changes (once, at
+ // the leaf->interior level boundary), since its offset array must be sized exactly to the fields written.
+ private ArrayTupleBuilder entryTupleBuilder;
+ private int entryTupleFieldCount = -1;
+ private final ArrayTupleReference entryTupleRef = new ArrayTupleReference();
+
+ // Build state — bottom-up traversal: leaf first, root last
+ private int currentLevel;
+ private int currentClusterInLevel;
+ private int currentCentroidInCluster;
+ private ICachedPage currentPage;
+ private ITreeIndexFrame currentFrame;
+ private int currentPageId;
+
+ // firstPageIdOfCluster[L][C] = page id of the first page of cluster C at level L.
+ // Filled in by startNewClusterPage() as we visit each cluster. Used by interior
+ // tuples (at level L) to look up child page ids in level L+1, which by virtue of
+ // bottom-up ordering has already been fully written.
+ private final int[][] firstPageIdOfCluster;
+
+ // For first_leaf_centroid_id metadata
+ private final int[] totalCentroidsUpToLevel;
+
+ public VTreeStaticStructureBuilder(IPageWriteCallback callback, VTree vectorTree, int numLevels,
+ List<Integer> clustersPerLevel, List<List<Integer>> centroidsPerCluster, int maxEntriesPerPage)
+ throws HyracksDataException {
+
+ this.bufferCache = vectorTree.getBufferCache();
+ this.freePageManager = vectorTree.getPageManager();
+ this.fileId = vectorTree.getFileId();
+ this.metaFrame = freePageManager.createMetadataFrame();
+
+ this.interiorFrame = (IVTreeInteriorFrame) vectorTree.getInteriorFrameFactory().createFrame();
+ this.leafFrame = (IVTreeLeafFrame) vectorTree.getLeafFrameFactory().createFrame();
+ this.slotSize = ((ITreeIndexFrame) leafFrame).getSlotSize();
+
+ this.pageWriter = bufferCache.createFIFOWriter(callback, this, DefaultBufferCacheWriteContext.INSTANCE);
+ this.leafPlacementWriter = bufferCache.createFIFOWriter(callback, this, LocalOnlyWriteContext.INSTANCE);
+ this.compressedPageWriter = bufferCache.getCompressedPageWriter(fileId);
+
+ this.numLevels = numLevels;
+ this.clustersPerLevel = new ArrayList<>(clustersPerLevel);
+ this.centroidsPerCluster = new ArrayList<>();
+ for (List<Integer> levelCentroids : centroidsPerCluster) {
+ this.centroidsPerCluster.add(new ArrayList<>(levelCentroids));
+ }
+ this.maxEntriesPerPage = maxEntriesPerPage;
+
+ // Bottom-up: start at leaf level
+ this.currentLevel = numLevels - 1;
+ this.currentClusterInLevel = 0;
+ this.currentCentroidInCluster = 0;
+
+ // Allocate the firstPageIdOfCluster grid (all -1 until populated)
+ this.firstPageIdOfCluster = new int[numLevels][];
+ for (int L = 0; L < numLevels; L++) {
+ this.firstPageIdOfCluster[L] = new int[this.clustersPerLevel.get(L)];
+ Arrays.fill(this.firstPageIdOfCluster[L], -1);
+ }
+
+ // Compute cumulative centroid counts (used for first_leaf_centroid_id metadata)
+ this.totalCentroidsUpToLevel = new int[numLevels + 1];
+ totalCentroidsUpToLevel[0] = 0;
+ for (int L = 0; L < numLevels; L++) {
+ int centroidsInLevel = 0;
+ for (int c : this.centroidsPerCluster.get(L)) {
+ centroidsInLevel += c;
+ }
+ totalCentroidsUpToLevel[L + 1] = totalCentroidsUpToLevel[L] + centroidsInLevel;
+ }
+
+ // Open the first page (cluster 0, leaf level) so the very first add() has a target.
+ startNewClusterPage();
+
+ LOGGER.log(Level.TRACE,
+ "VTreeStaticStructureBuilder (bottom-up) initialized: numLevels={}, maxEntriesPerPage={}", numLevels,
+ maxEntriesPerPage);
+ printStructureInfo();
+ }
+
+ @Override
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_4_8, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.ASSISTED)
+ public void add(ITupleReference tuple) throws HyracksDataException {
+ int childPageId = determineChildPageId();
+ ITupleReference entryTuple = createEntryTuple(tuple, childPageId);
+
+ int spaceNeeded = currentFrame.getBytesRequiredToWriteTuple(entryTuple) + slotSize;
+ int spaceAvailable = currentFrame.getTotalFreeSpace();
+
+ // An entry tuple larger than a fresh empty page's usable space (pageSize - pageHeaderSize -
+ // slotSize) can never fit; reject it here instead of looping/overrunning at insertSorted().
+ int maxUsableTupleBytes = bufferCache.getPageSize() - currentFrame.getPageHeaderSize() - slotSize;
+ int entryTupleBytes = currentFrame.getBytesRequiredToWriteTuple(entryTuple) - slotSize;
+ if (entryTupleBytes > maxUsableTupleBytes) {
+ throw HyracksDataException.create(org.apache.hyracks.api.exceptions.ErrorCode.RECORD_IS_TOO_LARGE,
+ entryTupleBytes, maxUsableTupleBytes);
+ }
+
+ if (spaceNeeded > spaceAvailable) {
+ createOverflowPage();
+ }
+
+ ((IVTreeFrame) currentFrame).insertSorted(entryTuple);
+ advancePosition();
+ }
+
+ /**
+ * Compute the child pointer to write for the next tuple.
+ * <p>
+ * Leaf tuples carry a sentinel — {@link VTreeBulkLoader} overwrites with the real
+ * directory page pointer when loading data. Interior tuples look up the first
+ * page of their target child cluster in level+1, which (by bottom-up order) has
+ * already been written and recorded in {@link #firstPageIdOfCluster}.
+ */
+ private int determineChildPageId() {
+ if (currentLevel == numLevels - 1) {
+ return LEAF_METADATA_PTR_SENTINEL;
+ }
+ int childClusterIndex = computeChildClusterIndex();
+ int childPageId = firstPageIdOfCluster[currentLevel + 1][childClusterIndex];
+ if (childPageId < 0) {
+ throw new IllegalStateException(
+ "Child page id not yet recorded for level=" + (currentLevel + 1) + ", cluster=" + childClusterIndex
+ + ". Input must be in bottom-up order (leaf level first, root last).");
+ }
+ return childPageId;
+ }
+
+ /**
+ * The Nth centroid emitted in level L points to the Nth cluster in level L+1.
+ * Returns N based on the current build position.
+ */
+ private int computeChildClusterIndex() {
+ int centroidsProcessedInCurrentLevel = 0;
+ for (int c = 0; c < currentClusterInLevel; c++) {
+ centroidsProcessedInCurrentLevel += centroidsPerCluster.get(currentLevel).get(c);
+ }
+ centroidsProcessedInCurrentLevel += currentCentroidInCluster;
+ return centroidsProcessedInCurrentLevel;
+ }
+
+ /**
+ * Create the stored entry tuple from an input centroid tuple by copying every input field through
+ * verbatim and appending {@code childPageId} as the LAST field — the navigation child pointer for
+ * interior tuples, the metadata-page pointer for leaves. Keeping it last means
+ * {@code getMetadataPagePointer()} resolves it via {@code getFieldCount() - 1} regardless of how many
+ * optional fields the leaf carried.
+ * <p>
+ * The copy is schema-agnostic: interior tuples and every leaf variant take the same path, since
+ * appending a field never depends on the existing field types. Passing the raw field bytes through
+ * (rather than deserializing to {@code double[]}/{@code byte[]} and re-serializing) avoids a per-centroid
+ * round trip; the field layout this preserves is documented in {@link VTreeStaticTupleConstants}.
+ * Interior vs leaf is still driven by {@link #currentLevel} in {@code confiscateAndInitFrame}, which
+ * selects the target frame; this method does not need to distinguish them.
+ */
+ private ITupleReference createEntryTuple(ITupleReference tuple, int childPageId) throws HyracksDataException {
+ int inputFieldCount = tuple.getFieldCount();
+
+ if (LOGGER.isTraceEnabled()) {
+ int centroidId =
+ IntegerPointable.getInteger(tuple.getFieldData(VTreeStaticTupleConstants.CENTROID_ID_FIELD),
+ tuple.getFieldStart(VTreeStaticTupleConstants.CENTROID_ID_FIELD));
+ LOGGER.log(Level.TRACE, "Adding centroid {} at level={}, cluster={}, position={}, inputFields={}",
+ centroidId, currentLevel, currentClusterInLevel, currentCentroidInCluster, inputFieldCount);
+ }
+
+ try {
+ // Rebuild the reused builder only when the entry field count changes (constant within a level, so
+ // this happens once at the leaf->interior boundary). ArrayTupleReference reads its field count from
+ // the offsets-array length, so the builder must be sized exactly to the fields written.
+ int fieldCount = inputFieldCount + 1;
+ if (entryTupleFieldCount != fieldCount) {
+ entryTupleBuilder = new ArrayTupleBuilder(fieldCount);
+ entryTupleFieldCount = fieldCount;
+ }
+ entryTupleBuilder.reset();
+ for (int i = 0; i < inputFieldCount; i++) {
+ entryTupleBuilder.addField(tuple.getFieldData(i), tuple.getFieldStart(i), tuple.getFieldLength(i));
+ }
+ entryTupleBuilder.getDataOutput().writeInt(childPageId);
+ entryTupleBuilder.addFieldEndOffset();
+
+ entryTupleRef.reset(entryTupleBuilder.getFieldEndOffsets(), entryTupleBuilder.getByteArray());
+ return entryTupleRef;
+ } catch (Exception e) {
+ LOGGER.log(Level.TRACE, "Error creating entry tuple: {}", e.getMessage());
+ throw new HyracksDataException("Failed to create entry tuple", e);
+ }
+ }
+
+ /**
+ * Allocate a fresh page id and confiscate it as the first page of the current
+ * cluster at the current level. Records the page id in firstPageIdOfCluster so
+ * the parent level can later resolve its child pointers.
+ */
+ private void startNewClusterPage() throws HyracksDataException {
+ int pageId = freePageManager.takePage(metaFrame);
+ firstPageIdOfCluster[currentLevel][currentClusterInLevel] = pageId;
+ confiscateAndInitFrame(pageId);
+ }
+
+ /**
+ * Allocate an overflow page within the current cluster, set the current page's
+ * next-page pointer to it (overflow flag = true), flush the current page, then
+ * continue building in the new page.
+ */
+ private void createOverflowPage() throws HyracksDataException {
+ int overflowPageId = freePageManager.takePage(metaFrame);
+ if (currentLevel == numLevels - 1) {
+ leafFrame.setNextLeaf(overflowPageId);
+ leafFrame.setOverflowFlagBit(true);
+ } else {
+ interiorFrame.setNextPage(overflowPageId);
+ interiorFrame.setOverflowFlagBit(true);
+ }
+ writePage(currentPage);
+ confiscateAndInitFrame(overflowPageId);
+ LOGGER.log(Level.TRACE, "Created overflow page {} for level {}, cluster {}", overflowPageId, currentLevel,
+ currentClusterInLevel);
+ }
+
+ /**
+ * Confiscate a page with the given id and initialize the appropriate frame.
+ */
+ private void confiscateAndInitFrame(int pageId) throws HyracksDataException {
+ long dpid = BufferedFileHandle.getDiskPageId(fileId, pageId);
+ currentPage = bufferCache.confiscatePage(dpid);
+ if (currentLevel == numLevels - 1) {
+ currentFrame = leafFrame;
+ leafFrame.setPage(currentPage);
+ leafFrame.initBuffer((byte) 0);
+ leafFrame.setLevel((byte) 0);
+ } else {
+ currentFrame = interiorFrame;
+ interiorFrame.setPage(currentPage);
+ interiorFrame.initBuffer((byte) 0);
+ interiorFrame.setLevel((byte) 1);
+ }
+ currentPageId = pageId;
+ LOGGER.log(Level.TRACE, "Opened page {} for level {}", pageId, currentLevel);
+ }
+
+ private void writePage(ICachedPage page) throws HyracksDataException {
+ if (currentLevel == numLevels - 1) {
+ // Leaf page: its neighbor lists are still provisional, so record each centroid's placement
+ // and publish the page LOCAL-ONLY. resolveAndUploadLeafNeighbors() later re-reads, resolves,
+ // and uploads it exactly once.
+ recordLeafPositions();
+ compressedPageWriter.prepareWrite(page);
+ leafPlacementWriter.write(page);
+ } else {
+ compressedPageWriter.prepareWrite(page);
+ pageWriter.write(page);
+ }
+ }
+
+ /** Record {@code centroidId -> (currentPageId, slot)} for every tuple on the leaf page being written. */
+ private void recordLeafPositions() throws HyracksDataException {
+ int tupleCount = ((ITreeIndexFrame) leafFrame).getTupleCount();
+ for (int i = 0; i < tupleCount; i++) {
+ leafDirectory.put(leafFrame.getCentroidId(i), new int[] { currentPageId, i });
+ }
+ leafPageIds.add(currentPageId);
+ }
+
+ /**
+ * Leaf resolution pass. Re-reads each leaf page (from local storage, where placement published it) into a
+ * <em>confiscated</em> buffer, rewrites its provisional neighbor entries to physical pointers
+ * {@code (pageId, slot)} using the directory, and uploads the page exactly once through the same FIFO
+ * writer used for interior pages (local + cloud). Pages are processed in ascending page-id order, so the
+ * append-only cloud writer sees monotonically increasing offsets (and all leaf uploads precede any
+ * interior/root upload, exactly as in the single-pass build).
+ * <p>
+ * The page is confiscated (not pinned) for the whole re-read/resolve/upload, so it is invisible to the
+ * cleaner sweep and to eviction — the build thread is the sole writer of each leaf, which the append-only
+ * cloud writer requires. This matches how interior pages are written; going through the FIFO writer also
+ * runs the compression prepare step (unlike the previous pin/flush path, which assumed no compression).
+ */
+ private void resolveAndUploadLeafNeighbors() throws HyracksDataException {
+ for (int leafPageId : leafPageIds) {
+ long dpid = BufferedFileHandle.getDiskPageId(fileId, leafPageId);
+ ICachedPage page = bufferCache.confiscateAndLoad(dpid);
+ try {
+ resolveLeafPageNeighbors(page);
+ compressedPageWriter.prepareWrite(page);
+ pageWriter.write(page);
+ } finally {
+ // pageWriter.write returns the page (clearing its confiscated flag). If resolve/prepare
+ // threw before it ran, the page is still confiscated, so we return it here (cf. abort()).
+ if (page.confiscated()) {
+ bufferCache.returnPage(page, false);
+ }
+ }
+ }
+ }
+
+ /** Rewrite provisional neighbor entries on a single leaf page to resolved {@code (pageId, slot)} pointers. */
+ private void resolveLeafPageNeighbors(ICachedPage page) throws HyracksDataException {
+ leafFrame.setPage(page);
+ VTreeLeafNeighborList.forEachLeafNeighborEntry(leafFrame, (neighborList, start, e) -> {
+ if (VTreeLeafNeighborList.isResolved(neighborList, start, e)) {
+ return;
+ }
+ int neighborCid = VTreeLeafNeighborList.readCentroidId(neighborList, start, e);
+ int[] loc = leafDirectory.get(neighborCid);
+ if (loc != null) {
+ VTreeLeafNeighborList.writeResolved(neighborList, start, e, loc[0], loc[1]);
+ }
+ });
+ }
+
+ /**
+ * Advance counters; if a cluster or level boundary is crossed, handle the
+ * transition (chain pages, flush, open a new page for the next position).
+ */
+ private void advancePosition() throws HyracksDataException {
+ currentCentroidInCluster++;
+
+ if (currentCentroidInCluster >= centroidsPerCluster.get(currentLevel).get(currentClusterInLevel)) {
+ currentCentroidInCluster = 0;
+ currentClusterInLevel++;
+
+ if (currentClusterInLevel >= clustersPerLevel.get(currentLevel)) {
+ transitionToNextLevel();
+ } else {
+ transitionToNextCluster();
+ }
+ }
+ }
+
+ /**
+ * Move to the next cluster within the current level. For leaf level we set
+ * the just-finished page's nextLeaf to the new cluster's first page id (with
+ * overflow=false, since this is a sibling link not an intra-cluster chain).
+ * For interior levels each cluster is independent, so we simply flush the
+ * old page and start fresh.
+ */
+ private void transitionToNextCluster() throws HyracksDataException {
+ if (currentLevel == numLevels - 1) {
+ int nextClusterPageId = freePageManager.takePage(metaFrame);
+ firstPageIdOfCluster[currentLevel][currentClusterInLevel] = nextClusterPageId;
+ leafFrame.setNextLeaf(nextClusterPageId);
+ leafFrame.setOverflowFlagBit(false);
+ writePage(currentPage);
+ confiscateAndInitFrame(nextClusterPageId);
+ LOGGER.log(Level.TRACE, "Leaf cluster transition: chained to page {} (cluster {})", nextClusterPageId,
+ currentClusterInLevel);
+ } else {
+ writePage(currentPage);
+ startNewClusterPage();
+ LOGGER.log(Level.TRACE, "Interior cluster transition at level {} to cluster {}", currentLevel,
+ currentClusterInLevel);
+ }
+ }
+
+ /**
+ * Move from the current level to the next level up. Flushes the last page of
+ * the current level (no successor pointer to set) and opens the first page of
+ * the next level if any. After level 0 (root) is done, currentPage is cleared.
+ */
+ private void transitionToNextLevel() throws HyracksDataException {
+ boolean finishingLeafLevel = (currentLevel == numLevels - 1);
+ writePage(currentPage);
+ if (finishingLeafLevel) {
+ // Every leaf page is now placed locally and the directory is complete; resolve neighbor
+ // pointers in place and upload the leaf pages once, before any higher-level page is written.
+ resolveAndUploadLeafNeighbors();
+ }
+ currentLevel--;
+ currentClusterInLevel = 0;
+ currentCentroidInCluster = 0;
+ if (currentLevel >= 0) {
+ startNewClusterPage();
+ LOGGER.log(Level.TRACE, "Moving up to level {}", currentLevel);
+ } else {
+ // All levels written; building is complete.
+ currentPage = null;
+ currentFrame = null;
+ }
+ }
+
+ @Override
+ public void end() throws HyracksDataException {
+ // Defensive: if the structure spec didn't drain currentPage via advancePosition
+ // (e.g. caller called end() with a partially-built page), flush it now.
+ if (currentPage != null) {
+ LOGGER.log(Level.TRACE, "end(): flushing unfinalized page {}", currentPageId);
+ writePage(currentPage);
+ currentPage = null;
+ }
+
+ if (hasFailed()) {
+ throw HyracksDataException.create(getFailure());
+ }
+
+ // Root cluster's first page is the entry point. In bottom-up build this is
+ // also the page with the highest id since the root level is processed last.
+ int rootPageId = firstPageIdOfCluster[0][0];
+ if (rootPageId < 0) {
+ throw HyracksDataException
+ .create(new IllegalStateException("Root page id was never recorded; static structure incomplete."));
+ }
+
+ metaFrame.put(VTreeMetadataKeys.NUM_LEAF_CENTROIDS,
+ LongPointable.FACTORY.createPointable(getNumLeafCentroids()));
+ metaFrame.put(VTreeMetadataKeys.FIRST_LEAF_CENTROID_ID,
+ LongPointable.FACTORY.createPointable(totalCentroidsUpToLevel[numLevels - 1]));
+
+ freePageManager.setRootPageId(rootPageId);
+
+ LOGGER.log(Level.TRACE, "VTreeStaticStructureBuilder (bottom-up) done; rootPageId={}", rootPageId);
+ }
+
+ @Override
+ public void abort() throws HyracksDataException {
+ LOGGER.log(Level.TRACE, "VTreeStaticStructureBuilder aborted");
+ compressedPageWriter.abort();
+ if (currentPage != null && currentPage.confiscated()) {
+ bufferCache.returnPage(currentPage, false);
+ }
+ currentPage = null;
+ freePageManager.returnAllPages();
+ }
+
+ @Override
+ public void force() throws HyracksDataException {
+ bufferCache.force(fileId, false);
+ }
+
+ private int getNumLeafCentroids() {
+ return totalCentroidsUpToLevel[numLevels] - totalCentroidsUpToLevel[numLevels - 1];
+ }
+
+ private void printStructureInfo() {
+ if (!LOGGER.isTraceEnabled()) {
+ return; // skip the StringBuilder work entirely when trace is off
+ }
+ LOGGER.log(Level.TRACE, "Structure configuration (bottom-up):");
+ for (int level = 0; level < numLevels; level++) {
+ StringBuilder sb = new StringBuilder();
+ sb.append("Level ").append(level).append(": ").append(clustersPerLevel.get(level))
+ .append(" clusters, centroids=[");
+ List<Integer> levelCentroids = centroidsPerCluster.get(level);
+ for (int cluster = 0; cluster < levelCentroids.size(); cluster++) {
+ sb.append(levelCentroids.get(cluster));
+ if (cluster < levelCentroids.size() - 1) {
+ sb.append(", ");
+ }
+ }
+ sb.append("]");
+ LOGGER.log(Level.TRACE, sb.toString());
+ }
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/tuples/VTreeTupleUtils.java b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/tuples/VTreeTupleUtils.java
new file mode 100644
index 0000000000..ba84c9c3f6
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/tuples/VTreeTupleUtils.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.vector.tuples;
+
+import java.util.Arrays;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.storage.am.vector.api.IVTreeBinaryAccessor;
+import org.apache.hyracks.storage.am.vector.api.IVTreeBinaryAccessorFactory;
+
+/**
+ * Helpers for working with VTree tuples (copying and primary-key extraction).
+ * Input tuple field layout: vector first, primary key last, include fields in between.
+ */
+public class VTreeTupleUtils {
+ /**
+ * Returns the bytes of the primary key field (the last field of the tuple).
+ * Only single-field primary keys are currently supported.
+ *
+ * @param tuple input tuple; must have at least 2 fields
+ * @return primary-key bytes, or {@code null} if the tuple is null/too short or the field is empty
+ */
+ public static byte[] extractPrimaryKeyFromTuple(ITupleReference tuple) {
+ if (tuple == null || tuple.getFieldCount() < 2) {
+ return null;
+ }
+
+ int pkFieldIndex = tuple.getFieldCount() - 1;
+
+ byte[] data = tuple.getFieldData(pkFieldIndex);
+ if (data == null) {
+ return null;
+ }
+
+ int offset = tuple.getFieldStart(pkFieldIndex);
+ int length = tuple.getFieldLength(pkFieldIndex);
+
+ if (length <= 0) {
+ return null;
+ }
+
+ return Arrays.copyOfRange(data, offset, offset + length);
+ }
+
+ /**
+ * Decode the vector stored at {@code fieldIndex} of the tuple using the supplied accessor factory.
+ *
+ * @param tuple input tuple; must have at least {@code fieldIndex + 1} fields
+ * @param fieldIndex zero-based index of the vector field
+ * @param factory accessor factory used to decode the field bytes
+ * @return decoded vector, or {@code null} if the tuple or factory is unusable
+ * @throws HyracksDataException if the underlying accessor fails to decode
+ */
+ public static double[] extractVectorFromTuple(ITupleReference tuple, int fieldIndex,
+ IVTreeBinaryAccessorFactory factory) throws HyracksDataException {
+ if (tuple == null || tuple.getFieldCount() <= fieldIndex || factory == null) {
+ return null;
+ }
+ IVTreeBinaryAccessor accessor = factory.createAccessor();
+ accessor.reset(tuple.getFieldData(fieldIndex), tuple.getFieldStart(fieldIndex),
+ tuple.getFieldLength(fieldIndex));
+ return accessor.getVector();
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/utils/CrossPollinationConfig.java b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/utils/CrossPollinationConfig.java
new file mode 100644
index 0000000000..92ac2f6e5f
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/utils/CrossPollinationConfig.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.vector.utils;
+
+import java.io.Serializable;
+
+/**
+ * Immutable cross-pollination parameters threaded from the index DDL (WITH clause) down to the
+ * storage-layer {@code VTree}, so that incremental insert/delete reproduce the exact multi-cluster
+ * placement that bulk-load performs.
+ * <p>
+ * At bulk-load a record is written into the {@code m} closest leaf centroids, after eps-filtering
+ * ({@code epsilon}) the candidate list and thinning it by the SPTAG-style RNG diversity rule
+ * ({@code rngFactor}; see {@link RngAcceptanceFilter}). For incremental delete to cancel <em>every</em>
+ * replica, the DML path must resolve the same centroid set — which means it must use the same three
+ * parameters. {@code m == 1} reproduces the legacy single-closest behavior and bypasses the
+ * level-wise/RNG machinery entirely.
+ *
+ * @param m replica count (>= 1). 1 disables cross-pollination.
+ * @param rngFactor RNG diversity multiplier (canonical SPTAG = 1.0; non-finite disables the rule).
+ * @param epsilon level-wise candidate window used to gather centroids before RNG thinning.
+ */
+public record CrossPollinationConfig(int m, double rngFactor, double epsilon) implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ /** Legacy single-closest placement: no cross-pollination. Matches the WITH-clause defaults. */
+ public static final CrossPollinationConfig LEGACY = new CrossPollinationConfig(1, 1.0, 0.3);
+
+ public CrossPollinationConfig {
+ m = Math.max(1, m);
+ // epsilon is a multiplicative window width; a negative value is meaningless (it would
+ // shrink rather than widen the candidate window). Reject it up front.
+ if (epsilon < 0.0) {
+ throw new IllegalArgumentException("epsilon must be >= 0, got " + epsilon);
+ }
+ // rngFactor is intentionally NOT validated: a non-finite value (NaN / +Infinity) is a
+ // legal input that disables the RNG diversity rule in RngAcceptanceFilter (degrading to a
+ // pure top-cap slice). Only m is clamped; rngFactor semantics are the caller's contract.
+ }
+
+
+ /** @return true when records are replicated into more than one cluster. */
+ public boolean enabled() {
+ return m > 1;
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/utils/NoOpVectorQuantizer.java b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/utils/NoOpVectorQuantizer.java
new file mode 100644
index 0000000000..33a97bb043
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/utils/NoOpVectorQuantizer.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.vector.utils;
+
+import java.nio.ByteBuffer;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.vector.api.IVTreeQuantizer;
+
+/**
+ * Identity {@link IVTreeQuantizer} for unit tests: {@code quantize} returns the input vector
+ * unchanged and {@code dequantize} reads raw big-endian doubles. Callers are responsible for
+ * stripping any {@code ByteArrayPointable} length prefix before invoking {@code dequantize}.
+ */
+public class NoOpVectorQuantizer implements IVTreeQuantizer {
+
+ public static final NoOpVectorQuantizer INSTANCE = new NoOpVectorQuantizer();
+
+ @Override
+ public double[] quantize(double[] vector) throws HyracksDataException {
+ return vector;
+ }
+
+ @Override
+ public double[] dequantize(byte[] quantizedBytes) throws HyracksDataException {
+ // Identity dequantize reads whole big-endian doubles; a non-multiple-of-8 length means the
+ // caller failed to strip a length prefix (or passed a truncated buffer). Fail loudly here
+ // (test-only helper) rather than silently dropping trailing bytes.
+ if (quantizedBytes.length % Double.BYTES != 0) {
+ throw new IllegalArgumentException("quantizedBytes length " + quantizedBytes.length
+ + " is not a multiple of Double.BYTES (" + Double.BYTES + ")");
+ }
+ int count = quantizedBytes.length / Double.BYTES;
+
+ ByteBuffer buf = ByteBuffer.wrap(quantizedBytes);
+ double[] vector = new double[count];
+ for (int i = 0; i < count; i++) {
+ vector[i] = buf.getDouble();
+ }
+ return vector;
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/utils/RngAcceptanceFilter.java b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/utils/RngAcceptanceFilter.java
new file mode 100644
index 0000000000..a031d3110c
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/utils/RngAcceptanceFilter.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.vector.utils;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.vector.api.IVTreeDistanceFunction;
+import org.apache.hyracks.storage.am.vector.impls.ClusterSearchResult;
+
+/**
+ * SPTAG/SPANN-style RNG (relative neighborhood graph) acceptance filter for
+ * vector-index cross-pollination at bulk-load time.
+ * <p>
+ * Given a candidate list of leaf centroids already sorted closest-first w.r.t. the record
+ * being inserted (typically the eps-filtered output of
+ * {@code VTreeNavigationUtils.findCloseCentroidsLevelWiseGlobalSort}), this helper walks
+ * the list and decides which candidates to accept as replica targets. A candidate
+ * {@code c_i} is rejected iff some already-accepted replica {@code r} satisfies
+ * {@code rngFactor * dist(c_i, r) < dist(x, c_i)} (the canonical SPTAG rule at
+ * {@code rngFactor = 1}). Acceptance stops at {@code cap}.
+ * <p>
+ * Distances between centroids are computed in raw (full-precision) space using the
+ * supplied {@link IVTreeDistanceFunction}, even for quantized indexes — diversity
+ * should reflect geometric truth, not quantization artifacts.
+ * <p>
+ * Edge handling: the filter is defensive — a missing centroid vector or a NaN /
+ * throwing pairwise distance causes the affected pair to be skipped (no veto from
+ * data we can't evaluate), never a silent candidate drop.
+ */
+public final class RngAcceptanceFilter {
+
+ /** Per-call accept/reject counters (for stats logging). Fields are reset by the caller. */
+ public static final class Stats {
+ public int accepted;
+ public int rejected;
+ }
+
+ private RngAcceptanceFilter() {
+ }
+
+ /**
+ * Greedy SPTAG-style acceptance pass over a closest-first candidate list.
+ *
+ * @param candidates ε-filtered, distance-sorted (ascending) candidate centroids;
+ * never null, may be empty.
+ * @param dist raw vector-to-vector distance function with the index's metric.
+ * @param rngFactor RNG multiplier. Strict SPTAG = 1.0; >1 looser; non-finite
+ * (e.g. {@link Double#POSITIVE_INFINITY}) disables the rule
+ * (degrades to a pure top-{@code cap} slice).
+ * @param cap upper bound on accepted count (>=1). Caller-side {@code M}.
+ * @param out optional stats sink; ignored if null.
+ * @return list of accepted candidates, length <= {@code cap} (possibly empty).
+ */
+ public static List<ClusterSearchResult> accept(List<ClusterSearchResult> candidates, IVTreeDistanceFunction dist,
+ double rngFactor, int cap, Stats out) throws HyracksDataException {
+ if (candidates == null || candidates.isEmpty() || cap <= 0) {
+ return Collections.emptyList();
+ }
+
+ final int n = candidates.size();
+ final boolean rngEnabled = Double.isFinite(rngFactor) && dist != null;
+ final List<ClusterSearchResult> accepted = new ArrayList<>(Math.min(cap, n));
+
+ int acceptedCount = 0;
+ int rejectedCount = 0;
+
+ for (int i = 0; i < n && accepted.size() < cap; i++) {
+ final ClusterSearchResult cand = candidates.get(i);
+ if (cand == null) {
+ continue;
+ }
+
+ boolean reject = false;
+ // Diversity test only runs once we have at least one accepted replica and RNG is enabled.
+ // If the candidate's centroid is missing we cannot evaluate the test — accept rather than
+ // silently drop a legitimate candidate.
+ if (rngEnabled && !accepted.isEmpty() && cand.centroid != null) {
+ final double dxc = cand.distance; // dist(x, c_i) — already computed by navigation
+ if (!Double.isNaN(dxc)) {
+ for (int j = 0; j < accepted.size(); j++) {
+ final ClusterSearchResult r = accepted.get(j);
+ if (r.centroid == null) {
+ continue;
+ }
+ double dcr;
+ try {
+ dcr = dist.apply(cand.centroid, r.centroid);
+ } catch (HyracksDataException dex) {
+ continue; // bad pair — don't let it veto
+ }
+ if (Double.isNaN(dcr)) {
+ continue;
+ }
+ if (rngFactor * dcr < dxc) {
+ reject = true;
+ break;
+ }
+ }
+ }
+ }
+
+ if (reject) {
+ rejectedCount++;
+ } else {
+ accepted.add(cand);
+ acceptedCount++;
+ }
+ }
+
+ if (out != null) {
+ out.accepted = acceptedCount;
+ out.rejected = rejectedCount;
+ }
+ return accepted;
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/utils/VTreeChildCentroid.java b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/utils/VTreeChildCentroid.java
new file mode 100644
index 0000000000..a7d214190b
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/utils/VTreeChildCentroid.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.vector.utils;
+
+/**
+ * Immutable record describing one child entry of an interior page: the child's page id,
+ * the distance from a query vector to the child's centroid, and the tuple index within the
+ * parent page ({@code tupleIndex}). Used as a sortable element when ranking interior children
+ * by distance.
+ */
+public record VTreeChildCentroid(int childPageId, double distance, int tupleIndex) {
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/utils/VTreeDataTupleAccessor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/utils/VTreeDataTupleAccessor.java
new file mode 100644
index 0000000000..69497f95f7
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/utils/VTreeDataTupleAccessor.java
@@ -0,0 +1,176 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.vector.utils;
+
+import org.apache.hyracks.data.std.primitive.ByteArrayPointable;
+import org.apache.hyracks.data.std.primitive.DoublePointable;
+import org.apache.hyracks.data.std.primitive.IntegerPointable;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+
+/**
+ * Single authority for the VTree DATA-tuple field layout
+ * {@code [distance, centroidId, (quantized_distance, quantized_embedding,) PK..., includes...]}.
+ * <p>
+ * Owns the field indices (as static constants), the {@code isQuantized}-dependent derivations, and the
+ * typed field reads so that no other class hardcodes a field position. The layout is a function of
+ * {@code isQuantized} alone — the PK and INCLUDE fields simply follow the secondary fields at
+ * {@link #pkStartField()} — so the accessor is immutable and trivially constructible anywhere that flag
+ * is known. The writer {@code VTreeDataTupleBuilder} builds tuples in this exact order, so the two must
+ * stay in lock-step.
+ * <p>
+ * Two formats are supported, determined by the presence of a "description" field in the DDL WITH clause
+ * at index creation time:
+ * <pre>
+ * Non-quantized (description == null):
+ * | distance(0) | centroidId(1) | PK...(2+) | includes...(N+) |
+ * Quantized (description != null):
+ * | distance(0) | centroidId(1) | quantized_distance(2) | quantized_embedding(3) | PK...(4+) | includes...(N+) |
+ * </pre>
+ */
+public final class VTreeDataTupleAccessor {
+
+ // ---------- page-chain sentinels ----------
+
+ /** Value stored in a frame's next-page pointer to mark the end of a page chain. */
+ public static final int NO_NEXT_PAGE = -1;
+
+ /** Value stored in a per-cluster directory-page slot before a real page id is assigned. */
+ public static final int UNASSIGNED_DIR_PAGE = -1;
+
+ // ---------- non-quantized format field offsets (description == null) ----------
+
+ /** Distance field index (same in both formats). */
+ public static final int DISTANCE_FIELD = 0;
+
+ /** Centroid ID field index in non-quantized format. */
+ public static final int NQ_CENTROID_ID_FIELD = 1;
+
+ /** First primary key field index in non-quantized format. */
+ public static final int NQ_PK_START_FIELD = 2;
+
+ /** Number of secondary (non-PK, non-include) fields in non-quantized format. */
+ public static final int NQ_NUM_SECONDARY_FIELDS = 2;
+
+ // ---------- quantized format field offsets (description != null) ----------
+ // Layout: [distance(0), centroidId(1), quantized_distance(2), quantized_embedding(3), PK...(4+)]
+ // This matches createTransformedTuple() in VTreeBulkLoaderAndGroupingOperatorDescriptor.
+
+ /** Centroid ID field index in quantized format. */
+ public static final int Q_CENTROID_ID_FIELD = 1;
+
+ /** Quantized distance field index in quantized format. */
+ public static final int Q_QUANTIZED_DISTANCE_FIELD = 2;
+
+ /** Quantized embedding field index in quantized format. */
+ public static final int Q_QUANTIZED_EMBEDDING_FIELD = 3;
+
+ /** First primary key field index in quantized format. */
+ public static final int Q_PK_START_FIELD = 4;
+
+ /** Number of secondary (non-PK, non-include) fields in quantized format. */
+ public static final int Q_NUM_SECONDARY_FIELDS = 4;
+
+ /** Returns the PK start field index for the given quantization mode. */
+ public static int getPkStartField(boolean isQuantized) {
+ return isQuantized ? Q_PK_START_FIELD : NQ_PK_START_FIELD;
+ }
+
+ /** Returns the number of secondary fields (before PKs and includes) for the given quantization mode. */
+ public static int getNumSecondaryFields(boolean isQuantized) {
+ return isQuantized ? Q_NUM_SECONDARY_FIELDS : NQ_NUM_SECONDARY_FIELDS;
+ }
+
+ /** Returns the centroid ID field index for the given quantization mode. */
+ public static int getCentroidIdField(boolean isQuantized) {
+ return isQuantized ? Q_CENTROID_ID_FIELD : NQ_CENTROID_ID_FIELD;
+ }
+
+ private final boolean quantized;
+
+ public VTreeDataTupleAccessor(boolean quantized) {
+ this.quantized = quantized;
+ }
+
+ public boolean isQuantized() {
+ return quantized;
+ }
+
+ // ---------- field indices (the single place these are derived) ----------
+
+ public int distanceField() {
+ return DISTANCE_FIELD;
+ }
+
+ public int centroidIdField() {
+ return getCentroidIdField(quantized);
+ }
+
+ /** Quantized-distance field index, or {@code -1} for a non-quantized layout. */
+ public int quantizedDistanceField() {
+ return quantized ? Q_QUANTIZED_DISTANCE_FIELD : -1;
+ }
+
+ /** Quantized-embedding field index, or {@code -1} for a non-quantized layout. */
+ public int quantizedEmbeddingField() {
+ return quantized ? Q_QUANTIZED_EMBEDDING_FIELD : -1;
+ }
+
+ public int numSecondaryFields() {
+ return getNumSecondaryFields(quantized);
+ }
+
+ public int pkStartField() {
+ return getPkStartField(quantized);
+ }
+
+ /** First INCLUDE field index, given the number of primary-key fields (they follow the PK). */
+ public int includeStartField(int numPrimaryKeyFields) {
+ return pkStartField() + numPrimaryKeyFields;
+ }
+
+ // ---------- typed reads ----------
+
+ /** Distance-to-centroid in field 0 (raw big-endian double, no type tag). */
+ public double getDistance(ITupleReference tuple) {
+ int f = distanceField();
+ return DoublePointable.getDouble(tuple.getFieldData(f), tuple.getFieldStart(f));
+ }
+
+ /** Centroid id in field 1 (raw big-endian int, no type tag). */
+ public int getCentroidId(ITupleReference tuple) {
+ int f = centroidIdField();
+ return IntegerPointable.getInteger(tuple.getFieldData(f), tuple.getFieldStart(f));
+ }
+
+ /**
+ * Content bytes of the quantized embedding (field 3) with the {@link ByteArrayPointable} varlen
+ * length prefix stripped. Only valid on a quantized layout.
+ */
+ public byte[] getQuantizedEmbedding(ITupleReference tuple) {
+ int f = quantizedEmbeddingField();
+ byte[] data = tuple.getFieldData(f);
+ int offset = tuple.getFieldStart(f);
+ int contentLength = ByteArrayPointable.getContentLength(data, offset);
+ int prefixSize = ByteArrayPointable.getNumberBytesToStoreMeta(contentLength);
+ byte[] content = new byte[contentLength];
+ System.arraycopy(data, offset + prefixSize, content, 0, contentLength);
+ return content;
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/utils/VTreeLeafNeighborList.java b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/utils/VTreeLeafNeighborList.java
new file mode 100644
index 0000000000..228d2af67a
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/utils/VTreeLeafNeighborList.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.vector.utils;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.primitive.ByteArrayPointable;
+import org.apache.hyracks.data.std.primitive.IntegerPointable;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleReference;
+import org.apache.hyracks.storage.am.vector.api.IVTreeLeafFrame;
+import org.apache.hyracks.storage.am.vector.api.VTreeStaticTupleConstants;
+
+/**
+ * Byte encoding for the graph-style neighbor list carried by a VTree leaf centroid.
+ * <p>
+ * The neighbor list is a variable-length byte field made of fixed {@link #ENTRY_SIZE}-byte entries.
+ * An entry has the same width whether it is provisional or resolved — the invariant the two-pass
+ * static-structure build depends on, so the resolution pass can overwrite each entry in place:
+ * <ul>
+ * <li><b>provisional</b>: {@code [int centroidId][int SENTINEL]} — neighbor identified by centroid id,
+ * not yet resolved to a physical location.</li>
+ * <li><b>resolved</b>: {@code [int pageId][int slot]} — direct physical pointer into a leaf page.</li>
+ * </ul>
+ * The entry count is recoverable as {@code length / ENTRY_SIZE}; there is no count header and no cap.
+ */
+public final class VTreeLeafNeighborList {
+
+ /** Bytes per neighbor entry: two 4-byte ints. */
+ public static final int ENTRY_SIZE = Integer.BYTES * 2;
+
+ /**
+ * Marker stored in an entry's second int while it is provisional. A real {@code slot} is a
+ * tuple index within a page and is always {@code >= 0}, so a negative marker can never collide.
+ */
+ public static final int SENTINEL = Integer.MIN_VALUE;
+
+ /** Shared empty neighbor list (no neighbors). */
+ public static final byte[] EMPTY = new byte[0];
+
+ private VTreeLeafNeighborList() {
+ }
+
+ /**
+ * Encode a provisional neighbor list from neighbor centroid ids. Each id becomes one
+ * {@code [centroidId, SENTINEL]} entry. A {@code null} or empty input yields {@link #EMPTY}.
+ */
+ public static byte[] encodeProvisional(int[] neighborCentroidIds) {
+ if (neighborCentroidIds == null || neighborCentroidIds.length == 0) {
+ return EMPTY;
+ }
+ byte[] buf = new byte[neighborCentroidIds.length * ENTRY_SIZE];
+ for (int i = 0; i < neighborCentroidIds.length; i++) {
+ int off = i * ENTRY_SIZE;
+ IntegerPointable.setInteger(buf, off, neighborCentroidIds[i]);
+ IntegerPointable.setInteger(buf, off + 4, SENTINEL);
+ }
+ return buf;
+ }
+
+ /** Number of entries in a neighbor-list byte field. */
+ public static int entryCount(byte[] buf, int start, int length) {
+ return length / ENTRY_SIZE;
+ }
+
+ /** First int of an entry: the neighbor's centroid id while provisional (also the page id once resolved). */
+ public static int readCentroidId(byte[] buf, int start, int entryIndex) {
+ return IntegerPointable.getInteger(buf, start + entryIndex * ENTRY_SIZE);
+ }
+
+ /** First int of a resolved entry: the leaf page id. */
+ public static int readPageId(byte[] buf, int start, int entryIndex) {
+ return IntegerPointable.getInteger(buf, start + entryIndex * ENTRY_SIZE);
+ }
+
+ /** Second int of a resolved entry: the slot (tuple index) within the page. */
+ public static int readSlot(byte[] buf, int start, int entryIndex) {
+ return IntegerPointable.getInteger(buf, start + entryIndex * ENTRY_SIZE + 4);
+ }
+
+ /** True if the entry has been resolved (its second int is no longer the {@link #SENTINEL}). */
+ public static boolean isResolved(byte[] buf, int start, int entryIndex) {
+ return IntegerPointable.getInteger(buf, start + entryIndex * ENTRY_SIZE + 4) != SENTINEL;
+ }
+
+ /**
+ * Overwrite one entry in place with its resolved physical pointer. Used by the resolution pass;
+ * does not change the byte width, so surrounding tuple/slot layout is untouched.
+ */
+ public static void writeResolved(byte[] buf, int start, int entryIndex, int pageId, int slot) {
+ int off = start + entryIndex * ENTRY_SIZE;
+ IntegerPointable.setInteger(buf, off, pageId);
+ IntegerPointable.setInteger(buf, off + 4, slot);
+ }
+
+ /**
+ * Per-entry callback invoked by {@link #forEachLeafNeighborEntry} for every neighbor entry on a
+ * leaf page. Receives the neighbor-list field bytes, the offset of the first entry within them,
+ * and the entry index; the action reads/rewrites the entry in place via the {@code read*}/
+ * {@link #writeResolved} accessors.
+ */
+ @FunctionalInterface
+ public interface EntryAction {
+ void apply(byte[] neighborList, int start, int entryIndex) throws HyracksDataException;
+ }
+
+ /**
+ * Decode every tuple's neighbor list on the given leaf page (already set on {@code lfFrame}) and
+ * invoke {@code action} once per neighbor entry. Tuples whose layout has no neighbor field
+ * (field count ≤ {@link VTreeStaticTupleConstants#LEAF_NEIGHBOR_LIST_FIELD}{@code + 1}) are
+ * skipped. This is the shared decode prologue used by the static-structure builder and the two
+ * loaders; only the per-entry action differs between them.
+ * <p>
+ * Drives the frame's own reusable tuple reference ({@link IVTreeLeafFrame#getFrameTuple()}) rather
+ * than allocating one per call; the callers set the page and do not hold a cursor position across
+ * this call, and the per-entry {@code action} operates on raw page bytes, so the shared reference
+ * is safe here.
+ */
+ public static void forEachLeafNeighborEntry(IVTreeLeafFrame lfFrame, EntryAction action)
+ throws HyracksDataException {
+ int tupleCount = lfFrame.getTupleCount();
+ ITreeIndexTupleReference frameTuple = lfFrame.getFrameTuple();
+ for (int t = 0; t < tupleCount; t++) {
+ frameTuple.resetByTupleIndex(lfFrame, t);
+ // Neighbor list lives at LEAF_NEIGHBOR_LIST_FIELD, before the last (metadata-pointer) field;
+ // absent in leaf layouts with fewer fields.
+ if (frameTuple.getFieldCount() <= VTreeStaticTupleConstants.LEAF_NEIGHBOR_LIST_FIELD + 1) {
+ continue;
+ }
+ byte[] neighborList = frameTuple.getFieldData(VTreeStaticTupleConstants.LEAF_NEIGHBOR_LIST_FIELD);
+ int fieldStart = frameTuple.getFieldStart(VTreeStaticTupleConstants.LEAF_NEIGHBOR_LIST_FIELD);
+ int contentLen = ByteArrayPointable.getContentLength(neighborList, fieldStart);
+ int start = fieldStart + ByteArrayPointable.getNumberBytesToStoreMeta(contentLen);
+ int numEntries = contentLen / ENTRY_SIZE;
+ for (int e = 0; e < numEntries; e++) {
+ action.apply(neighborList, start, e);
+ }
+ }
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/utils/VTreeLevelNode.java b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/utils/VTreeLevelNode.java
new file mode 100644
index 0000000000..099feea064
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/utils/VTreeLevelNode.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.vector.utils;
+
+/**
+ * Node in the level-wise traversal queue.
+ */
+record VTreeLevelNode(int pageId, int level) {
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/utils/VTreeMetadataKeys.java b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/utils/VTreeMetadataKeys.java
new file mode 100644
index 0000000000..e5c04ab3c7
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/utils/VTreeMetadataKeys.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.vector.utils;
+
+import java.nio.charset.StandardCharsets;
+
+import org.apache.hyracks.storage.am.common.freepage.MutableArrayValueReference;
+
+/**
+ * Keys for the VTree component metadata frame. Written by the static-structure builder,
+ * the bulk loader, and the flush loader; read back by the bulk loader and {@link
+ * org.apache.hyracks.storage.am.vector.impls.VTree} when a component is opened. The
+ * writer and reader sides must agree, so the keys live here as a single source of truth.
+ * <p>
+ * Exposed as reusable {@link MutableArrayValueReference} constants: the metadata frame's put/get COPY the
+ * key bytes, so a single shared, read-only reference is safe and avoids re-encoding the key on every access.
+ * The stored key bytes are short codes to keep the on-disk metadata compact.
+ */
+public final class VTreeMetadataKeys {
+
+ private VTreeMetadataKeys() {
+ }
+
+ /** Number of leaf centroids in the component (stored key {@code "VTNLC"}). */
+ public static final MutableArrayValueReference NUM_LEAF_CENTROIDS =
+ new MutableArrayValueReference("VTNLC".getBytes(StandardCharsets.UTF_8));
+
+ /** Centroid id of the first leaf centroid, BFS-from-root numbering (stored key {@code "VTFLC"}). */
+ public static final MutableArrayValueReference FIRST_LEAF_CENTROID_ID =
+ new MutableArrayValueReference("VTFLC".getBytes(StandardCharsets.UTF_8));
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/utils/VTreeMetadataTupleAccessor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/utils/VTreeMetadataTupleAccessor.java
new file mode 100644
index 0000000000..6370184e9f
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/utils/VTreeMetadataTupleAccessor.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.vector.utils;
+
+import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
+import org.apache.hyracks.api.dataflow.value.ITypeTraits;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.primitive.DoublePointable;
+import org.apache.hyracks.data.std.primitive.IntegerPointable;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.dataflow.common.data.marshalling.DoubleSerializerDeserializer;
+import org.apache.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import org.apache.hyracks.dataflow.common.utils.TupleUtils;
+
+/**
+ * Single authority for the VTree per-cluster directory ("metadata") tuple layout
+ * {@code [max_distance : double, data_page_pointer : int]}, mirroring {@code VTreeDataTupleAccessor}
+ * and {@code VTreeStaticTupleAccessor}. Owns the field indices, the {@link ITypeTraits} schema (so the
+ * frame factory reads it from here), and the typed field reads, so no other class hardcodes a position.
+ */
+public final class VTreeMetadataTupleAccessor {
+
+ /** Max distance-to-centroid of the data page this directory entry points at (field 0, raw double). */
+ public static final int MAX_DISTANCE_FIELD = 0;
+
+ /** Pointer to the data page (field 1, raw int). */
+ public static final int DATA_PAGE_POINTER_FIELD = 1;
+
+ /** Serializers for building a metadata tuple, in {@link #typeTraits()} field order. */
+ @SuppressWarnings("rawtypes")
+ public static final ISerializerDeserializer[] SERDES =
+ { DoubleSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE };
+
+ private VTreeMetadataTupleAccessor() {
+ }
+
+ /**
+ * Build a metadata entry tuple {@code <maxDistance:double, dataPageId:int>} for a directory frame,
+ * in {@link #typeTraits()} field order.
+ */
+ public static ITupleReference createMetadataTuple(double maxDistance, int dataPageId) throws HyracksDataException {
+ return TupleUtils.createTuple(SERDES, maxDistance, dataPageId);
+ }
+
+ /** Tuple schema {@code [max_distance : double, data_page_pointer : int]}. */
+ public static ITypeTraits[] typeTraits() {
+ ITypeTraits[] schema = new ITypeTraits[DATA_PAGE_POINTER_FIELD + 1];
+ schema[MAX_DISTANCE_FIELD] = DoublePointable.TYPE_TRAITS;
+ schema[DATA_PAGE_POINTER_FIELD] = IntegerPointable.TYPE_TRAITS;
+ return schema;
+ }
+
+ /** Max distance in field 0 (raw big-endian double, no type tag). */
+ public static double getMaxDistance(ITupleReference tuple) {
+ return DoublePointable.getDouble(tuple.getFieldData(MAX_DISTANCE_FIELD),
+ tuple.getFieldStart(MAX_DISTANCE_FIELD));
+ }
+
+ /** Data-page pointer in field 1 (raw big-endian int, no type tag). */
+ public static int getDataPagePointer(ITupleReference tuple) {
+ return IntegerPointable.getInteger(tuple.getFieldData(DATA_PAGE_POINTER_FIELD),
+ tuple.getFieldStart(DATA_PAGE_POINTER_FIELD));
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/utils/VTreeNavigationFrame.java b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/utils/VTreeNavigationFrame.java
new file mode 100644
index 0000000000..5c7c3c3823
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/utils/VTreeNavigationFrame.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.vector.utils;
+
+import java.util.List;
+
+import org.apache.hyracks.storage.am.vector.impls.ClusterSearchResult;
+
+/**
+ * One stack frame for the DFS cluster iterator. A frame represents either an interior page
+ * (carrying its children sorted by distance to the query) or a leaf page (carrying its centroids).
+ * An internal cursor tracks the next entry to emit; advance it through {@link #nextChild()} /
+ * {@link #nextCentroid()}. Build frames with {@link #newInteriorFrame} / {@link #newLeafFrame}.
+ * <p>
+ * Not thread-safe: the cursor is mutated during iteration; a frame is confined to a single
+ * search / operation context.
+ */
+public class VTreeNavigationFrame {
+
+ private final int pageId;
+ private final boolean isLeaf;
+ /** Children sorted by distance to query; non-null iff interior. */
+ private final List<VTreeChildCentroid> sortedChildren;
+ /** Leaf centroids sorted by distance to query; non-null iff leaf. */
+ private final List<ClusterSearchResult> sortedCentroids;
+ /** Index of the next child / centroid to emit. */
+ private int nextIndex;
+
+ private VTreeNavigationFrame(int pageId, boolean isLeaf, List<VTreeChildCentroid> sortedChildren,
+ List<ClusterSearchResult> sortedCentroids) {
+ this.pageId = pageId;
+ this.isLeaf = isLeaf;
+ this.sortedChildren = sortedChildren;
+ this.sortedCentroids = sortedCentroids;
+ this.nextIndex = 0;
+ }
+
+ /** Frame over an interior page's distance-sorted children. */
+ public static VTreeNavigationFrame newInteriorFrame(int pageId, List<VTreeChildCentroid> sortedChildren) {
+ return new VTreeNavigationFrame(pageId, false, sortedChildren, null);
+ }
+
+ /** Frame over a leaf page's distance-sorted centroids. */
+ public static VTreeNavigationFrame newLeafFrame(int pageId, List<ClusterSearchResult> sortedCentroids) {
+ return new VTreeNavigationFrame(pageId, true, null, sortedCentroids);
+ }
+
+ public int pageId() {
+ return pageId;
+ }
+
+ public boolean isLeaf() {
+ return isLeaf;
+ }
+
+ /** Number of entries already emitted from this frame (the current cursor position). */
+ public int emittedCount() {
+ return nextIndex;
+ }
+
+ /** Total centroids on this (leaf) frame. */
+ public int centroidCount() {
+ return sortedCentroids.size();
+ }
+
+ public boolean hasNextChild() {
+ return sortedChildren != null && nextIndex < sortedChildren.size();
+ }
+
+ public boolean hasNextCentroid() {
+ return sortedCentroids != null && nextIndex < sortedCentroids.size();
+ }
+
+ public VTreeChildCentroid nextChild() {
+ if (hasNextChild()) {
+ return sortedChildren.get(nextIndex++);
+ }
+ return null;
+ }
+
+ public ClusterSearchResult nextCentroid() {
+ if (hasNextCentroid()) {
+ return sortedCentroids.get(nextIndex++);
+ }
+ return null;
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/utils/VTreeNavigationState.java b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/utils/VTreeNavigationState.java
new file mode 100644
index 0000000000..48ab6b8e0c
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/utils/VTreeNavigationState.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.vector.utils;
+
+import java.util.ArrayDeque;
+import java.util.Deque;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+
+/**
+ * Mutable navigation state for the DFS cluster iterator: holds the buffer cache / frame factories
+ * needed to pin pages, the navigation stack of {@link VTreeNavigationFrame}s, and a (possibly
+ * shared) set of visited centroid ids used to deduplicate when level-wise selection and DFS
+ * fallback are combined.
+ * <p>
+ * Not thread-safe: an instance is confined to a single search / operation context and must not be
+ * shared concurrently across threads.
+ */
+public class VTreeNavigationState {
+
+ public final IBufferCache bufferCache;
+ public final int fileId;
+ public final int rootPageId;
+ public final ITreeIndexFrameFactory interiorFrameFactory;
+ public final ITreeIndexFrameFactory leafFrameFactory;
+ public final double[] queryVector;
+ public final Deque<VTreeNavigationFrame> stack;
+ public boolean initialized;
+
+ /** Visited centroid ids; may be shared across LSM components for cross-component dedup. */
+ private Set<Integer> visitedCentroidIds;
+
+ public VTreeNavigationState(IBufferCache bufferCache, int fileId, int rootPageId,
+ ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory leafFrameFactory,
+ double[] queryVector) {
+ this(bufferCache, fileId, rootPageId, interiorFrameFactory, leafFrameFactory, queryVector, new HashSet<>());
+ }
+
+ /**
+ * Constructor accepting a shared visited-id set. Use this when the same dedup set must be
+ * observed across multiple LSM components. A {@code null} {@code sharedVisitedSet} falls back
+ * to a fresh local set.
+ */
+ public VTreeNavigationState(IBufferCache bufferCache, int fileId, int rootPageId,
+ ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory leafFrameFactory, double[] queryVector,
+ Set<Integer> sharedVisitedSet) {
+ this.bufferCache = bufferCache;
+ this.fileId = fileId;
+ this.rootPageId = rootPageId;
+ this.interiorFrameFactory = interiorFrameFactory;
+ this.leafFrameFactory = leafFrameFactory;
+ this.queryVector = queryVector;
+ this.stack = new ArrayDeque<>();
+ this.initialized = false;
+ this.visitedCentroidIds = sharedVisitedSet != null ? sharedVisitedSet : new HashSet<>();
+ }
+
+ public boolean isVisited(int centroidId) {
+ return visitedCentroidIds.contains(centroidId);
+ }
+
+ public void markVisited(int centroidId) {
+ visitedCentroidIds.add(centroidId);
+ }
+
+ /** Swap in an external visited set (e.g. to share it across LSM components). */
+ public void setVisitedSet(Set<Integer> visitedSet) {
+ this.visitedCentroidIds = visitedSet;
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/utils/VTreeNavigationUtils.java b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/utils/VTreeNavigationUtils.java
new file mode 100644
index 0000000000..ba2511d867
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/utils/VTreeNavigationUtils.java
@@ -0,0 +1,715 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.vector.utils;
+
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Queue;
+import java.util.Set;
+
+import org.apache.hyracks.api.exceptions.ErrorCode;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.dataflow.common.data.marshalling.DoubleArraySerializerDeserializer;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleReference;
+import org.apache.hyracks.storage.am.vector.api.IVTreeDistanceFunction;
+import org.apache.hyracks.storage.am.vector.api.IVTreeInteriorFrame;
+import org.apache.hyracks.storage.am.vector.api.IVTreeLeafFrame;
+import org.apache.hyracks.storage.am.vector.api.IVTreeQuantizer;
+import org.apache.hyracks.storage.am.vector.api.VTreeStaticTupleConstants;
+import org.apache.hyracks.storage.am.vector.impls.ClusterSearchResult;
+import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.buffercache.ICachedPage;
+import org.apache.hyracks.storage.common.file.BufferedFileHandle;
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+/**
+ * Static helpers for navigating a VTree. Provides:
+ * <ul>
+ * <li>{@link #findClosestCentroid} - root-to-leaf descent picking the single closest centroid.</li>
+ * <li>{@link #initializeClusterIterator} / {@link #findNextClosestCluster} - iterative DFS
+ * over leaf centroids in non-decreasing distance order with visited-id dedup.</li>
+ * <li>{@link #findCloseCentroidsLevelWiseGlobalSort} - level-wise probing with an epsilon
+ * window followed by a global sort and threshold filter at the leaf layer.</li>
+ * </ul>
+ * All methods pin/unpin buffer-cache pages internally; callers must not hold latches on the
+ * traversed pages. Stateless and thread-safe.
+ */
+public class VTreeNavigationUtils {
+
+ private static final Logger LOGGER = LogManager.getLogger();
+
+ /**
+ * Maximum number of root-to-leaf hops in {@link #findClosestCentroid}; serves as a safety net
+ * against malformed trees / cyclic child pointers. The real VTree is shallow (~3-5 levels).
+ */
+ private static final int MAX_TREE_DEPTH = 10;
+
+ /**
+ * Find the closest centroid by traversing the tree from root to leaf,
+ * optionally computing a quantized distance for the best result.
+ *
+ * @param bufferCache Buffer cache for page access
+ * @param fileId File ID for page identification
+ * @param rootPageId Root page ID to start traversal
+ * @param interiorFrameFactory Factory for creating interior frames
+ * @param leafFrameFactory Factory for creating leaf frames
+ * @param queryVector Query vector to find closest centroid for
+ * @param distanceFunction Distance function to use for centroid finding
+ * @param quantizedQueryVector Query vector in dequantized/reconstructed space — i.e. quantized then
+ * dequantized so it lives in the same space as the reconstructed leaf centroids it is compared against
+ * (nullable — pass null to skip quantized distance)
+ * @param quantizer Quantizer for dequantizing leaf centroid bytes (nullable — pass null to skip)
+ * @return ClusterSearchResult containing closest centroid information (with quantizedDistance if quantizer provided)
+ * @throws HyracksDataException if any error occurs during traversal
+ */
+ public static ClusterSearchResult findClosestCentroid(IBufferCache bufferCache, int fileId, int rootPageId,
+ ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory leafFrameFactory, double[] queryVector,
+ IVTreeDistanceFunction distanceFunction, double[] quantizedQueryVector, IVTreeQuantizer quantizer)
+ throws HyracksDataException {
+
+ int currentPageId = rootPageId;
+ int hops = 0;
+
+ while (true) {
+ if (++hops > MAX_TREE_DEPTH) {
+ throw HyracksDataException.create(ErrorCode.ILLEGAL_STATE, "Infinite loop detected in tree traversal");
+ }
+
+ PageScan scan = scanPage(bufferCache, fileId, currentPageId, queryVector, interiorFrameFactory,
+ leafFrameFactory, distanceFunction, quantizedQueryVector, quantizer);
+
+ if (scan.isLeaf()) {
+ // Leaf level - the closest centroid is the sort's first entry
+ if (scan.leafCentroids.isEmpty()) {
+ throw HyracksDataException.create(ErrorCode.ILLEGAL_STATE, "No closest cluster found");
+ }
+ return scan.leafCentroids.get(0);
+ }
+
+ // Interior level - descend into the closest child
+ if (scan.children.isEmpty()) {
+ throw HyracksDataException.create(ErrorCode.ILLEGAL_STATE,
+ "No valid centroid found in interior cluster");
+ }
+ currentPageId = scan.children.get(0).childPageId();
+ }
+ }
+
+ /**
+ * Extract the full-precision centroid embedding from a static-structure tuple. The embedding sits at
+ * {@link VTreeStaticTupleConstants#EMBEDDING_FIELD} in every interior and leaf layout, so only that
+ * field is decoded; the surrounding fields (centroid id, child/metadata pointer, optional quantized
+ * bytes / neighbor list) are left untouched.
+ */
+ private static double[] extractCentroid(ITreeIndexTupleReference tuple) throws HyracksDataException {
+ int field = VTreeStaticTupleConstants.EMBEDDING_FIELD;
+ DataInputStream dis = new DataInputStream(new ByteArrayInputStream(tuple.getFieldData(field),
+ tuple.getFieldStart(field), tuple.getFieldLength(field)));
+ return DoubleArraySerializerDeserializer.INSTANCE.deserialize(dis);
+ }
+
+ /**
+ * Collects every child of an interior page (including its overflow chain) and returns them
+ * sorted by distance to {@code queryVector}, closest first. {@code initialFrame} must already
+ * be set to {@code startPageId} and that page must be pinned/latched by the caller; overflow
+ * pages are pinned and released internally. Malformed tuples are skipped.
+ */
+ private static List<VTreeChildCentroid> collectAllChildCentroids(IBufferCache bufferCache, int fileId,
+ double[] queryVector, int startPageId, IVTreeInteriorFrame initialFrame,
+ ITreeIndexFrameFactory interiorFrameFactory, IVTreeDistanceFunction distanceFunction)
+ throws HyracksDataException {
+
+ List<VTreeChildCentroid> children = new ArrayList<>();
+ int currentPageId = startPageId;
+ IVTreeInteriorFrame currentFrame = initialFrame;
+ boolean isFirstPage = true;
+ ICachedPage currentPage = null;
+
+ while (currentPageId != -1) {
+ try {
+ if (!isFirstPage) {
+ currentPage = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, currentPageId));
+ currentPage.acquireReadLatch();
+ currentFrame = (IVTreeInteriorFrame) interiorFrameFactory.createFrame();
+ currentFrame.setPage(currentPage);
+ }
+
+ int tupleCount = currentFrame.getTupleCount();
+ boolean hasOverflow = currentFrame.getOverflowFlagBit();
+ int nextPageId = hasOverflow ? currentFrame.getNextPage() : -1;
+
+ for (int i = 0; i < tupleCount; i++) {
+ try {
+ ITreeIndexTupleReference tuple = currentFrame.createTupleReference();
+ tuple.resetByTupleIndex(currentFrame, i);
+ double[] centroid = extractCentroid(tuple);
+
+ if (centroid.length != queryVector.length) {
+ continue;
+ }
+
+ double distance = distanceFunction.apply(queryVector, centroid);
+ int childPageId = currentFrame.getChildPageId(i);
+ children.add(new VTreeChildCentroid(childPageId, distance, i));
+ } catch (HyracksDataException e) {
+ // Skip only genuinely malformed interior tuples (decode/accessor failures);
+ // unexpected runtime failures (NPE, contract violations) propagate so silent
+ // recall loss is surfaced. Logged at WARN with page id + tuple index.
+ LOGGER.log(Level.WARN, "Skipping malformed interior tuple {} on page {}: {}", i, currentPageId,
+ e.getMessage());
+ }
+ }
+
+ currentPageId = nextPageId;
+ isFirstPage = false;
+
+ } finally {
+ if (!isFirstPage && currentPage != null) {
+ currentPage.releaseReadLatch();
+ bufferCache.unpin(currentPage);
+ currentPage = null;
+ }
+ }
+ }
+
+ children.sort(Comparator.comparingDouble(VTreeChildCentroid::distance));
+
+ return children;
+ }
+
+ /**
+ * Collects every centroid of a leaf page (including its overflow chain) and returns them
+ * sorted by distance to {@code queryVector}, closest first. {@code initialFrame} must already
+ * be set to {@code startPageId} and that page must be pinned/latched by the caller; overflow
+ * pages are pinned and released internally.
+ * <p>
+ * When both {@code quantizer} and {@code quantizedQueryVector} are non-null, the quantized
+ * distance D(q̃, C̃) is computed for each centroid and placed in the resulting
+ * {@link ClusterSearchResult#quantizedDistance}; otherwise that field is {@code NaN}.
+ * Malformed tuples are skipped.
+ */
+ private static List<ClusterSearchResult> collectAllLeafCentroids(IBufferCache bufferCache, int fileId,
+ double[] queryVector, int startPageId, IVTreeLeafFrame initialFrame,
+ ITreeIndexFrameFactory leafFrameFactory, IVTreeDistanceFunction distanceFunction,
+ double[] quantizedQueryVector, IVTreeQuantizer quantizer) throws HyracksDataException {
+
+ List<ClusterSearchResult> centroids = new ArrayList<>();
+ int currentPageId = startPageId;
+ IVTreeLeafFrame currentFrame = initialFrame;
+ boolean isFirstPage = true;
+ ICachedPage currentPage = null;
+
+ while (currentPageId != -1) {
+ try {
+ if (!isFirstPage) {
+ currentPage = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, currentPageId));
+ currentPage.acquireReadLatch();
+ currentFrame = (IVTreeLeafFrame) leafFrameFactory.createFrame();
+ currentFrame.setPage(currentPage);
+ }
+
+ int tupleCount = currentFrame.getTupleCount();
+ boolean hasOverflow = currentFrame.getOverflowFlagBit();
+ int nextPageId = hasOverflow ? currentFrame.getNextLeaf() : -1;
+
+ for (int i = 0; i < tupleCount; i++) {
+ try {
+ ITreeIndexTupleReference frameTuple = currentFrame.createTupleReference();
+ frameTuple.resetByTupleIndex(currentFrame, i);
+ double[] centroid = extractCentroid(frameTuple);
+ int centroidId = currentFrame.getCentroidId(i);
+ long directoryPageId = currentFrame.getMetadataPagePointer(i);
+
+ if (centroid.length != queryVector.length) {
+ continue;
+ }
+
+ double distance = distanceFunction.apply(queryVector, centroid);
+ double quantizedDistance = quantizedDistanceOrNaN(currentFrame, i, quantizedQueryVector,
+ quantizer, distanceFunction);
+
+ centroids.add(ClusterSearchResult.create(currentPageId, i, centroid.clone(), distance,
+ centroidId, directoryPageId, quantizedDistance));
+ } catch (HyracksDataException e) {
+ // Skip only genuinely malformed leaf tuples (decode/accessor failures);
+ // unexpected runtime failures (NPE, contract violations) propagate so silent
+ // recall loss is surfaced. Logged at WARN with page id + tuple index.
+ LOGGER.log(Level.WARN, "Skipping malformed leaf tuple {} on page {}: {}", i, currentPageId,
+ e.getMessage());
+ }
+ }
+
+ currentPageId = nextPageId;
+ isFirstPage = false;
+
+ } finally {
+ if (!isFirstPage && currentPage != null) {
+ currentPage.releaseReadLatch();
+ bufferCache.unpin(currentPage);
+ currentPage = null;
+ }
+ }
+ }
+
+ centroids.sort(Comparator.comparingDouble(c -> c.distance));
+ return centroids;
+ }
+
+ /**
+ * Quantized distance D(q̃, C̃) for the leaf centroid at {@code tupleIndex}: dequantize its stored
+ * quantized bytes and compare against {@code quantizedQueryVector} (the query already reconstructed
+ * into the same dequantized space). Returns {@code Double.NaN} when not in quantized mode
+ * ({@code quantizer}/{@code quantizedQueryVector} null) or the centroid carries no quantized bytes —
+ * the sentinel {@link ClusterSearchResult#quantizedDistance} readers treat as "not computed".
+ */
+ private static double quantizedDistanceOrNaN(IVTreeLeafFrame frame, int tupleIndex, double[] quantizedQueryVector,
+ IVTreeQuantizer quantizer, IVTreeDistanceFunction distanceFunction) throws HyracksDataException {
+ if (quantizer == null || quantizedQueryVector == null) {
+ return Double.NaN;
+ }
+ byte[] quantizedCentroidBytes = frame.getQuantizedCentroidBytes(tupleIndex);
+ if (quantizedCentroidBytes == null) {
+ return Double.NaN;
+ }
+ double[] dequantizedCentroid = quantizer.dequantize(quantizedCentroidBytes);
+ return distanceFunction.apply(quantizedQueryVector, dequantizedCentroid);
+ }
+
+ /**
+ * Result of scanning one static-structure page under a single read latch. A leaf page yields its
+ * centroids (distance-sorted, closest first) in {@link #leafCentroids}; an interior page yields its
+ * children (distance-sorted) in {@link #children}. Exactly one of the two is non-null — use
+ * {@link #isLeaf()} to discriminate.
+ */
+ private static final class PageScan {
+ private final List<ClusterSearchResult> leafCentroids;
+ private final List<VTreeChildCentroid> children;
+
+ // One private canonical constructor; the two factories below name leaf-vs-interior explicitly.
+ // (Two single-arg constructors would collide under generic erasure — both erase to PageScan(List).)
+ private PageScan(List<ClusterSearchResult> leafCentroids, List<VTreeChildCentroid> children) {
+ this.leafCentroids = leafCentroids;
+ this.children = children;
+ }
+
+ /** Leaf page: carries its distance-sorted centroids. */
+ private static PageScan forLeaf(List<ClusterSearchResult> leafCentroids) {
+ return new PageScan(leafCentroids, null);
+ }
+
+ /** Interior page: carries its distance-sorted children. */
+ private static PageScan forInterior(List<VTreeChildCentroid> children) {
+ return new PageScan(null, children);
+ }
+
+ private boolean isLeaf() {
+ return leafCentroids != null;
+ }
+ }
+
+ /**
+ * Pin {@code pageId}, read-latch it once, and collect its entries: for a leaf page its centroids, for
+ * an interior page its children — each distance-sorted, closest first, including the page's overflow
+ * chain. Determining leaf-vs-interior and collecting happen under the same latch (one pin per hop), and
+ * the page (plus any overflow pages pinned by the collectors) is released before returning. Shared by
+ * the three root-to-leaf descent walkers ({@link #findClosestCentroid}, {@link #initializeClusterIterator},
+ * {@link #descendToLeaf}).
+ */
+ private static PageScan scanPage(IBufferCache bufferCache, int fileId, int pageId, double[] queryVector,
+ ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory leafFrameFactory,
+ IVTreeDistanceFunction distanceFunction, double[] quantizedQueryVector, IVTreeQuantizer quantizer)
+ throws HyracksDataException {
+ ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId));
+ try {
+ page.acquireReadLatch();
+ // isLeaf() reads the level byte from the shared header, so a leaf frame can classify any page.
+ IVTreeLeafFrame leafFrame = (IVTreeLeafFrame) leafFrameFactory.createFrame();
+ leafFrame.setPage(page);
+ if (leafFrame.isLeaf()) {
+ return PageScan.forLeaf(collectAllLeafCentroids(bufferCache, fileId, queryVector, pageId, leafFrame,
+ leafFrameFactory, distanceFunction, quantizedQueryVector, quantizer));
+ }
+ IVTreeInteriorFrame interiorFrame = (IVTreeInteriorFrame) interiorFrameFactory.createFrame();
+ interiorFrame.setPage(page);
+ return PageScan.forInterior(collectAllChildCentroids(bufferCache, fileId, queryVector, pageId,
+ interiorFrame, interiorFrameFactory, distanceFunction));
+ } finally {
+ page.releaseReadLatch();
+ bufferCache.unpin(page);
+ }
+ }
+
+ // ==================== Multi-Cluster Iterative DFS Support ====================
+
+ /**
+ * Initialize the cluster iterator by building navigation stack from root to first leaf.
+ * This performs DFS to find the closest cluster and sets up the stack for backtracking.
+ *
+ * @param state Navigation state to initialize
+ * @return The first (closest) cluster, or null if tree is empty
+ * @throws HyracksDataException if any error occurs
+ */
+ public static ClusterSearchResult initializeClusterIterator(VTreeNavigationState state,
+ IVTreeDistanceFunction distanceFunction) throws HyracksDataException {
+ if (state.initialized) {
+ throw HyracksDataException.create(ErrorCode.ILLEGAL_STATE, "Iterator already initialized");
+ }
+
+ state.stack.clear();
+ state.initialized = true;
+
+ // Start DFS from root
+ int currentPageId = state.rootPageId;
+
+ while (true) {
+ PageScan scan = scanPage(state.bufferCache, state.fileId, currentPageId, state.queryVector,
+ state.interiorFrameFactory, state.leafFrameFactory, distanceFunction, null, null);
+
+ if (scan.isLeaf()) {
+ if (scan.leafCentroids.isEmpty()) {
+ return null; // Empty tree
+ }
+ // Push leaf frame; return its first centroid as the closest cluster, marking it visited
+ VTreeNavigationFrame leafNavFrame =
+ VTreeNavigationFrame.newLeafFrame(currentPageId, scan.leafCentroids);
+ state.stack.push(leafNavFrame);
+ ClusterSearchResult first = leafNavFrame.nextCentroid();
+ state.markVisited(first.centroidId);
+ return first;
+ }
+
+ // Interior level: push the frame and descend into the closest child
+ if (scan.children.isEmpty()) {
+ throw HyracksDataException.create(ErrorCode.ILLEGAL_STATE, "Interior node has no valid children");
+ }
+ VTreeNavigationFrame interiorNavFrame = VTreeNavigationFrame.newInteriorFrame(currentPageId, scan.children);
+ state.stack.push(interiorNavFrame);
+ currentPageId = interiorNavFrame.nextChild().childPageId();
+ }
+ }
+
+ /**
+ * Find the next closest cluster using DFS with backtracking.
+ *
+ * Algorithm:
+ * 1. Try next centroid on current leaf page
+ * 2. If leaf exhausted, pop stack (backtrack to parent)
+ * 3. Try next child from parent
+ * 4. Descend to new leaf
+ * 5. Return next centroid
+ *
+ * @param state Navigation state with stack
+ * @return Next closest cluster, or null if all clusters exhausted
+ * @throws HyracksDataException if any error occurs
+ */
+ public static ClusterSearchResult findNextClosestCluster(VTreeNavigationState state,
+ IVTreeDistanceFunction distanceFunction) throws HyracksDataException {
+ if (!state.initialized) {
+ throw HyracksDataException.create(ErrorCode.ILLEGAL_STATE,
+ "Iterator not initialized. Call initializeClusterIterator() first");
+ }
+
+ while (!state.stack.isEmpty()) {
+ VTreeNavigationFrame topFrame = state.stack.peek();
+
+ if (topFrame.isLeaf()) {
+ // At leaf level: try next centroid in current page, skipping visited ones
+ while (topFrame.hasNextCentroid()) {
+ ClusterSearchResult next = topFrame.nextCentroid();
+
+ if (state.isVisited(next.centroidId)) {
+ LOGGER.log(Level.TRACE, "[DFS] skipping visited cid={} d={}", next.centroidId, next.distance);
+ continue;
+ }
+
+ state.markVisited(next.centroidId);
+ LOGGER.log(Level.TRACE, "[DFS] return cid={} d={} pageId={} idx={}/{}", next.centroidId,
+ next.distance, topFrame.pageId(), topFrame.emittedCount(), topFrame.centroidCount());
+ return next;
+ }
+
+ // All centroids in this leaf exhausted or visited, backtrack
+ state.stack.pop();
+ continue;
+
+ } else {
+ // At interior level: try next child
+ if (topFrame.hasNextChild()) {
+ VTreeChildCentroid nextChild = topFrame.nextChild();
+
+ // Descend to this child and navigate to leaf
+ ClusterSearchResult result = descendToLeaf(state, nextChild.childPageId(), distanceFunction);
+
+ if (result != null) {
+ return result;
+ }
+ // If descend failed, continue with next child
+ continue;
+
+ } else {
+ // All children explored, backtrack
+ state.stack.pop();
+ continue;
+ }
+ }
+ }
+
+ LOGGER.log(Level.TRACE, "[DFS] stack exhausted");
+ return null;
+ }
+
+ /**
+ * Descend from given page to leaf level, building stack along the way.
+ * Always picks closest child at each interior level.
+ *
+ * @param state Navigation state
+ * @param startPageId Page to start descent from
+ * @return First centroid at leaf level, or null if no valid path
+ * @throws HyracksDataException if any error occurs
+ */
+ private static ClusterSearchResult descendToLeaf(VTreeNavigationState state, int startPageId,
+ IVTreeDistanceFunction distanceFunction) throws HyracksDataException {
+
+ int currentPageId = startPageId;
+
+ while (true) {
+ PageScan scan = scanPage(state.bufferCache, state.fileId, currentPageId, state.queryVector,
+ state.interiorFrameFactory, state.leafFrameFactory, distanceFunction, null, null);
+
+ if (scan.isLeaf()) {
+ if (scan.leafCentroids.isEmpty()) {
+ return null; // Empty leaf
+ }
+ VTreeNavigationFrame leafNavFrame =
+ VTreeNavigationFrame.newLeafFrame(currentPageId, scan.leafCentroids);
+ state.stack.push(leafNavFrame);
+
+ // Return the first unvisited centroid on this leaf
+ while (leafNavFrame.hasNextCentroid()) {
+ ClusterSearchResult first = leafNavFrame.nextCentroid();
+ if (!state.isVisited(first.centroidId)) {
+ state.markVisited(first.centroidId);
+ return first;
+ }
+ LOGGER.log(Level.TRACE, "[DFS descendToLeaf] skipping visited cid={}", first.centroidId);
+ }
+ return null;
+ }
+
+ // Interior: push the frame and descend into the closest child
+ if (scan.children.isEmpty()) {
+ return null; // No valid children
+ }
+ VTreeNavigationFrame interiorNavFrame = VTreeNavigationFrame.newInteriorFrame(currentPageId, scan.children);
+ state.stack.push(interiorNavFrame);
+ currentPageId = interiorNavFrame.nextChild().childPageId();
+ }
+ }
+
+ // ==================== Level-Wise Cluster Selection Support ====================
+
+ /**
+ * Multiplicative-epsilon distance threshold relative to {@code |closestDistance|}: yields
+ * {@code (1+epsilon)*d} for positive {@code d} and {@code (1-epsilon)*d} for negative {@code d}
+ * (negated dot product, where smaller is better). An additive {@code d+epsilon} form is a near
+ * no-op for euclidean_squared / high-dim L2 where distances are O(10)-O(10^3), which collapses the
+ * search to ~1 cluster regardless of nprobe/epsilon.
+ */
+ private static double epsilonThreshold(double closestDistance, double epsilon) {
+ return closestDistance + Math.abs(closestDistance) * epsilon;
+ }
+
+ /**
+ * Find close centroids using level-by-level cross-pollination with global sorting.
+ * At each interior node, explores all children within closestDistance * (1 + epsilon)
+ * (for positive distances) / closestDistance * (1 - epsilon) (for negative distances
+ * such as negated dot product).
+ * At leaf level, collects ALL centroids, then sorts globally and filters by epsilon.
+ * <p>
+ * 1. Traverse tree using epsilon threshold at interior levels
+ * 2. Collect ALL reachable leaf centroids
+ * 3. Sort globally by distance to query
+ * 4. Filter by global closest distance + epsilon
+ *
+ * @param bufferCache Buffer cache for page access
+ * @param fileId File ID for page identification
+ * @param rootPageId Root page ID to start traversal
+ * @param interiorFrameFactory Factory for creating interior frames
+ * @param leafFrameFactory Factory for creating leaf frames
+ * @param queryVector Query vector to find closest centroids for
+ * @param distanceFunction Distance function to use
+ * @param epsilon Relative distance threshold (multiplicative). Threshold for level/global
+ * pruning is computed as {@code closestDistance + |closestDistance| * epsilon},
+ * i.e. (1+epsilon)*d for positive d and (1-epsilon)*d for negative d.
+ * @return List of ClusterSearchResult containing all qualifying centroids, sorted by distance
+ * @throws HyracksDataException if any error occurs during traversal
+ */
+ public static List<ClusterSearchResult> findCloseCentroidsLevelWiseGlobalSort(IBufferCache bufferCache, int fileId,
+ int rootPageId, ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory leafFrameFactory,
+ double[] queryVector, IVTreeDistanceFunction distanceFunction, double epsilon) throws HyracksDataException {
+ return findCloseCentroidsLevelWiseGlobalSort(bufferCache, fileId, rootPageId, interiorFrameFactory,
+ leafFrameFactory, queryVector, distanceFunction, epsilon, null, null);
+ }
+
+ /**
+ * Overload that accepts quantizer parameters for computing quantized D(q,C).
+ * Navigation still uses full-precision distances; quantizedDistance is extra metadata
+ * populated in each ClusterSearchResult for triangle inequality pruning at the cursor level.
+ *
+ * @param quantizedQueryVector Query vector in dequantized/reconstructed space — quantized then
+ * dequantized so it lives in the same space as the reconstructed leaf centroids (nullable)
+ * @param quantizer Quantizer for dequantizing leaf centroid bytes (nullable)
+ */
+ public static List<ClusterSearchResult> findCloseCentroidsLevelWiseGlobalSort(IBufferCache bufferCache, int fileId,
+ int rootPageId, ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory leafFrameFactory,
+ double[] queryVector, IVTreeDistanceFunction distanceFunction, double epsilon,
+ double[] quantizedQueryVector, IVTreeQuantizer quantizer) throws HyracksDataException {
+
+ List<ClusterSearchResult> allCentroids = new ArrayList<>();
+ Set<Integer> visitedLeafPages = new HashSet<>();
+ Queue<VTreeLevelNode> queue = new ArrayDeque<>();
+ queue.add(new VTreeLevelNode(rootPageId, 0));
+
+ // Phase 1: Collect all centroids from all reachable leaf pages
+ while (!queue.isEmpty()) {
+ int currentLevel = queue.peek().level();
+
+ List<VTreeLevelNode> currentLevelNodes = new ArrayList<>();
+
+ // Collect all nodes at current level
+ while (!queue.isEmpty() && queue.peek().level() == currentLevel) {
+
+ currentLevelNodes.add(queue.poll());
+ }
+
+ // Process all nodes at current level
+ for (VTreeLevelNode node : currentLevelNodes) {
+ ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, node.pageId()));
+
+ try {
+ page.acquireReadLatch();
+
+ IVTreeLeafFrame leafFrame = (IVTreeLeafFrame) leafFrameFactory.createFrame();
+ leafFrame.setPage(page);
+ boolean isLeaf = leafFrame.isLeaf();
+
+ if (isLeaf) {
+ // Leaf node processing - collect ALL centroids (no threshold filtering yet)
+ if (!visitedLeafPages.add(node.pageId())) {
+
+ continue; // Already visited
+ }
+
+ List<ClusterSearchResult> leafCentroids =
+ collectAllLeafCentroids(bufferCache, fileId, queryVector, node.pageId(), leafFrame,
+
+ leafFrameFactory, distanceFunction, quantizedQueryVector, quantizer);
+
+ if (leafCentroids.isEmpty()) {
+ continue;
+ }
+
+ // Add ALL centroids from this leaf page to global collection
+ allCentroids.addAll(leafCentroids);
+
+ } else {
+ // Interior node processing - explore children within epsilon
+ IVTreeInteriorFrame interiorFrame = (IVTreeInteriorFrame) interiorFrameFactory.createFrame();
+ interiorFrame.setPage(page);
+ queue.addAll(childrenWithinEpsilon(bufferCache, fileId, queryVector, node, interiorFrame,
+ interiorFrameFactory, distanceFunction, epsilon));
+ }
+
+ } finally {
+ page.releaseReadLatch();
+ bufferCache.unpin(page);
+ }
+ }
+ }
+
+ if (allCentroids.isEmpty()) {
+ throw HyracksDataException.create(ErrorCode.ILLEGAL_STATE, "No closest clusters found");
+ }
+
+ // Phase 2: Sort ALL centroids globally by distance to query vector
+ allCentroids.sort(Comparator.comparingDouble(r -> r.distance));
+
+ // Phase 3: Apply epsilon threshold based on the globally closest centroid
+ return applyGlobalEpsilonFilter(allCentroids, epsilon);
+ }
+
+ /**
+ * Interior-node frontier expansion for the level-wise BFS: collect this interior page's children
+ * (closest first) and return the next-level nodes whose distance is within the epsilon window of the
+ * closest child. {@code interiorFrame} must be set to {@code node}'s page and that page must be
+ * pinned/latched by the caller (collectAllChildCentroids pins only the overflow chain). Empty if the
+ * page has no valid children.
+ */
+ private static List<VTreeLevelNode> childrenWithinEpsilon(IBufferCache bufferCache, int fileId,
+ double[] queryVector, VTreeLevelNode node, IVTreeInteriorFrame interiorFrame,
+ ITreeIndexFrameFactory interiorFrameFactory, IVTreeDistanceFunction distanceFunction, double epsilon)
+ throws HyracksDataException {
+ List<VTreeChildCentroid> sortedChildren = collectAllChildCentroids(bufferCache, fileId, queryVector,
+ node.pageId(), interiorFrame, interiorFrameFactory, distanceFunction);
+ if (sortedChildren.isEmpty()) {
+ return List.of();
+ }
+ double localThreshold = epsilonThreshold(sortedChildren.get(0).distance(), epsilon);
+ List<VTreeLevelNode> nextLevel = new ArrayList<>();
+ for (VTreeChildCentroid child : sortedChildren) {
+ if (child.distance() <= localThreshold) {
+ nextLevel.add(new VTreeLevelNode(child.childPageId(), node.level() + 1));
+ } else {
+ break; // Children are sorted, no more qualify
+ }
+ }
+ return nextLevel;
+ }
+
+ /**
+ * Phase 3 of level-wise search: given all collected centroids already sorted by distance ascending,
+ * keep only those within the epsilon window of the globally closest centroid. Returns the input list
+ * unchanged when {@code epsilon <= 0.0} (no filtering). Pure list operation — no page access.
+ */
+ private static List<ClusterSearchResult> applyGlobalEpsilonFilter(List<ClusterSearchResult> sortedCentroids,
+ double epsilon) {
+ if (epsilon <= 0.0) {
+ return sortedCentroids;
+ }
+ double globalThreshold = epsilonThreshold(sortedCentroids.get(0).distance, epsilon);
+ List<ClusterSearchResult> filteredCentroids = new ArrayList<>();
+ for (ClusterSearchResult result : sortedCentroids) {
+ if (result.distance <= globalThreshold) {
+ filteredCentroids.add(result);
+ } else {
+ break; // Centroids are sorted, so we can break early
+ }
+ }
+ return filteredCentroids;
+ }
+
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/utils/VTreeStaticTupleAccessor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/utils/VTreeStaticTupleAccessor.java
new file mode 100644
index 0000000000..b036b92a34
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/utils/VTreeStaticTupleAccessor.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.vector.utils;
+
+import org.apache.hyracks.api.dataflow.value.ITypeTraits;
+import org.apache.hyracks.data.std.primitive.IntegerPointable;
+import org.apache.hyracks.data.std.primitive.VarLengthTypeTrait;
+import org.apache.hyracks.storage.am.vector.api.VTreeStaticTupleConstants;
+
+/**
+ * Single authority for the VTree STATIC-structure tuple layout, mirroring {@code VTreeDataTupleAccessor}
+ * for the data tuple. It builds the per-field {@link ITypeTraits} schemas <em>positionally from
+ * {@link VTreeStaticTupleConstants}</em>, so the frame factories no longer hand-maintain (and hand-keep
+ * in sync) their own {@code ITypeTraits[]} arrays.
+ * <p>
+ * The pointer field — a child-page pointer for interior tuples, a metadata-page pointer for leaf tuples —
+ * is always the LAST field:
+ * <pre>
+ * Interior / non-quantized leaf: [ cid, centroid, pointer ]
+ * Quantized leaf: [ cid, centroid, quantizedBytes, neighborList, pointer ]
+ * </pre>
+ */
+public final class VTreeStaticTupleAccessor {
+
+ // Schemas are fixed per layout, so build them once. The returned arrays are treated as read-only by
+ // callers (frame factories hand them to a tuple-writer factory and never mutate them).
+ private static final ITypeTraits[] BASE_SCHEMA = buildBaseSchema();
+ private static final ITypeTraits[] LEAF_QUANTIZED_SCHEMA = buildLeafQuantizedSchema();
+
+ private VTreeStaticTupleAccessor() {
+ }
+
+ /** Interior tuple schema: {@code [cid, centroid, childPagePointer]}. */
+ public static ITypeTraits[] interiorTypeTraits() {
+ return BASE_SCHEMA;
+ }
+
+ /**
+ * Leaf tuple schema. A quantized leaf inserts the {@code quantizedBytes} and {@code neighborList}
+ * fields before the trailing pointer; a non-quantized leaf has the same shape as an interior tuple.
+ */
+ public static ITypeTraits[] leafTypeTraits(boolean quantized) {
+ return quantized ? LEAF_QUANTIZED_SCHEMA : BASE_SCHEMA;
+ }
+
+ /** {@code [cid, centroid, pointer]} — shared by interior and non-quantized leaf tuples. */
+ private static ITypeTraits[] buildBaseSchema() {
+ ITypeTraits[] schema = new ITypeTraits[VTreeStaticTupleConstants.EMBEDDING_FIELD + 2];
+ schema[VTreeStaticTupleConstants.CENTROID_ID_FIELD] = IntegerPointable.TYPE_TRAITS;
+ schema[VTreeStaticTupleConstants.EMBEDDING_FIELD] = VarLengthTypeTrait.INSTANCE;
+ schema[schema.length - 1] = IntegerPointable.TYPE_TRAITS;
+ return schema;
+ }
+
+ private static ITypeTraits[] buildLeafQuantizedSchema() {
+ // Length is (last non-pointer field index) + 1 for that field + 1 for the trailing pointer.
+ ITypeTraits[] schema = new ITypeTraits[VTreeStaticTupleConstants.LEAF_NEIGHBOR_LIST_FIELD + 2];
+ schema[VTreeStaticTupleConstants.CENTROID_ID_FIELD] = IntegerPointable.TYPE_TRAITS;
+ schema[VTreeStaticTupleConstants.EMBEDDING_FIELD] = VarLengthTypeTrait.INSTANCE;
+ schema[VTreeStaticTupleConstants.LEAF_QUANTIZED_BYTES_FIELD] = VarLengthTypeTrait.INSTANCE;
+ schema[VTreeStaticTupleConstants.LEAF_NEIGHBOR_LIST_FIELD] = VarLengthTypeTrait.INSTANCE;
+ schema[schema.length - 1] = IntegerPointable.TYPE_TRAITS;
+ return schema;
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java
index 40020a6f14..5f022098f7 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java
@@ -1180,6 +1180,19 @@ public class BufferCache implements IBufferCacheInternal, ILifeCycleComponent, I
return cachedPage;
}
+ @Override
+ public ICachedPage confiscateAndLoad(long dpid) throws HyracksDataException {
+ CachedPage page = (CachedPage) confiscatePage(dpid);
+ try {
+ // Load the on-disk content into the confiscated buffer using the same read path as pin(dpid).
+ tryRead(page, defaultContext);
+ } catch (HyracksDataException e) {
+ returnPage(page, false);
+ throw e;
+ }
+ return page;
+ }
+
private ICachedPage confiscatePage(long dpid, int multiplier) throws HyracksDataException {
ICachedPage page = getPageLoop(dpid, multiplier, true);
page.getBuffer().clear();
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/DebugBufferCache.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/DebugBufferCache.java
index f1298b7950..cf48fa4170 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/DebugBufferCache.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/DebugBufferCache.java
@@ -196,6 +196,11 @@ public class DebugBufferCache implements IBufferCache {
return bufferCache.confiscatePage(dpid);
}
+ @Override
+ public ICachedPage confiscateAndLoad(long dpid) throws HyracksDataException {
+ return bufferCache.confiscateAndLoad(dpid);
+ }
+
@Override
public ICachedPage confiscateLargePage(long dpid, int multiplier, int extraBlockPageId)
throws HyracksDataException {
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IBufferCache.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IBufferCache.java
index 81d6d561fe..0c45ef39f4 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IBufferCache.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IBufferCache.java
@@ -159,6 +159,19 @@ public interface IBufferCache {
*/
ICachedPage confiscateLargePage(long dpid, int multiplier, int extraBlockPageId) throws HyracksDataException;
+ /**
+ * Confiscate a page for an <em>already-persisted</em> {@code dpid} and load its on-disk content into the
+ * confiscated buffer (using the same read path as {@link #pin(long)}). This yields a page that carries the
+ * page's current bytes but is invisible to the cleaner and eviction (like any confiscated page), so a
+ * multi-pass writer can re-read, mutate, and re-write it without a background writer racing the write.
+ * Return it with {@link #returnPage(ICachedPage, boolean)} (a FIFO writer's {@code write} does this).
+ *
+ * @param dpid the unique (fileId,pageId) of a page that already exists on disk
+ * @return the confiscated page pre-loaded with the on-disk content, or null if no page is available
+ * @throws HyracksDataException
+ */
+ ICachedPage confiscateAndLoad(long dpid) throws HyracksDataException;
+
/**
* Return and re-insert a confiscated page
*
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/context/write/LocalOnlyWriteContext.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/context/write/LocalOnlyWriteContext.java
new file mode 100644
index 0000000000..5acf50acb3
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/context/write/LocalOnlyWriteContext.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.common.buffercache.context.write;
+
+import java.nio.ByteBuffer;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.IFileHandle;
+import org.apache.hyracks.control.nc.io.IOManager;
+import org.apache.hyracks.storage.common.buffercache.context.IBufferCacheWriteContext;
+
+/**
+ * Writes pages to local storage only, deferring any long-term (cloud) upload. Off-cloud this is an
+ * ordinary local write; in cloud mode it skips the upload (see {@code IOManager#localWriteOnly}).
+ * <p>
+ * Intended for the first pass of a multi-pass writer that publishes pages locally, mutates them, and
+ * uploads them exactly once in a later pass — required because the cloud writer is append-only, so a
+ * page must not be uploaded before its final bytes are known.
+ */
+public final class LocalOnlyWriteContext implements IBufferCacheWriteContext {
+ public static final IBufferCacheWriteContext INSTANCE = new LocalOnlyWriteContext();
+
+ private LocalOnlyWriteContext() {
+ }
+
+ @Override
+ public int write(IOManager ioManager, IFileHandle handle, long offset, ByteBuffer data)
+ throws HyracksDataException {
+ return ioManager.localWriteOnly(handle, offset, data);
+ }
+
+ @Override
+ public long write(IOManager ioManager, IFileHandle handle, long offset, ByteBuffer[] data)
+ throws HyracksDataException {
+ return ioManager.localWriteOnly(handle, offset, data);
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/impl/TestVirtualBufferCache.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/impl/TestVirtualBufferCache.java
index d4ea5a01e5..55bf15988e 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/impl/TestVirtualBufferCache.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/impl/TestVirtualBufferCache.java
@@ -135,6 +135,11 @@ public class TestVirtualBufferCache implements IVirtualBufferCache {
return vbc.confiscatePage(dpid);
}
+ @Override
+ public ICachedPage confiscateAndLoad(long dpid) throws HyracksDataException {
+ return vbc.confiscateAndLoad(dpid);
+ }
+
@Override
public ICachedPage confiscateLargePage(long dpid, int multiplier, int extraBlockPageId)
throws HyracksDataException {
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/annotations/AiProvenance.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/annotations/AiProvenance.java
index 2d32983795..f746d8a053 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/annotations/AiProvenance.java
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/annotations/AiProvenance.java
@@ -188,6 +188,11 @@ public @interface AiProvenance {
O4(OPENAI, "o4", "o4"),
O4_MINI(OPENAI, "o4-mini", "o4 Mini"),
+ // =========================
+ // Anthropic — Claude Fable (Mythos-class tier)
+ // =========================
+ CLAUDE_FABLE_5(ANTHROPIC, "claude-fable-5", "Claude Fable 5"),
+
// =========================
// Anthropic — Claude Opus
// =========================
diff --git a/hyracks-fullstack/hyracks/pom.xml b/hyracks-fullstack/hyracks/pom.xml
index 0232325fd0..0d1830d1c2 100644
--- a/hyracks-fullstack/hyracks/pom.xml
+++ b/hyracks-fullstack/hyracks/pom.xml
@@ -84,6 +84,7 @@
<module>hyracks-storage-am-common</module>
<module>hyracks-storage-am-bloomfilter</module>
<module>hyracks-storage-am-btree</module>
+ <module>hyracks-storage-am-vtree</module>
<module>hyracks-storage-am-lsm-invertedindex</module>
<module>hyracks-storage-am-lsm-common</module>
<module>hyracks-storage-am-lsm-btree</module>