You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by im...@apache.org on 2026/08/06 18:19:48 UTC
(asterixdb) 10/14: [ASTERIXDB-3771][SQL++][COMP] Vector index DDL grammar and ANN top-k query support
This is an automated email from the ASF dual-hosted git repository.
parshimers pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 4364b18776e6331808c62d1163592c62f30464d9
Author: Le0shy <le...@gmail.com>
AuthorDate: Tue May 26 13:25:47 2026 -0700
[ASTERIXDB-3771][SQL++][COMP] Vector index DDL grammar and ANN top-k query support
- user model changes: yes
- storage format changes: no
- interface changes: yes
Add the SQL++ DDL/grammar for creating VTree vector indexes, plus the
query-side optimizer, physical operators, and runtime for approximate
nearest neighbor (ANN) top-k search over them.
DDL / grammar:
- SQLPP.jj: CREATE INDEX <name> ON <ds>(<field> VECTOR) TYPE VTREE WITH
{ dimension, similarity, num_clusters, train_list_fraction, ... }
- CreateIndexStatement: vector-index AST (VECTOR field spec, TYPE VTREE,
INCLUDE fields, EXCLUDE UNKNOWN KEY)
- VectorIndexDeclUtil: validates the WITH-clause parameters (required
similarity from an allowlist, dimension, quantization, etc.)
ANN function (compile-time desugar):
- SqlppVectorDistanceRewriteVisitor: rewrites ann_distance(...) at the
SQL++ layer into the concrete vector-distance builtin (ASTERIXDB-3676)
carrying an AnnSearchPreferenceAnnotation that marks the call ANN /
vector-index-eligible.
- AnnSearchPreferenceAnnotation: the hint the optimizer matches to
select the vector index.
Query optimizer / physical:
- IntroduceTopKAccessMethodRule: matches LIMIT k -> ORDER BY over the
ANN-annotated distance call and rewrites it to a vector index access
- VectorIndexAccessMethod: IAccessMethod for vector index selection
- VectorSearchPOperator / VectorJobGenParams: physical vector-search
operator
- PushFilterIntoVectorSearchRule: pushes INCLUDE-field filters into the
vector search
Runtime / metadata:
- VectorDistanceFunctionFactory / VectorSimilarityMetric: distance
metric, fixed at index creation and carried by the factory
- MetadataProvider: builds the vector-search operator for the query path
Ext-ref: MB-72975
Change-Id: Iaa502b57915c75921cd5aa5c62fc0e24261bb0f6
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21287
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Ian Maxon <im...@apache.org>
---
.../physical/VectorIndexFilterSchema.java | 132 +++
.../physical/VectorIndexFilterTypeEnvironment.java | 103 ++
.../operators/physical/VectorSearchPOperator.java | 207 ++++
.../provider/SqlppCompilationProvider.java | 1 +
.../asterix/optimizer/base/RuleCollections.java | 6 +-
.../IntroduceSecondaryIndexInsertDeleteRule.java | 77 ++
.../rules/PushFilterIntoVectorSearchRule.java | 474 +++++++++
.../rules/SetAsterixPhysicalOperatorsRule.java | 4 +
.../am/AbstractIntroduceAccessMethodRule.java | 11 +
.../optimizer/rules/am/AccessMethodUtils.java | 27 +-
.../rules/am/IntroduceTopKAccessMethodRule.java | 1017 ++++++++++++++++++++
.../rules/am/VectorIndexAccessMethod.java | 768 +++++++++++++++
.../optimizer/rules/am/VectorJobGenParams.java | 102 ++
.../asterix/translator/util/ValidateUtil.java | 5 +-
asterixdb/asterix-app/data/vector/glove-sub.adm | 150 +++
asterixdb/asterix-app/data/vector/movie-sub.adm | 100 ++
.../asterix/app/function/DatasetRewriter.java | 15 +-
.../asterix/app/io/PersistedResourceRegistry.java | 6 +
.../asterix/app/translator/QueryTranslator.java | 130 ++-
.../test/runtime/NullMissingExceptionTest.java | 10 +-
.../asterix/test/runtime/NullMissingTest.java | 10 +-
.../queries_sqlpp/vector/VectorQueries.xml | 62 ++
.../ann-distance-invalid-metric.1.query.sqlpp | 23 +
.../create-index-vtree-composite-pk.1.ddl.sqlpp | 35 +
.../create-index-vtree-composite-pk.2.update.sqlpp | 58 ++
.../create-index-vtree-composite-pk.3.ddl.sqlpp | 22 +
.../create-index-vtree-composite-pk.4.ddl.sqlpp | 29 +
.../create-index-vtree-composite-pk.5.plans.sqlpp | 33 +
.../create-index-vtree-composite-pk.6.query.sqlpp | 33 +
.../create-index-vtree-composite-pk.99.ddl.sqlpp | 20 +
.../create-index-vtree-glove.1.ddl.sqlpp | 25 +
.../create-index-vtree-glove.2.update.sqlpp | 23 +
.../create-index-vtree-glove.3.ddl.sqlpp | 21 +
.../create-index-vtree-glove.4.ddl.sqlpp | 25 +
.../create-index-vtree-glove.5.query.sqlpp | 29 +
.../create-index-vtree-glove.99.ddl.sqlpp | 21 +
.../create-index-vtree-include-filter.1.ddl.sqlpp | 30 +
...reate-index-vtree-include-filter.2.update.sqlpp | 52 +
.../create-index-vtree-include-filter.3.ddl.sqlpp | 20 +
.../create-index-vtree-include-filter.4.ddl.sqlpp | 26 +
...create-index-vtree-include-filter.5.query.sqlpp | 27 +
.../create-index-vtree-include-filter.99.ddl.sqlpp | 19 +
.../create-index-vtree-include-nested.1.ddl.sqlpp | 31 +
...reate-index-vtree-include-nested.2.update.sqlpp | 53 +
.../create-index-vtree-include-nested.3.ddl.sqlpp | 20 +
.../create-index-vtree-include-nested.4.ddl.sqlpp | 27 +
...create-index-vtree-include-nested.5.query.sqlpp | 25 +
...create-index-vtree-include-nested.6.query.sqlpp | 26 +
.../create-index-vtree-include-nested.99.ddl.sqlpp | 20 +
...eate-index-vtree-invalid-field-type.1.ddl.sqlpp | 39 +
.../create-index-vtree-movie.1.ddl.sqlpp | 25 +
.../create-index-vtree-movie.2.update.sqlpp | 23 +
.../create-index-vtree-movie.3.ddl.sqlpp | 21 +
.../create-index-vtree-movie.4.ddl.sqlpp | 25 +
.../create-index-vtree-movie.5.query.sqlpp | 29 +
.../create-index-vtree-movie.99.ddl.sqlpp | 21 +
.../create-index-vtree-nested.1.ddl.sqlpp | 37 +
.../create-index-vtree-nested.2.update.sqlpp | 53 +
.../create-index-vtree-nested.3.ddl.sqlpp | 21 +
.../create-index-vtree-nested.4.ddl.sqlpp | 28 +
.../create-index-vtree-nested.5.plans.sqlpp | 28 +
.../create-index-vtree-nested.99.ddl.sqlpp | 20 +
.../create-index-vtree-open-undeclared.1.ddl.sqlpp | 32 +
...eate-index-vtree-open-undeclared.2.update.sqlpp | 56 ++
.../create-index-vtree-open-undeclared.3.ddl.sqlpp | 20 +
.../create-index-vtree-open-undeclared.4.ddl.sqlpp | 27 +
...reate-index-vtree-open-undeclared.5.query.sqlpp | 25 +
...reate-index-vtree-open-undeclared.6.plans.sqlpp | 29 +
...create-index-vtree-open-undeclared.99.ddl.sqlpp | 20 +
...te-index-vtree-similarity-normalize.1.ddl.sqlpp | 32 +
...index-vtree-similarity-normalize.2.update.sqlpp | 55 ++
...te-index-vtree-similarity-normalize.3.ddl.sqlpp | 22 +
...te-index-vtree-similarity-normalize.4.ddl.sqlpp | 28 +
...-index-vtree-similarity-normalize.5.query.sqlpp | 23 +
...e-index-vtree-similarity-normalize.99.ddl.sqlpp | 20 +
.../create-index-vtree.1.ddl.sqlpp | 32 +
.../create-index-vtree.2.update.sqlpp | 55 ++
.../create-index-vtree.3.ddl.sqlpp | 22 +
.../create-index-vtree.4.ddl.sqlpp | 29 +
.../create-index-vtree.5.query.sqlpp | 23 +
.../create-index-vtree.6.query.sqlpp | 27 +
.../create-index-vtree.7.plans.sqlpp | 28 +
.../create-index-vtree.8.plans.sqlpp | 29 +
.../create-index-vtree.9.plans.sqlpp | 29 +
.../create-index-vtree.99.ddl.sqlpp | 20 +
.../soft-keyword-identifiers.1.ddl.sqlpp | 33 +
.../soft-keyword-identifiers.2.update.sqlpp | 26 +
.../soft-keyword-identifiers.3.query.sqlpp | 26 +
.../soft-keyword-identifiers.99.ddl.sqlpp | 20 +
.../create-index-vtree-composite-pk.5.regex | 1 +
.../create-index-vtree-composite-pk.6.adm | 3 +
.../create-index-vtree-glove.5.adm | 5 +
.../create-index-vtree-include-filter.5.adm | 3 +
.../create-index-vtree-include-nested.5.adm | 1 +
.../create-index-vtree-include-nested.6.adm | 3 +
.../create-index-vtree-movie.5.adm | 5 +
.../create-index-vtree-nested.5.regex | 1 +
.../create-index-vtree-open-undeclared.5.adm | 3 +
.../create-index-vtree-open-undeclared.6.regex | 1 +
.../create-index-vtree-similarity-normalize.5.adm | 1 +
.../create-index-vtree/create-index-vtree.5.adm | 1 +
.../create-index-vtree/create-index-vtree.6.adm | 3 +
.../create-index-vtree/create-index-vtree.7.regex | 1 +
.../create-index-vtree/create-index-vtree.8.regex | 1 +
.../create-index-vtree/create-index-vtree.9.regex | 1 +
.../soft-keyword-identifiers.3.adm | 2 +
.../annotations/AnnSearchPreferenceAnnotation.java | 61 ++
.../asterix/common/config/CompilerProperties.java | 1 +
.../common/dataflow/DatasetLocalResource.java | 4 +-
.../common/statement/CreateIndexStatement.java | 20 +-
.../lang/common/util/CommonFunctionMapUtil.java | 4 +
.../asterix/lang/common/util/FunctionUtil.java | 10 +-
.../lang/common/util/VectorIndexDeclUtil.java | 318 ++++++
.../visitor/SqlppVectorDistanceRewriteVisitor.java | 81 ++
.../asterix-lang-sqlpp/src/main/javacc/SQLPP.jj | 178 +++-
asterixdb/asterix-metadata/pom.xml | 4 +
.../declared/BTreeResourceFactoryProvider.java | 38 +-
.../metadata/declared/MetadataManagerUtil.java | 29 +
.../metadata/declared/MetadataProvider.java | 190 ++++
.../apache/asterix/metadata/entities/Dataset.java | 7 +
.../IndexTupleTranslator.java | 356 ++++++-
.../apache/asterix/metadata/utils/DatasetUtil.java | 3 +
.../asterix/metadata/utils/KeyFieldTypeUtil.java | 28 +
.../asterix/om/functions/BuiltinFunctions.java | 7 +
.../core/algebra/base/PhysicalOperatorTag.java | 3 +-
125 files changed, 6666 insertions(+), 60 deletions(-)
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/VectorIndexFilterSchema.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/VectorIndexFilterSchema.java
new file mode 100644
index 0000000000..f9d9b7e482
--- /dev/null
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/VectorIndexFilterSchema.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.algebra.operators.physical;
+
+import java.util.Iterator;
+import java.util.Map;
+
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+
+/**
+ * A wrapper schema for vector index filter evaluation that maps variables to
+ * their correct physical tuple positions.
+ *
+ * Vector index physical tuple format (quantized):
+ * [distance, qDist, qEmbed, centroidId, pk, include_fields...]
+ *
+ * The original opSchema only contains [pk] because:
+ * - Secondary fields (distance, centroidId, etc.) are internal to the index
+ * - INCLUDE fields are only used for filtering, not output
+ *
+ * This wrapper supports two mapping modes:
+ * 1. Variables in opSchema (like pk): position + numSecondaryKeys offset
+ * 2. Filter-only variables (INCLUDE fields): direct mapping via filterVarToFieldIndex
+ *
+ * This allows the TupleFilter to correctly access INCLUDE fields during vector index search
+ * without exposing them to downstream operators (which only need pk for primary index lookup).
+ */
+public class VectorIndexFilterSchema implements IOperatorSchema {
+
+ // Number of internal fields before pk in vector index tuple
+ // Quantized: 4 (distance, qDist, qEmbed, centroidId)
+ private final int fieldOffset;
+
+ private final IOperatorSchema delegate;
+
+ // Direct mapping for filter-only variables (INCLUDE fields not in output schema)
+ // Maps LogicalVariable -> physical tuple field index
+ private final Map<LogicalVariable, Integer> filterVarToFieldIndex;
+
+ /**
+ * Constructor without direct mapping (backward compatibility).
+ * All variables must be in the delegate schema.
+ */
+ public VectorIndexFilterSchema(IOperatorSchema delegate, int numSecondaryKeys) {
+ this(delegate, null, numSecondaryKeys);
+ }
+
+ /**
+ * Constructor with direct mapping for filter-only variables.
+ *
+ * @param delegate The operator output schema (contains pk)
+ * @param filterVarToFieldIndex Direct mapping for INCLUDE field variables to physical field indexes
+ * @param numSecondaryKeys Number of secondary fields before PK (4 for quantized)
+ */
+ public VectorIndexFilterSchema(IOperatorSchema delegate, Map<LogicalVariable, Integer> filterVarToFieldIndex,
+ int numSecondaryKeys) {
+ this.delegate = delegate;
+ this.filterVarToFieldIndex = filterVarToFieldIndex;
+ this.fieldOffset = numSecondaryKeys;
+ }
+
+ @Override
+ public int findVariable(LogicalVariable var) {
+ // First check direct mapping for filter-only variables
+ if (filterVarToFieldIndex != null && filterVarToFieldIndex.containsKey(var)) {
+ return filterVarToFieldIndex.get(var);
+ }
+
+ // Fall back to delegate schema with offset
+ int originalPos = delegate.findVariable(var);
+ if (originalPos < 0) {
+ return originalPos;
+ }
+ return originalPos + fieldOffset;
+ }
+
+ @Override
+ public LogicalVariable getVariable(int index) {
+ if (index < fieldOffset) {
+ // distance and centroidId have no logical variables
+ return null;
+ }
+ return delegate.getVariable(index - fieldOffset);
+ }
+
+ @Override
+ public int getSize() {
+ return delegate.getSize() + fieldOffset;
+ }
+
+ @Override
+ public void addAllVariables(IOperatorSchema source) {
+ delegate.addAllVariables(source);
+ }
+
+ @Override
+ public void addAllNewVariables(IOperatorSchema source) {
+ delegate.addAllNewVariables(source);
+ }
+
+ @Override
+ public int addVariable(LogicalVariable var) {
+ return delegate.addVariable(var) + fieldOffset;
+ }
+
+ @Override
+ public void clear() {
+ delegate.clear();
+ }
+
+ @Override
+ public Iterator<LogicalVariable> iterator() {
+ return delegate.iterator();
+ }
+}
\ No newline at end of file
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/VectorIndexFilterTypeEnvironment.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/VectorIndexFilterTypeEnvironment.java
new file mode 100644
index 0000000000..a0bde1f891
--- /dev/null
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/VectorIndexFilterTypeEnvironment.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.algebra.operators.physical;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.asterix.om.types.IAType;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import org.apache.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import org.apache.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+
+/**
+ * A wrapper type environment for vector index filter compilation.
+ *
+ * Filter-only variables (INCLUDE fields) are not in the operator's output variables,
+ * so the regular type environment doesn't know their types. This wrapper provides
+ * types for filter-only variables from the annotation map while delegating other
+ * variable lookups to the original type environment.
+ */
+public class VectorIndexFilterTypeEnvironment implements IVariableTypeEnvironment {
+
+ private final IVariableTypeEnvironment delegate;
+ private final Map<LogicalVariable, IAType> filterVarTypes;
+ private final JobGenContext context;
+
+ /**
+ * Creates a wrapper type environment with filter variable types.
+ *
+ * @param delegate The original type environment
+ * @param filterVarTypes Map from filter-only variables to their types
+ * @param context The JobGenContext for expression type computation
+ */
+ public VectorIndexFilterTypeEnvironment(IVariableTypeEnvironment delegate,
+ Map<LogicalVariable, IAType> filterVarTypes, JobGenContext context) {
+ this.delegate = delegate;
+ this.filterVarTypes = filterVarTypes;
+ this.context = context;
+ }
+
+ @Override
+ public Object getVarType(LogicalVariable var) throws AlgebricksException {
+ if (filterVarTypes != null && filterVarTypes.containsKey(var)) {
+ return filterVarTypes.get(var);
+ }
+ return delegate.getVarType(var);
+ }
+
+ @Override
+ public Object getVarType(LogicalVariable var, List<LogicalVariable> nonMissableVariables,
+ List<List<LogicalVariable>> correlatedMissableVariableLists, List<LogicalVariable> nonNullableVariables,
+ List<List<LogicalVariable>> correlatedNullableVariableLists) throws AlgebricksException {
+ if (filterVarTypes != null && filterVarTypes.containsKey(var)) {
+ return filterVarTypes.get(var);
+ }
+ return delegate.getVarType(var, nonMissableVariables, correlatedMissableVariableLists, nonNullableVariables,
+ correlatedNullableVariableLists);
+ }
+
+ @Override
+ public void setVarType(LogicalVariable var, Object type) {
+ delegate.setVarType(var, type);
+ }
+
+ @Override
+ public Object getType(ILogicalExpression expr) throws AlgebricksException {
+ if (expr.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
+ VariableReferenceExpression varRef = (VariableReferenceExpression) expr;
+ LogicalVariable var = varRef.getVariableReference();
+ if (filterVarTypes != null && filterVarTypes.containsKey(var)) {
+ return filterVarTypes.get(var);
+ }
+ }
+ // For function expressions, route type computation through this wrapper so recursive
+ // lookups resolve filter-only variables instead of falling back to the delegate.
+ return context.getType(expr, this);
+ }
+
+ @Override
+ public boolean substituteProducedVariable(LogicalVariable v1, LogicalVariable v2) throws AlgebricksException {
+ return delegate.substituteProducedVariable(v1, v2);
+ }
+}
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/VectorSearchPOperator.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/VectorSearchPOperator.java
new file mode 100644
index 0000000000..0e95e5a89e
--- /dev/null
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/VectorSearchPOperator.java
@@ -0,0 +1,207 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.algebra.operators.physical;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.asterix.metadata.declared.DataSourceId;
+import org.apache.asterix.metadata.declared.DataSourceIndex;
+import org.apache.asterix.metadata.declared.DatasetDataSource;
+import org.apache.asterix.metadata.declared.MetadataProvider;
+import org.apache.asterix.metadata.entities.Dataset;
+import org.apache.asterix.metadata.entities.Index;
+import org.apache.asterix.object.base.AdmObjectNode;
+import org.apache.asterix.om.functions.BuiltinFunctions;
+import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.optimizer.rules.PushFilterIntoVectorSearchRule;
+import org.apache.asterix.optimizer.rules.am.VectorJobGenParams;
+import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.common.utils.Pair;
+import org.apache.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import org.apache.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import org.apache.hyracks.algebricks.core.algebra.metadata.IDataSource;
+import org.apache.hyracks.algebricks.core.algebra.metadata.IDataSourceIndex;
+import org.apache.hyracks.algebricks.core.algebra.metadata.IDataSourcePropertiesProvider;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractScanOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractUnnestMapOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import org.apache.hyracks.algebricks.core.algebra.properties.INodeDomain;
+import org.apache.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import org.apache.hyracks.api.dataflow.IOperatorDescriptor;
+import org.apache.hyracks.storage.am.common.api.ITupleFilterFactory;
+
+/**
+ * Contributes the runtime operator for an unnest-map representing a vector index search.
+ */
+public class VectorSearchPOperator extends IndexSearchPOperator {
+
+ public VectorSearchPOperator(IDataSourceIndex<String, DataSourceId> idx, INodeDomain domain,
+ boolean requiresBroadcast) {
+ super(idx, domain, requiresBroadcast);
+ }
+
+ @Override
+ public PhysicalOperatorTag getOperatorTag() {
+ return PhysicalOperatorTag.VECTOR_SEARCH;
+ }
+
+ /**
+ * The vector index unnest-map emits its scan variables as
+ * {@code [pk_0..pk_{N-1}, (distance?), pushedIncludeFilterVars...]} -- the primary keys come
+ * FIRST, unlike a BTree/RTree secondary search where they come last.
+ * {@link IndexSearchPOperator#computeDeliveredProperties} assumes the primary keys are the LAST
+ * {@code numPrimaryKeys} scan variables, so an appended index-only distance field or a pushed
+ * INCLUDE-filter variable would be mislabeled as a primary key and yield a wrong delivered
+ * partitioning property on a partitioned cluster. Take the FIRST {@code numPrimaryKeys} variables
+ * instead. When nothing extra is appended this is identical to the inherited behavior.
+ */
+ @Override
+ public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context)
+ throws AlgebricksException {
+ IDataSource<?> ds = idx.getDataSource();
+ List<LogicalVariable> scanVariables = new ArrayList<>();
+ if (idx instanceof DataSourceIndex) {
+ Index index = ((DataSourceIndex) idx).getIndex();
+ if (index.isSecondaryIndex() && ds instanceof DatasetDataSource) {
+ Dataset dataset = ((DatasetDataSource) ds).getDataset();
+ int numOfPrimaryKeys = dataset.getPrimaryKeys().size();
+ if (op.getOperatorTag() == LogicalOperatorTag.UNNEST_MAP
+ || op.getOperatorTag() == LogicalOperatorTag.LEFT_OUTER_UNNEST_MAP) {
+ List<LogicalVariable> opVars = ((AbstractUnnestMapOperator) op).getScanVariables();
+ // Vector search: primary keys are at the FRONT of the scan-variable list.
+ scanVariables.addAll(opVars.subList(0, numOfPrimaryKeys));
+ scanVariables.add(new LogicalVariable(-1));
+ if (dataset.hasMetaPart()) {
+ scanVariables.add(new LogicalVariable(-1));
+ }
+ }
+ }
+ }
+ if (scanVariables.isEmpty()) {
+ AbstractScanOperator as = (AbstractScanOperator) op;
+ scanVariables.addAll(as.getScanVariables());
+ }
+ IDataSourcePropertiesProvider dspp = ds.getPropertiesProvider();
+ deliveredProperties = dspp.computeDeliveredProperties(scanVariables, context);
+ }
+
+ @Override
+ public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+ IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+ throws AlgebricksException {
+ AbstractUnnestMapOperator unnestMap = (AbstractUnnestMapOperator) op;
+ ILogicalExpression unnestExpr = unnestMap.getExpressionRef().getValue();
+ if (unnestExpr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
+ throw new AlgebricksException("Vector search unnest expression must be a function call");
+ }
+ AbstractFunctionCallExpression unnestFuncExpr = (AbstractFunctionCallExpression) unnestExpr;
+ FunctionIdentifier funcIdent = unnestFuncExpr.getFunctionIdentifier();
+ if (!funcIdent.equals(BuiltinFunctions.INDEX_SEARCH)) {
+ return;
+ }
+
+ VectorJobGenParams jobGenParams = new VectorJobGenParams();
+ jobGenParams.readFromFuncArgs(unnestFuncExpr.getArguments());
+
+ // queryVarList layout: [query_vector, k, metric, min_probe_fraction, k_multiplier]
+ int[] queryIndexes = getKeyIndexes(jobGenParams.getQueryVarList(), inputSchemas);
+
+ MetadataProvider mp = (MetadataProvider) context.getMetadataProvider();
+ Dataset dataset = mp.findDataset(jobGenParams.getDatabaseName(), jobGenParams.getDataverseName(),
+ jobGenParams.getDatasetName());
+ IVariableTypeEnvironment typeEnv = context.getTypeEnvironment(unnestMap);
+
+ List<LogicalVariable> outputVars = unnestMap.getVariables();
+ if (jobGenParams.getRetainInput()) {
+ outputVars = new ArrayList<>();
+ VariableUtilities.getLiveVariables(unnestMap, outputVars);
+ }
+
+ // Determine quantization from vector index metadata
+ Index vectorIndex = mp.getIndex(jobGenParams.getDatabaseName(), jobGenParams.getDataverseName(),
+ jobGenParams.getDatasetName(), jobGenParams.getIndexName());
+ Index.VectorIndexDetails vectorDetails = (Index.VectorIndexDetails) vectorIndex.getIndexDetails();
+ AdmObjectNode withObjectNode = vectorDetails.getWithObjectNode();
+ String quantization = (withObjectNode != null) ? withObjectNode.getOptionalString("quantization", null) : null;
+ boolean isQuantized = (quantization != null);
+ int numSecondaryKeys = isQuantized ? 4 : 2;
+
+ // Create tuple filter factory if selectCondition is present (for INCLUDE field filtering)
+ // The opSchema only has [pk] because INCLUDE fields are only used for filtering.
+ // Filter variables are mapped directly to physical field indexes via annotation.
+ ITupleFilterFactory tupleFilterFactory = null;
+ if (unnestMap instanceof UnnestMapOperator) {
+ UnnestMapOperator unnestMapOp = (UnnestMapOperator) unnestMap;
+ if (unnestMapOp.getSelectCondition() != null) {
+ // Get filter variable to physical field index mapping from annotation
+ @SuppressWarnings("unchecked")
+ Map<LogicalVariable, Integer> filterVarToFieldIndex = (Map<LogicalVariable, Integer>) unnestMapOp
+ .getAnnotations().get(PushFilterIntoVectorSearchRule.VECTOR_FILTER_VAR_MAPPING);
+
+ // Get filter variable types from annotation
+ @SuppressWarnings("unchecked")
+ Map<LogicalVariable, IAType> filterVarTypes = (Map<LogicalVariable, IAType>) unnestMapOp
+ .getAnnotations().get(PushFilterIntoVectorSearchRule.VECTOR_FILTER_VAR_TYPES);
+
+ // Create filter schema with direct mapping for filter-only variables
+ // numSecondaryKeys: offset from physical tuple start to PK field
+ IOperatorSchema filterSchema =
+ new VectorIndexFilterSchema(opSchema, filterVarToFieldIndex, numSecondaryKeys);
+
+ // Create type environment with filter variable types
+ // Pass context so function expressions can use this wrapper for recursive type lookups
+ IVariableTypeEnvironment filterTypeEnv =
+ new VectorIndexFilterTypeEnvironment(typeEnv, filterVarTypes, context);
+
+ tupleFilterFactory = mp.createTupleFilterFactory(new IOperatorSchema[] { filterSchema }, filterTypeEnv,
+ unnestMapOp.getSelectCondition().getValue(), context);
+ }
+ }
+
+ // jobGenParams.isIndexOnly() (set by IntroduceTopKAccessMethodRule when the projection above
+ // LIMIT references only PK columns) tells the runtime to emit [pk..., D(q,x)] per candidate so
+ // the downstream sort can rank without the primary BTree lookup.
+ Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> vectorSearch = mp.getVectorSearchRuntime(
+ builder.getJobSpec(), outputVars, opSchema, typeEnv, context, jobGenParams.getRetainInput(), dataset,
+ jobGenParams.getIndexName(), queryIndexes, tupleFilterFactory, jobGenParams.isIndexOnly());
+
+ IOperatorDescriptor opDesc = vectorSearch.first;
+ opDesc.setSourceLocation(unnestMap.getSourceLocation());
+
+ builder.contributeHyracksOperator(unnestMap, opDesc);
+ builder.contributeAlgebricksPartitionConstraint(opDesc, vectorSearch.second);
+
+ ILogicalOperator srcExchange = unnestMap.getInputs().get(0).getValue();
+ builder.contributeGraphEdge(srcExchange, 0, unnestMap, 0);
+ }
+}
\ No newline at end of file
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/SqlppCompilationProvider.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/SqlppCompilationProvider.java
index 77605ee9c3..69bba773d3 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/SqlppCompilationProvider.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/SqlppCompilationProvider.java
@@ -122,6 +122,7 @@ public class SqlppCompilationProvider implements ILangCompilationProvider {
CompilerProperties.COMPILER_HDFS_SPLIT_PARALLELISM_KEY,
CompilerProperties.COMPILER_AGGREGATE_DISTINCT_HASH_MEMORY_KEY,
CompilerProperties.COMPILER_AGGREGATE_DISTINCT_HASH_KEY,
+ CompilerProperties.COMPILER_VECTOR_K_MULTIPLIER_KEY,
DisjunctivePredicateToJoinRule.REWRITE_OR_AS_JOIN_OPTION,
SetAsterixPhysicalOperatorsRule.REWRITE_ATTEMPT_BATCH_ASSIGN,
EquivalenceClassUtils.REWRITE_INTERNAL_QUERYUID_PK, SqlppQueryRewriter.SQL_COMPAT_OPTION,
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java
index c65b758961..97c826d352 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java
@@ -77,6 +77,7 @@ import org.apache.asterix.optimizer.rules.PullSelectOutOfSpatialJoin;
import org.apache.asterix.optimizer.rules.PushAggFuncIntoStandaloneAggregateRule;
import org.apache.asterix.optimizer.rules.PushAggregateIntoNestedSubplanRule;
import org.apache.asterix.optimizer.rules.PushFieldAccessRule;
+import org.apache.asterix.optimizer.rules.PushFilterIntoVectorSearchRule;
import org.apache.asterix.optimizer.rules.PushGroupByThroughProduct;
import org.apache.asterix.optimizer.rules.PushLimitIntoOrderByRule;
import org.apache.asterix.optimizer.rules.PushLimitIntoPrimarySearchRule;
@@ -106,6 +107,7 @@ import org.apache.asterix.optimizer.rules.am.IntroduceJoinAccessMethodRule;
import org.apache.asterix.optimizer.rules.am.IntroduceLSMComponentFilterRule;
import org.apache.asterix.optimizer.rules.am.IntroducePrimaryIndexForAggregationRule;
import org.apache.asterix.optimizer.rules.am.IntroduceSelectAccessMethodRule;
+import org.apache.asterix.optimizer.rules.am.IntroduceTopKAccessMethodRule;
import org.apache.asterix.optimizer.rules.cbo.EnumerateJoinsRule;
import org.apache.asterix.optimizer.rules.cbo.JoinEnum;
import org.apache.asterix.optimizer.rules.cbo.indexadvisor.AdviseIndexRule;
@@ -338,6 +340,7 @@ public final class RuleCollections {
List<IAlgebraicRewriteRule> accessMethod = new LinkedList<>();
accessMethod.add(new AttachCompileTimeRangeMapRule());
accessMethod.add(new IntroduceSelectAccessMethodRule());
+ accessMethod.add(new IntroduceTopKAccessMethodRule());
accessMethod.add(new IntroduceJoinAccessMethodRule());
accessMethod.add(new IntroduceLSMComponentFilterRule());
accessMethod.add(new IntroducePrimaryIndexForAggregationRule());
@@ -440,7 +443,8 @@ public final class RuleCollections {
physicalRewritesTopLevel.add(new ConstantFoldingRule(appCtx));
physicalRewritesTopLevel.add(new PushLimitIntoOrderByRule());
physicalRewritesTopLevel.add(new PushLimitIntoPrimarySearchRule());
- // remove assigns that could become unused after PushLimitIntoPrimarySearchRule
+ physicalRewritesTopLevel.add(new PushFilterIntoVectorSearchRule());
+ // remove assigns that could become unused after PushLimitIntoPrimarySearchRule or PushFilterIntoVectorSearchRule
physicalRewritesTopLevel.add(new RemoveUnusedAssignAndAggregateRule());
physicalRewritesTopLevel.add(new IntroduceProjectsRule());
//Infer the types for the pushed down condition
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
index 887b18ebd6..37e8635c8d 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
@@ -318,6 +318,44 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
case ARRAY:
// These details are handled separately for array indexes.
break;
+ case VTREE:
+ // Vector indexes are handled similarly to VALUE indexes
+ Index.VectorIndexDetails vectorIndexDetails2 = (Index.VectorIndexDetails) index.getIndexDetails();
+ // Start with the vector field (key field)
+ secondaryKeyFields = new ArrayList<>(vectorIndexDetails2.getKeyFieldNames());
+ secondaryKeyTypes = new ArrayList<>();
+ // Get actual type from schema for vector field (must match injectFieldAccessesForIndexes)
+ List<String> vectorFieldName2 = secondaryKeyFields.get(0);
+ IAType vectorFieldType2 = recType.getSubFieldType(vectorFieldName2);
+ if (vectorFieldType2 == null) {
+ vectorFieldType2 = BuiltinType.ANY;
+ }
+ secondaryKeyTypes.add(vectorFieldType2);
+ secondaryKeySources = new ArrayList<>();
+ secondaryKeySources.add(Index.RECORD_INDICATOR);
+
+ // Add include fields after vector field
+ // This produces tuple format: [vector, include_fields..., pk...]
+ List<List<String>> includeFieldNames = vectorIndexDetails2.getIncludeFieldNames();
+ List<Integer> includeFieldSources = vectorIndexDetails2.getIncludeFieldSourceIndicators();
+ if (includeFieldNames != null && !includeFieldNames.isEmpty()) {
+ for (int j = 0; j < includeFieldNames.size(); j++) {
+ List<String> includeFieldName = includeFieldNames.get(j);
+ secondaryKeyFields.add(includeFieldName);
+ // Get actual type from schema (must match injectFieldAccessesForIndexes)
+ IAType includeFieldType = recType.getSubFieldType(includeFieldName);
+ if (includeFieldType == null) {
+ includeFieldType = BuiltinType.ANY;
+ }
+ secondaryKeyTypes.add(includeFieldType);
+ if (includeFieldSources != null && j < includeFieldSources.size()) {
+ secondaryKeySources.add(includeFieldSources.get(j));
+ } else {
+ secondaryKeySources.add(Index.RECORD_INDICATOR);
+ }
+ }
+ }
+ break;
default:
continue;
}
@@ -834,6 +872,45 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
skTypes = textIndexDetails.getKeyFieldTypes();
indicators = textIndexDetails.getKeyFieldSourceIndicators();
break;
+ case VTREE:
+ Index.VectorIndexDetails vectorIndexDetails = (Index.VectorIndexDetails) index.getIndexDetails();
+ // For vector indexes, get the vector field name
+ skNames = new ArrayList<>(vectorIndexDetails.getKeyFieldNames());
+ // For vector fields, try to get the actual type from the schema
+ // If not available (open field), use ANY type for type inference
+ skTypes = new ArrayList<>();
+ List<String> vectorFieldName = skNames.get(0);
+ IAType vectorFieldType = recType.getSubFieldType(vectorFieldName);
+ if (vectorFieldType == null) {
+ // Open field - use ANY type to allow runtime type inference
+ vectorFieldType = BuiltinType.ANY;
+ }
+ skTypes.add(vectorFieldType);
+ // Vector field is always from the record
+ indicators = new ArrayList<>();
+ indicators.add(Index.RECORD_INDICATOR);
+
+ // Handle include fields
+ List<List<String>> vectorIncludeFieldNames = vectorIndexDetails.getIncludeFieldNames();
+ List<Integer> vectorIncludeFieldSources = vectorIndexDetails.getIncludeFieldSourceIndicators();
+ if (vectorIncludeFieldNames != null && !vectorIncludeFieldNames.isEmpty()) {
+ for (int j = 0; j < vectorIncludeFieldNames.size(); j++) {
+ List<String> includeFieldName = vectorIncludeFieldNames.get(j);
+ skNames.add(includeFieldName);
+ // Get actual type from schema (not from index metadata which may be ANY)
+ IAType includeFieldType = recType.getSubFieldType(includeFieldName);
+ if (includeFieldType == null) {
+ includeFieldType = BuiltinType.ANY;
+ }
+ skTypes.add(includeFieldType);
+ if (vectorIncludeFieldSources != null && j < vectorIncludeFieldSources.size()) {
+ indicators.add(vectorIncludeFieldSources.get(j));
+ } else {
+ indicators.add(Index.RECORD_INDICATOR);
+ }
+ }
+ }
+ break;
default:
throw new CompilationException(ErrorCode.COMPILATION_UNKNOWN_INDEX_TYPE,
String.valueOf(index.getIndexType()));
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushFilterIntoVectorSearchRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushFilterIntoVectorSearchRule.java
new file mode 100644
index 0000000000..67aaef6cbf
--- /dev/null
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushFilterIntoVectorSearchRule.java
@@ -0,0 +1,474 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.optimizer.rules;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.asterix.common.config.DatasetConfig.IndexType;
+import org.apache.asterix.metadata.declared.MetadataProvider;
+import org.apache.asterix.metadata.entities.Dataset;
+import org.apache.asterix.metadata.entities.Index;
+import org.apache.asterix.object.base.AdmObjectNode;
+import org.apache.asterix.om.functions.BuiltinFunctions;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.BuiltinType;
+import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.optimizer.rules.am.AccessMethodJobGenParams;
+import org.apache.asterix.optimizer.rules.am.AccessMethodUtils;
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import org.apache.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
+import org.apache.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
+import org.apache.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
+import org.apache.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+import org.apache.hyracks.algebricks.rewriter.rules.InlineVariablesRule;
+import org.apache.hyracks.util.annotations.AiProvenance;
+
+/**
+ * Pushes filter conditions into a vector index search when the filter only references
+ * INCLUDE fields of that index. Must run in physicalRewritesTopLevel (after
+ * SetClosedRecordConstructorsRule) so that record constructors have already been closed.
+ *
+ * Pattern:
+ * <pre>
+ * SELECT (condition on INCLUDE fields)
+ * └── ASSIGN* (optional)
+ * └── PRIMARY_INDEX_UNNEST
+ * └── ...
+ * └── VECTOR_INDEX_UNNEST [$pk]
+ * </pre>
+ *
+ * Transforms to:
+ * <pre>
+ * ASSIGN* (optional, SELECT removed)
+ * └── PRIMARY_INDEX_UNNEST
+ * └── ...
+ * └── VECTOR_INDEX_UNNEST [$pk, $includeField1, ...]
+ * selectCondition: (rewritten to use $includeField1, ...)
+ * </pre>
+ *
+ * New variables are created for the INCLUDE fields produced by VECTOR_INDEX_UNNEST, and
+ * field-access expressions in the filter are rewritten to reference those variables.
+ */
+public class PushFilterIntoVectorSearchRule implements IAlgebraicRewriteRule {
+
+ /**
+ * Annotation key for filter variable to physical field index mapping.
+ * Value type: Map<LogicalVariable, Integer>
+ */
+ public static final String VECTOR_FILTER_VAR_MAPPING = "VECTOR_FILTER_VAR_MAPPING";
+
+ /**
+ * Annotation key for filter variable to type mapping.
+ * Value type: Map<LogicalVariable, IAType>
+ */
+ public static final String VECTOR_FILTER_VAR_TYPES = "VECTOR_FILTER_VAR_TYPES";
+
+ @Override
+ public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
+ return false;
+ }
+
+ @Override
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_FABLE_5, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.ASSISTED)
+ public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+ throws AlgebricksException {
+ ILogicalOperator op = opRef.getValue();
+
+ if (op.getOperatorTag() != LogicalOperatorTag.SELECT) {
+ return false;
+ }
+ if (context.checkIfInDontApplySet(this, op)) {
+ return false;
+ }
+
+ SelectOperator selectOp = (SelectOperator) op;
+
+ VectorSearchInfo searchInfo = findVectorIndexUnnest(selectOp, context);
+ if (searchInfo == null) {
+ return false;
+ }
+ if (searchInfo.includeFieldNames() == null || searchInfo.includeFieldNames().isEmpty()) {
+ return false;
+ }
+
+ // Inline ASSIGNs into the select condition so that filter references hit field-access
+ // expressions on the source record directly.
+ ILogicalExpression selectCondition = selectOp.getCondition().getValue();
+ MutableObject<ILogicalExpression> conditionRef = new MutableObject<>(selectCondition.cloneExpression());
+
+ Set<LogicalVariable> usedVariables = new HashSet<>();
+ conditionRef.getValue().getUsedVariables(usedVariables);
+
+ InlineVariablesRule.InlineVariablesVisitor inlineVisitor = null;
+ Map<LogicalVariable, ILogicalExpression> varAssignRhs = new HashMap<>();
+
+ for (ILogicalOperator child = selectOp.getInputs().get(0).getValue(); child
+ .getOperatorTag() == LogicalOperatorTag.ASSIGN; child = child.getInputs().get(0).getValue()) {
+ varAssignRhs.clear();
+ AssignOperator assignOp = (AssignOperator) child;
+ extractInlinableVariables(assignOp, usedVariables, varAssignRhs);
+
+ if (!varAssignRhs.isEmpty()) {
+ if (inlineVisitor == null) {
+ inlineVisitor = new InlineVariablesRule.InlineVariablesVisitor(varAssignRhs, null);
+ inlineVisitor.setContext(context);
+ inlineVisitor.setOperator(selectOp);
+ }
+ if (!inlineVisitor.transform(conditionRef)) {
+ break;
+ }
+ usedVariables.clear();
+ conditionRef.getValue().getUsedVariables(usedVariables);
+ }
+ }
+
+ Set<String> filterFieldNames = new HashSet<>();
+ extractFieldNames(conditionRef.getValue(), searchInfo.recordType(), filterFieldNames);
+
+ if (filterFieldNames.isEmpty()) {
+ return false;
+ }
+
+ // Bail out if the filter references any field that is not in the index's INCLUDE list.
+ Map<String, Integer> includeFieldIndex = buildIncludeFieldIndex(searchInfo.includeFieldNames());
+ // Leaf-name-keyed matching (here and in the variable-creation loop below) is ambiguous if two
+ // INCLUDE fields share a leaf name -- only possible with nested paths, e.g. INCLUDE(a.x, b.x).
+ // A collapsed map would silently bind the filter to whichever field was inserted last. Bail
+ // rather than risk pushing a predicate that resolves to the wrong physical field.
+ if (includeFieldIndex.size() < searchInfo.includeFieldNames().size()) {
+ return false;
+ }
+ for (String fieldName : filterFieldNames) {
+ if (!includeFieldIndex.containsKey(fieldName)) {
+ return false;
+ }
+ }
+
+ // Create fresh logical variables for INCLUDE fields referenced by the filter. These variables
+ // are produced by VECTOR_INDEX_UNNEST solely for filter evaluation.
+ Map<String, LogicalVariable> fieldToNewVar = new HashMap<>();
+ Map<LogicalVariable, Integer> filterVarToFieldIndex = new HashMap<>();
+ Map<LogicalVariable, IAType> filterVarToType = new HashMap<>();
+
+ // Physical tuple format depends on quantization:
+ // Non-quantized: [distance, centroidId, pk_0..pk_{N-1}, include_fields...]
+ // Quantized: [distance, centroidId, qDist, qEmbed, pk_0..pk_{N-1}, include_fields...]
+ // INCLUDE fields start after the secondary keys and ALL primary key columns.
+ int numSecondaryKeys = searchInfo.isQuantized() ? 4 : 2;
+ int includeFieldPhysicalIndex = numSecondaryKeys + searchInfo.numPrimaryKeys();
+
+ for (List<String> fieldPath : searchInfo.includeFieldNames()) {
+ String fieldName = fieldPath.get(fieldPath.size() - 1);
+
+ if (filterFieldNames.contains(fieldName)) {
+ LogicalVariable newVar = context.newVar();
+ fieldToNewVar.put(fieldName, newVar);
+ filterVarToFieldIndex.put(newVar, includeFieldPhysicalIndex);
+
+ // For open-schema fields not in the type declaration, getSubFieldType returns null;
+ // default to ANY so the type environment can still resolve the variable type.
+ IAType fieldType = searchInfo.recordType().getSubFieldType(fieldPath);
+ if (fieldType == null) {
+ fieldType = BuiltinType.ANY;
+ }
+ filterVarToType.put(newVar, fieldType);
+ }
+
+ includeFieldPhysicalIndex++;
+ }
+
+ // Rewrite field-access expressions in the condition to reference the new INCLUDE variables.
+ ILogicalExpression rewrittenCondition =
+ rewriteFieldAccess(conditionRef.getValue(), fieldToNewVar, searchInfo.recordType());
+
+ // Completeness guard: the pushed condition must reference ONLY the freshly-created INCLUDE
+ // variables. The earlier checks act on field names that extractFieldNames recognizes; an
+ // access pattern it does not recognize, or a non-inlinable ASSIGN output, can leave a
+ // reference to the source record (produced ABOVE the vector unnest-map) in the condition.
+ // Embedding such a condition would reference a variable that is not in scope at the unnest --
+ // an invalid plan. Bail and leave the SELECT untouched rather than emit a broken plan.
+ Set<LogicalVariable> rewrittenUsed = new HashSet<>();
+ rewrittenCondition.getUsedVariables(rewrittenUsed);
+ if (!new HashSet<>(fieldToNewVar.values()).containsAll(rewrittenUsed)) {
+ return false;
+ }
+
+ // Hand the mappings to VectorSearchPOperator so it can construct the filter schema.
+ searchInfo.vectorUnnest().getAnnotations().put(VECTOR_FILTER_VAR_MAPPING, filterVarToFieldIndex);
+ searchInfo.vectorUnnest().getAnnotations().put(VECTOR_FILTER_VAR_TYPES, filterVarToType);
+
+ // Register filter variables as produced by the unnest-map so sanity checks recognize them.
+ for (Map.Entry<String, LogicalVariable> entry : fieldToNewVar.entrySet()) {
+ LogicalVariable var = entry.getValue();
+ IAType type = filterVarToType.get(var);
+ searchInfo.vectorUnnest().getVariables().add(var);
+ searchInfo.vectorUnnest().getVariableTypes().add(type);
+ }
+
+ searchInfo.vectorUnnest().setSelectCondition(new MutableObject<>(rewrittenCondition));
+
+ // Drop the SELECT now that the predicate has been embedded in the vector index search.
+ opRef.setValue(selectOp.getInputs().get(0).getValue());
+
+ context.addToDontApplySet(this, op);
+
+ context.computeAndSetTypeEnvironmentForOperator(searchInfo.vectorUnnest());
+ OperatorPropertiesUtil.typeOpRec(opRef, context);
+
+ return true;
+ }
+
+ /**
+ * Information about a vector index search found in the plan.
+ */
+ private record VectorSearchInfo(UnnestMapOperator vectorUnnest, List<List<String>> includeFieldNames,
+ ARecordType recordType, boolean isQuantized, int numPrimaryKeys) {
+ }
+
+ /**
+ * Finds VECTOR_INDEX_UNNEST below the SELECT operator, skipping intervening ASSIGNs.
+ */
+ private VectorSearchInfo findVectorIndexUnnest(SelectOperator selectOp, IOptimizationContext context)
+ throws AlgebricksException {
+ ILogicalOperator current = selectOp.getInputs().get(0).getValue();
+ while (current.getOperatorTag() == LogicalOperatorTag.ASSIGN) {
+ current = current.getInputs().get(0).getValue();
+ }
+ return searchForVectorUnnest(current, context);
+ }
+
+ /**
+ * Recursively searches for a VECTOR_INDEX_UNNEST under the given operator.
+ */
+ private VectorSearchInfo searchForVectorUnnest(ILogicalOperator op, IOptimizationContext context)
+ throws AlgebricksException {
+
+ if (op.getOperatorTag() == LogicalOperatorTag.UNNEST_MAP) {
+ UnnestMapOperator unnest = (UnnestMapOperator) op;
+ ILogicalExpression expr = unnest.getExpressionRef().getValue();
+
+ if (expr.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
+ AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) expr;
+
+ if (funcExpr.getFunctionIdentifier().equals(BuiltinFunctions.INDEX_SEARCH)) {
+ AccessMethodJobGenParams params = new AccessMethodJobGenParams();
+ params.readFromFuncArgs(funcExpr.getArguments());
+
+ if (params.getIndexType() == IndexType.VTREE) {
+ if (unnest.getSelectCondition() != null) {
+ return null;
+ }
+ return buildSearchInfo(unnest, params, context);
+ }
+ }
+ }
+ }
+
+ for (Mutable<ILogicalOperator> inputRef : op.getInputs()) {
+ VectorSearchInfo result = searchForVectorUnnest(inputRef.getValue(), context);
+ if (result != null) {
+ return result;
+ }
+ }
+
+ return null;
+ }
+
+ /**
+ * Builds VectorSearchInfo from the found vector index.
+ */
+ @AiProvenance(agent = AiProvenance.Agent.CLAUDE_FABLE_5, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.ASSISTED)
+ private VectorSearchInfo buildSearchInfo(UnnestMapOperator unnest, AccessMethodJobGenParams params,
+ IOptimizationContext context) throws AlgebricksException {
+
+ MetadataProvider mp = (MetadataProvider) context.getMetadataProvider();
+
+ Dataset dataset = mp.findDataset(params.getDatabaseName(), params.getDataverseName(), params.getDatasetName());
+ if (dataset == null) {
+ return null;
+ }
+
+ Index index = mp.getIndex(params.getDatabaseName(), params.getDataverseName(), params.getDatasetName(),
+ params.getIndexName());
+ if (index == null || index.getIndexType() != IndexType.VTREE) {
+ return null;
+ }
+
+ Index.VectorIndexDetails details = (Index.VectorIndexDetails) index.getIndexDetails();
+
+ // Determine quantization from WITH clause quantization field
+ AdmObjectNode withObjectNode = details.getWithObjectNode();
+ String quantization = (withObjectNode != null) ? withObjectNode.getOptionalString("quantization", null) : null;
+
+ ARecordType recordType = (ARecordType) mp.findType(dataset.getItemTypeDatabaseName(),
+ dataset.getItemTypeDataverseName(), dataset.getItemTypeName());
+
+ return new VectorSearchInfo(unnest, details.getIncludeFieldNames(), recordType, quantization != null,
+ dataset.getPrimaryKeys().size());
+ }
+
+ /**
+ * Extracts variables from ASSIGN that can be inlined.
+ */
+ private void extractInlinableVariables(AssignOperator assignOp, Set<LogicalVariable> targetVars,
+ Map<LogicalVariable, ILogicalExpression> outMap) {
+ List<LogicalVariable> vars = assignOp.getVariables();
+ List<Mutable<ILogicalExpression>> exprs = assignOp.getExpressions();
+
+ for (int i = 0; i < vars.size(); i++) {
+ LogicalVariable var = vars.get(i);
+ if (targetVars.contains(var)) {
+ ILogicalExpression expr = exprs.get(i).getValue();
+ if (expr.isFunctional()) {
+ outMap.put(var, expr);
+ }
+ }
+ }
+ }
+
+ /**
+ * Builds a map from field name to its index in the INCLUDE list.
+ */
+ private Map<String, Integer> buildIncludeFieldIndex(List<List<String>> includeFieldNames) {
+ Map<String, Integer> result = new HashMap<>();
+ for (int i = 0; i < includeFieldNames.size(); i++) {
+ List<String> path = includeFieldNames.get(i);
+ String fieldName = path.get(path.size() - 1);
+ result.put(fieldName, i);
+ }
+ return result;
+ }
+
+ /**
+ * Extracts field names from field-access expressions in the filter.
+ */
+ private void extractFieldNames(ILogicalExpression expr, ARecordType recordType, Set<String> fieldNames) {
+ if (expr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
+ return;
+ }
+
+ AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) expr;
+ FunctionIdentifier fid = funcExpr.getFunctionIdentifier();
+
+ if (fid.equals(BuiltinFunctions.FIELD_ACCESS_BY_NAME)) {
+ if (funcExpr.getArguments().size() >= 2) {
+ String fieldName = AccessMethodUtils.getStringConstant(funcExpr.getArguments().get(1));
+ if (fieldName != null) {
+ fieldNames.add(fieldName);
+ }
+ }
+ } else if (fid.equals(BuiltinFunctions.FIELD_ACCESS_BY_INDEX)) {
+ if (funcExpr.getArguments().size() >= 2 && recordType != null) {
+ Integer idx = AccessMethodUtils.getInt32Constant(funcExpr.getArguments().get(1));
+ if (idx != null) {
+ String[] names = recordType.getFieldNames();
+ if (idx >= 0 && idx < names.length) {
+ fieldNames.add(names[idx]);
+ }
+ }
+ }
+ }
+
+ // Recurse into arguments
+ for (Mutable<ILogicalExpression> arg : funcExpr.getArguments()) {
+ extractFieldNames(arg.getValue(), recordType, fieldNames);
+ }
+ }
+
+ /**
+ * Rewrites field-access expressions to use new INCLUDE field variables.
+ * Example: gt($row.getField(2), 2000) -> gt($year, 2000)
+ */
+ private ILogicalExpression rewriteFieldAccess(ILogicalExpression expr, Map<String, LogicalVariable> fieldToVar,
+ ARecordType recordType) {
+
+ if (expr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
+ return expr;
+ }
+
+ AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) expr;
+ FunctionIdentifier fid = funcExpr.getFunctionIdentifier();
+
+ // Check if this is a field access to replace
+ String fieldName = null;
+
+ if (fid.equals(BuiltinFunctions.FIELD_ACCESS_BY_NAME)) {
+ if (funcExpr.getArguments().size() >= 2) {
+ fieldName = AccessMethodUtils.getStringConstant(funcExpr.getArguments().get(1));
+ }
+ } else if (fid.equals(BuiltinFunctions.FIELD_ACCESS_BY_INDEX)) {
+ if (funcExpr.getArguments().size() >= 2 && recordType != null) {
+ Integer idx = AccessMethodUtils.getInt32Constant(funcExpr.getArguments().get(1));
+ if (idx != null) {
+ String[] names = recordType.getFieldNames();
+ if (idx >= 0 && idx < names.length) {
+ fieldName = names[idx];
+ }
+ }
+ }
+ }
+
+ // Replace field access with variable reference
+ if (fieldName != null && fieldToVar.containsKey(fieldName)) {
+ LogicalVariable newVar = fieldToVar.get(fieldName);
+ VariableReferenceExpression varRef = new VariableReferenceExpression(newVar);
+ varRef.setSourceLocation(funcExpr.getSourceLocation());
+ return varRef;
+ }
+
+ // Recurse into arguments
+ List<Mutable<ILogicalExpression>> newArgs = new ArrayList<>();
+ boolean changed = false;
+
+ for (Mutable<ILogicalExpression> argRef : funcExpr.getArguments()) {
+ ILogicalExpression newArg = rewriteFieldAccess(argRef.getValue(), fieldToVar, recordType);
+ newArgs.add(new MutableObject<>(newArg));
+ if (newArg != argRef.getValue()) {
+ changed = true;
+ }
+ }
+
+ if (changed) {
+ ScalarFunctionCallExpression newFunc =
+ new ScalarFunctionCallExpression(funcExpr.getFunctionInfo(), newArgs);
+ newFunc.setSourceLocation(funcExpr.getSourceLocation());
+ return newFunc;
+ }
+
+ return expr;
+ }
+}
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetAsterixPhysicalOperatorsRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetAsterixPhysicalOperatorsRule.java
index 9f69d17f66..d7ff177008 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetAsterixPhysicalOperatorsRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetAsterixPhysicalOperatorsRule.java
@@ -28,6 +28,7 @@ import org.apache.asterix.algebra.operators.physical.AssignBatchPOperator;
import org.apache.asterix.algebra.operators.physical.BTreeSearchPOperator;
import org.apache.asterix.algebra.operators.physical.InvertedIndexPOperator;
import org.apache.asterix.algebra.operators.physical.RTreeSearchPOperator;
+import org.apache.asterix.algebra.operators.physical.VectorSearchPOperator;
import org.apache.asterix.common.config.DatasetConfig.IndexType;
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.exceptions.ErrorCode;
@@ -368,6 +369,9 @@ public class SetAsterixPhysicalOperatorsRule extends SetAlgebricksPhysicalOperat
case RTREE: {
return new RTreeSearchPOperator(dsi, storageDomain, requiresBroadcast);
}
+ case VTREE: {
+ return new VectorSearchPOperator(dsi, storageDomain, requiresBroadcast);
+ }
case SINGLE_PARTITION_WORD_INVIX:
case SINGLE_PARTITION_NGRAM_INVIX: {
return new InvertedIndexPOperator(dsi, storageDomain, requiresBroadcast, false);
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java
index c36fc0bda9..d8c0f0c257 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java
@@ -18,6 +18,8 @@
*/
package org.apache.asterix.optimizer.rules.am;
+import static org.apache.asterix.om.types.AOrderedListType.FULL_OPEN_ORDEREDLIST_TYPE;
+
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -1008,6 +1010,15 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew
keySources = textIndexDetails.getKeyFieldSourceIndicators();
isOverridingKeyFieldTypes = textIndexDetails.isOverridingKeyFieldTypes();
break;
+ case VTREE:
+ Index.VectorIndexDetails vectorIndexDetails = (Index.VectorIndexDetails) index.getIndexDetails();
+ keyFieldNames = vectorIndexDetails.getKeyFieldNames();
+ keyFieldTypes = new ArrayList<>();
+ keyFieldTypes.add(FULL_OPEN_ORDEREDLIST_TYPE);
+ keySources = null;
+ isOverridingKeyFieldTypes = vectorIndexDetails.isOverridingKeyFieldTypes();
+ break;
+
default:
throw new CompilationException(ErrorCode.COMPILATION_UNKNOWN_INDEX_TYPE,
String.valueOf(index.getIndexType()));
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
index 8c854dab99..cbc7fe4a11 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
@@ -442,8 +442,9 @@ public class AccessMethodUtils {
*/
public static void appendSecondaryIndexTypes(Dataset dataset, ARecordType recordType, ARecordType metaRecordType,
Index index, List<Object> dest) throws AlgebricksException {
- // In case of an inverted-index search, secondary keys will not be generated.
- boolean primaryKeysOnly = isInvertedIndex(index);
+ // In case of an inverted-index or vector-index search, secondary keys will not be generated.
+ // For vector indexes, this avoids transferring large embeddings when only PKs are needed.
+ boolean primaryKeysOnly = isInvertedIndex(index) || index.getIndexType() == IndexType.VTREE;
if (!primaryKeysOnly) {
switch (index.getIndexType()) {
case ARRAY:
@@ -464,6 +465,9 @@ public class AccessMethodUtils {
case RTREE:
dest.addAll(KeyFieldTypeUtil.getRTreeIndexKeyTypes(index, recordType, metaRecordType));
break;
+ case VTREE:
+ dest.addAll(KeyFieldTypeUtil.getVectorIndexKeyTypes(index, recordType, metaRecordType));
+ break;
default:
break;
}
@@ -481,6 +485,10 @@ public class AccessMethodUtils {
/**
* Creates output variables for the given unnest-map or left-outer-unnestmap operator
* that does a secondary index lookup.
+ * The order:
+ * - For most indexes: SK, PK
+ * - For inverted/vector indexes: PK
+ * Vector and inverted indexes skip SK output to avoid materializing unnecessary data.
*/
public static void appendSecondaryIndexOutputVars(Dataset dataset, ARecordType recordType,
ARecordType metaRecordType, Index index, IOptimizationContext context, List<LogicalVariable> dest)
@@ -493,8 +501,10 @@ public class AccessMethodUtils {
numPrimaryKeys = dataset.getPrimaryKeys().size();
}
int numSecondaryKeys = KeyFieldTypeUtil.getNumSecondaryKeys(index, recordType, metaRecordType);
- // In case of an inverted-index search, secondary keys will not be generated.
- int numVars = isInvertedIndex(index) ? numPrimaryKeys : numPrimaryKeys + numSecondaryKeys;
+ // In case of an inverted-index or vector-index search, secondary keys will not be generated.
+ // For vector indexes, this is a significant optimization since embeddings are very large (1024+ floats).
+ boolean skipSecondaryKeys = isInvertedIndex(index) || index.getIndexType() == IndexType.VTREE;
+ int numVars = skipSecondaryKeys ? numPrimaryKeys : numPrimaryKeys + numSecondaryKeys;
for (int i = 0; i < numVars; i++) {
dest.add(context.newVar());
@@ -504,6 +514,9 @@ public class AccessMethodUtils {
/**
* Gets the primary key variables from the unnest-map or left-outer-unnest-map operator
* that does a secondary index lookup.
+ * The order:
+ * - For most indexes: SK, PK
+ * - For inverted/vector indexes: PK
*/
public static List<LogicalVariable> getKeyVarsFromSecondaryUnnestMap(Dataset dataset, ARecordType recordType,
ARecordType metaRecordType, ILogicalOperator unnestMapOp, Index index,
@@ -524,9 +537,9 @@ public class AccessMethodUtils {
int stop;
// If a secondary-index search didn't generate SKs, set it to zero.
- // Currently, only an inverted-index search doesn't generate any SKs.
- boolean isNgramOrKeywordIndex = isInvertedIndex(index);
- if (isNgramOrKeywordIndex) {
+ // Currently, inverted-index and vector-index searches don't generate any SKs.
+ boolean skipSecondaryKeys = isInvertedIndex(index) || index.getIndexType() == IndexType.VTREE;
+ if (skipSecondaryKeys) {
numSecondaryKeys = 0;
}
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceTopKAccessMethodRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceTopKAccessMethodRule.java
new file mode 100644
index 0000000000..56736fb194
--- /dev/null
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceTopKAccessMethodRule.java
@@ -0,0 +1,1017 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.optimizer.rules.am;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+
+import org.apache.asterix.common.annotations.AnnSearchPreferenceAnnotation;
+import org.apache.asterix.common.config.DatasetConfig.IndexType;
+import org.apache.asterix.metadata.declared.IIndexProvider;
+import org.apache.asterix.metadata.declared.MetadataProvider;
+import org.apache.asterix.metadata.entities.Index;
+import org.apache.asterix.om.base.AString;
+import org.apache.asterix.om.base.IAObject;
+import org.apache.asterix.om.constants.AsterixConstantValue;
+import org.apache.asterix.om.functions.BuiltinFunctions;
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.common.utils.Pair;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import org.apache.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
+import org.apache.hyracks.algebricks.core.algebra.expressions.IAlgebricksConstantValue;
+import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import org.apache.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractScanOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import org.apache.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+/**
+ * Optimization rule for introducing vector index access for top-k ANN queries.
+ *
+ * Pattern: LIMIT k → ORDER BY ANN_DISTANCE(vectorField, queryVector, metric) → ... → DATASOURCE_SCAN
+ *
+ * Transformation:
+ * - If vector index exists: Replaces DATASOURCE_SCAN with UNNEST-MAP(vector_index_search)
+ * → Returns candidate tuples for approximate ANN search
+ * - If no vector index: Leaves plan unchanged
+ * → Falls back to exact KNN search (exhaustive distance computation on all tuples)
+ *
+ * The ORDER BY ANN_DISTANCE operator handles distance computation in both cases.
+ */
+public class IntroduceTopKAccessMethodRule extends AbstractIntroduceAccessMethodRule {
+
+ private static final Logger LOGGER = LogManager.getLogger();
+
+ // Operators representing the pattern to be matched
+ protected Mutable<ILogicalOperator> limitRef = null;
+ protected LimitOperator limitOp = null;
+ protected Mutable<ILogicalOperator> orderRef = null;
+ protected OrderOperator orderOp = null;
+ protected AbstractFunctionCallExpression annDistanceExpr = null;
+ protected IVariableTypeEnvironment typeEnvironment = null;
+ protected final OptimizableOperatorSubTree subTree = new OptimizableOperatorSubTree();
+ protected String queryDistanceMetric = null; // Distance metric from the query (e.g., "euclidean", "cosine")
+
+ // SELECT operator info for filter pushdown
+ protected SelectOperator selectOp = null;
+ protected Set<List<String>> filterFieldNames = null; // Field names referenced in WHERE clause
+
+ /**
+ * Master switch for the index-only ANN plan optimization. Enabled by default: when the projection above
+ * the LIMIT is PK-only, the plan emits (pk, dist) directly from the secondary VTree and skips the
+ * primary BTree lookup + rerank. The plan rewrite handles above-LIMIT query-parameter ASSIGNs (see
+ * {@link #isProjectionPkOnly}), and the secondary VTree reconciles anti-matter (delete) tuples itself,
+ * so a PK-only ANN query over a dataset with deletes returns no deleted PKs.
+ */
+ private static final boolean INDEX_ONLY_ENABLED = true;
+
+ /**
+ * Ancestor chain from the rewrite-pre entry point down to (but not including) the matched
+ * {@code limitOp}. Populated by {@link #checkAndApplyTopKTransformation} as it recurses; consumed
+ * by {@link #isProjectionPkOnly} to gather variables used above the LIMIT.
+ */
+ protected final List<AbstractLogicalOperator> aboveLimitOps = new ArrayList<>();
+
+ // Register vector index access method
+ protected static Map<FunctionIdentifier, List<IAccessMethod>> accessMethods = new HashMap<>();
+
+ static {
+ registerAccessMethod(VectorIndexAccessMethod.INSTANCE, accessMethods);
+ }
+
+ @Override
+ public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+ throws AlgebricksException {
+ clear();
+ setMetadataIndexDeclarations(context, (IIndexProvider) context.getMetadataProvider());
+
+ AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+
+ // Already checked?
+ if (context.checkIfInDontApplySet(this, op)) {
+ return false;
+ }
+
+ // Start from root operators
+ if (op.getOperatorTag() != LogicalOperatorTag.DISTRIBUTE_RESULT
+ && op.getOperatorTag() != LogicalOperatorTag.SINK
+ && op.getOperatorTag() != LogicalOperatorTag.DELEGATE_OPERATOR) {
+ return false;
+ }
+
+ // Recursively find pattern: LIMIT → ORDER BY ANN_DISTANCE → ... → DATASOURCE_SCAN
+ boolean planTransformed = checkAndApplyTopKTransformation(opRef, context);
+
+ if (planTransformed) {
+ OperatorPropertiesUtil.typeOpRec(opRef, context);
+ }
+
+ return planTransformed;
+ }
+
+ /**
+ * Recursively checks the plan for LIMIT → ORDER BY ANN_DISTANCE pattern
+ * and applies vector index optimization if applicable.
+ */
+ protected boolean checkAndApplyTopKTransformation(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+ throws AlgebricksException {
+
+ AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+
+ // Check if current operator is LIMIT
+ if (op.getOperatorTag() == LogicalOperatorTag.LIMIT) {
+ limitRef = opRef;
+ limitOp = (LimitOperator) op;
+
+ if (context.checkIfInDontApplySet(this, limitOp)) {
+ return false;
+ }
+
+ // Find ORDER operator by skipping intermediate operators (ASSIGN, EXCHANGE, nested LIMIT)
+ Pair<Mutable<ILogicalOperator>, OrderOperator> orderPair = findOrderOperator(limitOp);
+ if (orderPair != null) {
+ orderRef = orderPair.first;
+ orderOp = orderPair.second;
+ // Skip the index transform when the LIMIT carries an OFFSET: the vector search is sized
+ // from k alone, so LIMIT k OFFSET o would skip into an undersized candidate pool and return
+ // truncated/empty results. Fall through to the exact ORDER BY ... LIMIT plan, which honors
+ // OFFSET correctly. (A future enhancement could size the search to (k + offset).)
+ if (!limitOp.hasOffset() && matchesAnnDistancePattern()) {
+ return analyzeAndTransform(context);
+ }
+ }
+ }
+
+ // Recursively check children — push self onto the ancestor chain so that, when we reach a
+ // LIMIT below, isProjectionPkOnly() has the full list of operators above it.
+ aboveLimitOps.add(op);
+ try {
+ for (Mutable<ILogicalOperator> inputOpRef : op.getInputs()) {
+ boolean transformed = checkAndApplyTopKTransformation(inputOpRef, context);
+ if (transformed) {
+ return true;
+ }
+ }
+ } finally {
+ aboveLimitOps.remove(aboveLimitOps.size() - 1);
+ }
+
+ return false;
+ }
+
+ /**
+ * Finds the ORDER operator by traversing through intermediate operators.
+ * Skips ASSIGN, EXCHANGE, and nested LIMIT operators that are added for
+ * distributed execution optimization.
+ *
+ * The optimizer often adds intermediate operators between LIMIT and ORDER:
+ * - ASSIGN: For result projection
+ * - EXCHANGE: For data redistribution in distributed execution
+ * - LIMIT: For distributed top-k optimization
+ *
+ * This method traverses through these intermediate operators until it finds
+ * the ORDER operator or encounters an operator it doesn't recognize.
+ *
+ * @param limitOp The LIMIT operator to start from
+ * @return Pair of (orderRef, orderOp) if found, null otherwise
+ */
+ protected Pair<Mutable<ILogicalOperator>, OrderOperator> findOrderOperator(LimitOperator limitOp) {
+ if (limitOp.getInputs().isEmpty()) {
+ return null;
+ }
+
+ Mutable<ILogicalOperator> currentRef = limitOp.getInputs().get(0);
+ AbstractLogicalOperator currentOp = (AbstractLogicalOperator) currentRef.getValue();
+
+ // Traverse through intermediate operators until we find ORDER or fail
+ // The loop will terminate when:
+ // 1. We find ORDER operator (success)
+ // 2. We hit an empty input list (no more children)
+ // 3. We hit an operator we don't know how to skip through
+ while (true) {
+ // Check if we found the ORDER operator
+ if (currentOp.getOperatorTag() == LogicalOperatorTag.ORDER) {
+ return new Pair<>(currentRef, (OrderOperator) currentOp);
+ }
+
+ // Skip through known intermediate operators
+ if (currentOp.getOperatorTag() == LogicalOperatorTag.ASSIGN
+ || currentOp.getOperatorTag() == LogicalOperatorTag.EXCHANGE
+ || currentOp.getOperatorTag() == LogicalOperatorTag.LIMIT) {
+
+ if (currentOp.getInputs().isEmpty()) {
+ // No more children to traverse
+ return null;
+ }
+ currentRef = currentOp.getInputs().get(0);
+ currentOp = (AbstractLogicalOperator) currentRef.getValue();
+ } else {
+ // Hit an operator we don't know how to skip through
+ // This means LIMIT is not directly above ORDER BY ANN_DISTANCE
+ return null;
+ }
+ }
+ }
+
+ /**
+ * Checks if ORDER BY pattern exists (has at least one ordering expression).
+ * We don't resolve the actual ANN_DISTANCE function here - that happens later
+ * after subtree initialization in analyzeAnnDistanceFunction().
+ */
+ protected boolean matchesAnnDistancePattern() {
+ List<Pair<IOrder, Mutable<ILogicalExpression>>> orderExprs = orderOp.getOrderExpressions();
+
+ // Just check that ORDER BY has exactly one expression
+ // We'll verify it's ANN_DISTANCE later after subtree init
+ return orderExprs.size() == 1;
+ }
+
+ /**
+ * Analyzes the pattern and attempts to apply vector index transformation.
+ */
+ protected boolean analyzeAndTransform(IOptimizationContext context) throws AlgebricksException {
+ // 1. Initialize subtree from ORDER down to DATASOURCE_SCAN
+ if (!initializeSubTree()) {
+ return false;
+ }
+
+ // 2. Get type environment for type checking
+ typeEnvironment = context.getOutputTypeEnvironment(orderOp);
+
+ // 3. Load dataset metadata (including vector indexes)
+ // This MUST be done before extracting filter fields because field-access-by-index
+ // needs the record type to resolve field index to field name
+ MetadataProvider metadataProvider = (MetadataProvider) context.getMetadataProvider();
+ if (!subTree.setDatasetAndTypeMetadata(metadataProvider)) {
+ return false;
+ }
+
+ // 4. Find SELECT operator (if any) and extract filter fields.
+ // Populates selectOp and filterFieldNames if a SELECT exists.
+ // Must be called AFTER setDatasetAndTypeMetadata so recordType is available.
+ findSelectOperatorInSubTree();
+
+ // 5. Analyze ANN_DISTANCE function arguments
+ Map<IAccessMethod, AccessMethodAnalysisContext> analyzedAMs = new TreeMap<>();
+ if (!analyzeAnnDistanceFunction(analyzedAMs, context)) {
+ return false;
+ }
+
+ // 6. Find applicable vector indexes on the dataset
+ fillSubTreeIndexExprs(subTree, analyzedAMs, context, false);
+
+ // 7. Choose best vector index (considering INCLUDE fields if filter exists)
+ List<Pair<IAccessMethod, Index>> chosenIndexes = new ArrayList<>();
+ chooseVectorIndex(analyzedAMs, chosenIndexes);
+
+ if (chosenIndexes.isEmpty()) {
+ // No vector index available (either none exists, or none has required INCLUDE fields).
+ // Fall back to data scan + sort.
+ context.addToDontApplySet(this, limitOp);
+ return false;
+ }
+
+ // 8. Apply plan transformation
+ Index vectorIndex = chosenIndexes.get(0).second;
+ AccessMethodAnalysisContext analysisCtx = analyzedAMs.get(VectorIndexAccessMethod.INSTANCE);
+
+ boolean transformed = applyTopKPlanTransformation(vectorIndex, analysisCtx, context);
+
+ // Always mark as processed to avoid re-attempting optimization on this operator
+ context.addToDontApplySet(this, limitOp);
+
+ return transformed;
+ }
+
+ /**
+ * Finds SELECT operator between ORDER and DATASOURCE_SCAN.
+ * Also extracts filter field names from the SELECT condition.
+ *
+ * @return The SELECT operator if found, null otherwise
+ */
+ protected SelectOperator findSelectOperatorInSubTree() {
+ if (orderOp.getInputs().isEmpty()) {
+ return null;
+ }
+
+ // Traverse from ORDER down to DATASOURCE_SCAN
+ AbstractLogicalOperator currentOp = (AbstractLogicalOperator) orderOp.getInputs().get(0).getValue();
+
+ while (currentOp != null) {
+ if (currentOp.getOperatorTag() == LogicalOperatorTag.SELECT) {
+ selectOp = (SelectOperator) currentOp;
+ filterFieldNames = extractFilterFieldsFromCondition(selectOp.getCondition().getValue());
+ return selectOp;
+ }
+
+ if (currentOp.getOperatorTag() == LogicalOperatorTag.DATASOURCESCAN) {
+ break;
+ }
+ if (currentOp.getInputs().isEmpty()) {
+ break;
+ }
+ currentOp = (AbstractLogicalOperator) currentOp.getInputs().get(0).getValue();
+ }
+
+ return null;
+ }
+
+ /**
+ * Extracts field names referenced in a filter condition expression.
+ * Traverses function call expressions to find field access operations.
+ *
+ * @param condition The filter condition expression
+ * @return Set of field names (as List<String> for nested field paths)
+ */
+ protected Set<List<String>> extractFilterFieldsFromCondition(ILogicalExpression condition) {
+ Set<List<String>> fields = new HashSet<>();
+ extractFieldsFromExpressionRecursive(condition, fields);
+ return fields;
+ }
+
+ /**
+ * Recursively extracts field names from an expression.
+ * Handles function calls (AND, OR, comparison operators, field-access).
+ */
+ private void extractFieldsFromExpressionRecursive(ILogicalExpression expr, Set<List<String>> fields) {
+ if (expr.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
+ AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) expr;
+
+ // Check if this is a field access function (e.g., field-access-by-name)
+ if (funcExpr.getFunctionIdentifier().equals(BuiltinFunctions.FIELD_ACCESS_BY_NAME)
+ || funcExpr.getFunctionIdentifier().equals(BuiltinFunctions.FIELD_ACCESS_BY_INDEX)) {
+ // Extract field name from field access function
+ List<String> fieldPath = extractFieldPathFromFieldAccess(funcExpr);
+ if (fieldPath != null && !fieldPath.isEmpty()) {
+ fields.add(fieldPath);
+ }
+ } else {
+ // Recursively process arguments for other functions (AND, OR, GT, LT, etc.)
+ for (Mutable<ILogicalExpression> arg : funcExpr.getArguments()) {
+ extractFieldsFromExpressionRecursive(arg.getValue(), fields);
+ }
+ }
+ } else if (expr.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
+ // Variable reference - trace back through assigns to find field access.
+ // First check subTree assigns; if not found, scan from ORDER down to DATASOURCE_SCAN.
+ VariableReferenceExpression varRef = (VariableReferenceExpression) expr;
+ LogicalVariable var = varRef.getVariableReference();
+ boolean found = findFieldFromAssigns(var, subTree.getAssignsAndUnnests(), fields);
+ if (!found && orderOp != null) {
+ searchAssignsInPlan(var, orderOp, fields);
+ }
+ }
+ }
+
+ /**
+ * Searches for variable definition in a list of assigns and extracts field info.
+ */
+ private boolean findFieldFromAssigns(LogicalVariable var, List<AbstractLogicalOperator> assigns,
+ Set<List<String>> fields) {
+ for (AbstractLogicalOperator op : assigns) {
+ if (op.getOperatorTag() == LogicalOperatorTag.ASSIGN) {
+ AssignOperator assignOp = (AssignOperator) op;
+ List<LogicalVariable> assignVars = assignOp.getVariables();
+ List<Mutable<ILogicalExpression>> assignExprs = assignOp.getExpressions();
+
+ for (int i = 0; i < assignVars.size(); i++) {
+ if (assignVars.get(i).equals(var)) {
+ extractFieldsFromExpressionRecursive(assignExprs.get(i).getValue(), fields);
+ return true;
+ }
+ }
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Searches all operators in the plan from the given operator downward for variable definition.
+ */
+ private boolean searchAssignsInPlan(LogicalVariable var, ILogicalOperator startOp, Set<List<String>> fields) {
+ ILogicalOperator currentOp = startOp;
+
+ while (currentOp != null) {
+ if (currentOp.getOperatorTag() == LogicalOperatorTag.ASSIGN) {
+ AssignOperator assignOp = (AssignOperator) currentOp;
+ List<LogicalVariable> assignVars = assignOp.getVariables();
+ List<Mutable<ILogicalExpression>> assignExprs = assignOp.getExpressions();
+
+ for (int i = 0; i < assignVars.size(); i++) {
+ if (assignVars.get(i).equals(var)) {
+ // Found the assignment - recursively extract fields
+ extractFieldsFromExpressionRecursive(assignExprs.get(i).getValue(), fields);
+ return true;
+ }
+ }
+ }
+
+ // Move to next operator
+ if (currentOp.getInputs().isEmpty()) {
+ break;
+ }
+ currentOp = currentOp.getInputs().get(0).getValue();
+ }
+ return false;
+ }
+
+ /**
+ * Extracts field path from a field-access function expression.
+ * For nested fields like row.nested.field, returns ["nested", "field"].
+ */
+ private List<String> extractFieldPathFromFieldAccess(AbstractFunctionCallExpression funcExpr) {
+ List<String> fieldPath = new ArrayList<>();
+ extractFieldPathRecursive(funcExpr, fieldPath);
+ return fieldPath;
+ }
+
+ /**
+ * Recursively builds field path from nested field access expressions.
+ */
+ private void extractFieldPathRecursive(ILogicalExpression expr, List<String> fieldPath) {
+ if (expr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
+ return;
+ }
+
+ AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) expr;
+
+ if (funcExpr.getFunctionIdentifier().equals(BuiltinFunctions.FIELD_ACCESS_BY_NAME)) {
+ // First argument is the record, second argument is the field name
+ if (funcExpr.getArguments().size() >= 2) {
+ // Recursively process the record argument (for nested field access)
+ extractFieldPathRecursive(funcExpr.getArguments().get(0).getValue(), fieldPath);
+
+ // Extract field name from second argument
+ ILogicalExpression fieldNameExpr = funcExpr.getArguments().get(1).getValue();
+ String fieldName = AccessMethodUtils.getStringConstant(new MutableObject<>(fieldNameExpr));
+ if (fieldName != null) {
+ fieldPath.add(fieldName);
+ }
+ }
+ } else if (funcExpr.getFunctionIdentifier().equals(BuiltinFunctions.FIELD_ACCESS_BY_INDEX)) {
+ // First argument is the record, second argument is the field index
+ if (funcExpr.getArguments().size() >= 2) {
+ // Recursively process the record argument (for nested field access)
+ extractFieldPathRecursive(funcExpr.getArguments().get(0).getValue(), fieldPath);
+
+ // Extract field index from second argument and look up field name
+ ILogicalExpression fieldIndexExpr = funcExpr.getArguments().get(1).getValue();
+ Integer fieldIndex = AccessMethodUtils.getInt32Constant(new MutableObject<>(fieldIndexExpr));
+ if (fieldIndex != null && subTree.getRecordType() != null) {
+ String[] fieldNames = subTree.getRecordType().getFieldNames();
+ if (fieldIndex >= 0 && fieldIndex < fieldNames.length) {
+ fieldPath.add(fieldNames[fieldIndex]);
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Checks if a vector index has all required filter fields in its INCLUDE fields.
+ *
+ * @param index The vector index to check
+ * @param filterFields The set of field names referenced in the filter
+ * @return true if all filter fields are in the index's INCLUDE fields
+ */
+ protected boolean indexHasIncludeFields(Index index, Set<List<String>> filterFields) {
+ if (filterFields == null || filterFields.isEmpty()) {
+ // No filter fields - index can be used
+ return true;
+ }
+
+ if (index.getIndexType() != IndexType.VTREE) {
+ return false;
+ }
+
+ Index.VectorIndexDetails vectorDetails = (Index.VectorIndexDetails) index.getIndexDetails();
+ List<List<String>> includeFieldNames = vectorDetails.getIncludeFieldNames();
+
+ if (includeFieldNames == null || includeFieldNames.isEmpty()) {
+ return false;
+ }
+
+ // Check if all filter fields are present in INCLUDE fields.
+ for (List<String> filterField : filterFields) {
+ boolean found = false;
+ for (List<String> includeField : includeFieldNames) {
+ if (includeField.equals(filterField)) {
+ found = true;
+ break;
+ }
+ }
+ if (!found) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Initializes the subtree from ORDER operator's child down to DATASOURCE_SCAN.
+ * OptimizableOperatorSubTree doesn't recognize ORDER as a valid top operator,
+ * so we start from ORDER's child (usually ASSIGN).
+ */
+ protected boolean initializeSubTree() throws AlgebricksException {
+ if (orderOp.getInputs().isEmpty()) {
+ return false;
+ }
+ subTree.initFromSubTree(orderOp.getInputs().get(0));
+ return subTree.hasDataSourceScan();
+ }
+
+ /**
+ * Analyzes ANN_DISTANCE function and updates analysis context.
+ * This is called AFTER subtree initialization, so assigns/unnests are available
+ * for resolving variable references.
+ */
+ protected boolean analyzeAnnDistanceFunction(Map<IAccessMethod, AccessMethodAnalysisContext> analyzedAMs,
+ IOptimizationContext context) throws AlgebricksException {
+
+ // Get ORDER BY expression
+ List<Pair<IOrder, Mutable<ILogicalExpression>>> orderExprs = orderOp.getOrderExpressions();
+ ILogicalExpression orderExpr = orderExprs.get(0).second.getValue();
+
+ // Resolve to actual ANN_DISTANCE function (handle both direct and variable reference cases)
+ annDistanceExpr = resolveAnnDistanceExpr(orderExpr, subTree.getAssignsAndUnnests());
+
+ if (annDistanceExpr == null) {
+ // ORDER BY expression is not ANN_DISTANCE
+ return false;
+ }
+
+ // The top-K cursor returns the nearest K candidates (ascending distance). A DESC order-by asks
+ // for the farthest K, which this plan cannot produce, so only match ASC and let DESC fall back
+ // to the non-index (full-scan + sort) plan.
+ if (orderExprs.get(0).first.getKind() != IOrder.OrderKind.ASC) {
+ return false;
+ }
+
+ // The distance metric now comes from the ANN hint: the desugared call is a 2-arg distance builtin,
+ // so there is no metric argument to read. It is used only for compile-time index selection.
+ AnnSearchPreferenceAnnotation annHint = annDistanceExpr.getAnnotation(AnnSearchPreferenceAnnotation.class);
+ if (annHint != null) {
+ queryDistanceMetric = VectorIndexAccessMethod.normalizeDistanceMetric(annHint.getMetric());
+ }
+
+ // Now analyze the ANN_DISTANCE function arguments
+ AccessMethodAnalysisContext analysisCtx = new AccessMethodAnalysisContext();
+
+ boolean matchFound = VectorIndexAccessMethod.INSTANCE.analyzeFuncExprArgsAndUpdateAnalysisCtx(annDistanceExpr,
+ subTree.getAssignsAndUnnests(), analysisCtx, context, typeEnvironment);
+
+ if (!matchFound) {
+ return false;
+ }
+
+ analyzedAMs.put(VectorIndexAccessMethod.INSTANCE, analysisCtx);
+ return true;
+ }
+
+ /**
+ * Resolves ORDER BY expression to ANN_DISTANCE function.
+ * Handles two cases:
+ * 1. Direct function call: ANN_DISTANCE(...)
+ * 2. Variable reference: $$var where $$var := ANN_DISTANCE(...) in an ASSIGN
+ *
+ * This method is called after subtree init, so assigns/unnests are available.
+ */
+ protected AbstractFunctionCallExpression resolveAnnDistanceExpr(ILogicalExpression orderExpr,
+ List<AbstractLogicalOperator> assignsAndUnnests) {
+
+ // Case 1: Direct function call
+ AbstractFunctionCallExpression direct = asAnnDistanceCall(orderExpr);
+ if (direct != null) {
+ return direct;
+ }
+
+ // Case 2: Variable reference - trace back through assigns
+ if (orderExpr.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
+ VariableReferenceExpression varRef = (VariableReferenceExpression) orderExpr;
+ LogicalVariable orderVar = varRef.getVariableReference();
+
+ // Search assigns/unnests for the variable (similar to InvertedIndexAccessMethod)
+ for (AbstractLogicalOperator op : assignsAndUnnests) {
+ if (op.getOperatorTag() == LogicalOperatorTag.ASSIGN) {
+ AssignOperator assignOp = (AssignOperator) op;
+ List<LogicalVariable> assignVars = assignOp.getVariables();
+ List<Mutable<ILogicalExpression>> assignExprs = assignOp.getExpressions();
+
+ for (int i = 0; i < assignVars.size(); i++) {
+ if (assignVars.get(i).equals(orderVar)) {
+ // Found the assignment
+ return asAnnDistanceCall(assignExprs.get(i).getValue());
+ }
+ }
+ }
+ }
+ }
+
+ return null;
+ }
+
+ /** Returns {@code expr} as an ann-distance function call, or {@code null} if it is not one. */
+ private AbstractFunctionCallExpression asAnnDistanceCall(ILogicalExpression expr) {
+ if (expr.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
+ AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) expr;
+ if (isAnnDistance(funcExpr)) {
+ return funcExpr;
+ }
+ }
+ return null;
+ }
+
+ private boolean isAnnDistance(AbstractFunctionCallExpression funcExpr) {
+ // A desugared ann_distance is a concrete distance builtin carrying the ANN hint; a plain
+ // vector_distance carries the same builtin without the hint and must stay exact (no index).
+ return funcExpr.hasAnnotation(AnnSearchPreferenceAnnotation.class);
+ }
+
+ /**
+ * Chooses the best vector index from candidates.
+ * Considers:
+ * 1. INCLUDE fields: If query has filter (WHERE clause), index must have all filter fields in INCLUDE
+ * 2. Distance metric: Prefers indexes with matching distance metrics.
+ * If the query specifies a constant distance metric that does not match the index metadata, compilation fails.
+ */
+ protected void chooseVectorIndex(Map<IAccessMethod, AccessMethodAnalysisContext> analyzedAMs,
+ List<Pair<IAccessMethod, Index>> result) throws AlgebricksException {
+
+ AccessMethodAnalysisContext analysisCtx = analyzedAMs.get(VectorIndexAccessMethod.INSTANCE);
+ if (analysisCtx == null) {
+ return;
+ }
+
+ // Check if query has filter predicates
+ boolean hasFilter = selectOp != null && filterFieldNames != null && !filterFieldNames.isEmpty();
+
+ // Iterate over candidate vector indexes
+ Iterator<Map.Entry<Index, List<Pair<Integer, Integer>>>> indexIt =
+ analysisCtx.getIteratorForIndexExprsAndVars();
+
+ Pair<IAccessMethod, Index> exactMatch = null; // Index with matching field, metric, AND INCLUDE fields
+ Pair<IAccessMethod, Index> fieldMatch = null; // Index with matching field only (and INCLUDE if needed)
+
+ while (indexIt.hasNext()) {
+ Map.Entry<Index, List<Pair<Integer, Integer>>> indexEntry = indexIt.next();
+ Index index = indexEntry.getKey();
+
+ if (index.getIndexType() == IndexType.VTREE) {
+ // If query has filter, index must have all filter fields in its INCLUDE list.
+ if (hasFilter && !indexHasIncludeFields(index, filterFieldNames)) {
+ continue;
+ }
+
+ if (queryDistanceMetric != null && !queryDistanceMetric.isEmpty()) {
+ String indexMetric = VectorIndexAccessMethod.getIndexDistanceMetric(index);
+ if (queryDistanceMetric.equals(indexMetric)) {
+ // Exact match: field name AND distance metric match.
+ exactMatch = new Pair<>(VectorIndexAccessMethod.INSTANCE, index);
+ break;
+ } else if (fieldMatch == null) {
+ // Field matches but metric doesn't - store as fallback only if no exact match.
+ fieldMatch = new Pair<>(VectorIndexAccessMethod.INSTANCE, index);
+ }
+ } else {
+ // No query metric available - use first field match (backward compatibility).
+ result.add(new Pair<>(VectorIndexAccessMethod.INSTANCE, index));
+ break;
+ }
+ }
+ }
+
+ // Prefer exact match. If only a field match exists (metric mismatch), fall back to full scan (KNN).
+ if (exactMatch != null) {
+ result.add(exactMatch);
+ } else if (fieldMatch != null) {
+ Index idx = fieldMatch.second;
+ String indexMetric = VectorIndexAccessMethod.getIndexDistanceMetric(idx);
+ LOGGER.warn("Distance metric mismatch: query uses '{}' but index '{}' uses '{}'. "
+ + "Falling back to full scan (KNN).", queryDistanceMetric, idx.getIndexName(), indexMetric);
+ }
+ }
+
+ /**
+ * Applies the top-k plan transformation using the chosen vector index.
+ *
+ * Transforms:
+ * LIMIT k → ORDER BY ANN_DISTANCE(vectorField, qvec, metric) → ... → DATASOURCE_SCAN
+ * Into:
+ * LIMIT k → ORDER BY ANN_DISTANCE(vectorField, qvec, metric) → ... → UNNEST-MAP(vector_index_search)
+ *
+ * Key points:
+ * - **ONLY replaces DATASOURCE_SCAN** with vector index search (UNNEST-MAP)
+ * - ORDER BY and LIMIT operators **remain unchanged**
+ * - Vector index returns candidate tuples (may be > k from multiple partitions)
+ * - ORDER BY ANN_DISTANCE computes exact distances on candidates
+ * - ORDER BY + LIMIT extract the true top-k results
+ * - Similar to B+Tree/R-Tree: keeps top operator (ORDER/SELECT), only replaces data scan
+ */
+ protected boolean applyTopKPlanTransformation(Index vectorIndex, AccessMethodAnalysisContext analysisCtx,
+ IOptimizationContext context) throws AlgebricksException {
+
+ // Decide whether the index-only plan branch can be taken (PK-only projection above LIMIT):
+ // when every variable consumed above the LIMIT is PK-derived, the assembled record from the
+ // primary BTree lookup is never needed, so the vector index emits (pk, dist) and the plan
+ // skips the lookup + rerank entirely.
+ boolean indexOnly = isProjectionPkOnly();
+
+ // Build the index-search subplan (UNNEST-MAP over vector index). selectOp is passed so the
+ // access method can attach a selectCondition for filter pushdown when applicable.
+ ILogicalOperator indexSearchOp = VectorIndexAccessMethod.INSTANCE.createIndexSearchPlan(limitRef, orderRef,
+ annDistanceExpr, subTree, vectorIndex, analysisCtx, context, selectOp, indexOnly, aboveLimitOps);
+
+ if (indexSearchOp == null) {
+ return false;
+ }
+
+ // Replace DATASOURCE_SCAN with the vector index search subplan. ORDER BY ANN_DISTANCE
+ // remains in place to compute distances on the returned candidates.
+ subTree.getDataSourceRef().setValue(indexSearchOp);
+ return true;
+ }
+
+ /**
+ * Detects whether consumers above the matched {@code limitOp} only reference primary-key columns
+ * of the dataset — meaning the assembled record produced by a downstream primary BTree lookup is
+ * never needed and the plan can take the index-only branch (vector index emits {@code (pk, dist)},
+ * SORT on {@code $$dist}, LIMIT, done — no primary lookup, no rerank).
+ *
+ * <p>Algorithm:
+ * <ol>
+ * <li>Identify dataset PK variables and "record" variables (the non-PK variables produced by the
+ * {@code DataSourceScan}: the dataset record and, if present, the meta record).</li>
+ * <li>Collect bindings from every {@code ASSIGN} in the subtree below {@code limitOp}
+ * (variable → defining expression).</li>
+ * <li>Compute the live-out set of {@code limitOp} by collecting variables used by every operator
+ * in {@link #aboveLimitOps}.</li>
+ * <li>For each live-out variable, trace through the bindings: it is PK-safe iff it resolves to a
+ * PK variable, a constant, or a {@code field-access-by-name($$rec, f)} where {@code f} is the
+ * (top-level, single-segment) name of a PK column. If any live-out variable fails the trace,
+ * the optimization cannot be safely applied.</li>
+ * </ol>
+ *
+ * <p>Conservatively returns {@code false} for any dataset with a meta record (currently out of
+ * scope), composite PK paths, external data sources, or unfamiliar plan shapes.
+ */
+ protected boolean isProjectionPkOnly() {
+ // Index-only skips the primary BTree lookup and emits (pk, dist) straight from the secondary VTree.
+ // Correct even with deletes: the VTree search cursor reconciles anti-matter (delete) tuples itself,
+ // so a PK-only ANN query over a dataset with deletes returns no deleted PKs. Enabled by default; the
+ // gate is a kill-switch, not a correctness guard, so when disabled the optimizer falls back to the
+ // legacy lookup-and-rerank plan.
+ if (!INDEX_ONLY_ENABLED) {
+ return false;
+ }
+ if (subTree == null || subTree.getDataset() == null || subTree.getDataSourceRef() == null) {
+ return false;
+ }
+ ILogicalOperator dataSourceOpRaw = subTree.getDataSourceRef().getValue();
+ if (!(dataSourceOpRaw instanceof AbstractScanOperator)) {
+ return false;
+ }
+ AbstractScanOperator dataSourceOp = (AbstractScanOperator) dataSourceOpRaw;
+ List<LogicalVariable> dsVars = dataSourceOp.getVariables();
+
+ // PK fields: only support top-level single-segment paths (e.g. ["id"]); composite or nested PK
+ // paths are out of scope for the initial activation.
+ List<List<String>> pkPaths = subTree.getDataset().getPrimaryKeys();
+ Set<String> pkFieldNames = new HashSet<>();
+ for (List<String> p : pkPaths) {
+ if (p == null || p.size() != 1) {
+ return false;
+ }
+ pkFieldNames.add(p.get(0));
+ }
+ int numPK = pkPaths.size();
+ if (dsVars.size() < numPK + 1) {
+ return false; // need at least PKs + a record variable
+ }
+ Set<LogicalVariable> pkVars = new HashSet<>(dsVars.subList(0, numPK));
+ // Anything after PKs is the dataset record (+ optional meta record). Direct references to these
+ // mean we need the assembled record; only field-access for PK names is OK.
+ Set<LogicalVariable> recordVars = new HashSet<>(dsVars.subList(numPK, dsVars.size()));
+
+ // Collect ASSIGN bindings throughout the entire subtree.
+ Map<LogicalVariable, ILogicalExpression> bindings = new HashMap<>();
+ collectAssignBindings(subTree.getRoot(), bindings);
+
+ // Gather variables used above the LIMIT.
+ Set<LogicalVariable> liveOut = new HashSet<>();
+ for (AbstractLogicalOperator op : aboveLimitOps) {
+ try {
+ VariableUtilities.getUsedVariables(op, liveOut);
+ } catch (AlgebricksException e) {
+ LOGGER.trace("isProjectionPkOnly: failed to collect used vars from {}", op.getOperatorTag(), e);
+ return false;
+ }
+ }
+ // Trace each live-out variable; fail on the first non-PK-derived one.
+ Set<LogicalVariable> visiting = new HashSet<>();
+ for (LogicalVariable v : liveOut) {
+ if (!isVarPkSafe(v, bindings, pkVars, recordVars, pkFieldNames, visiting)) {
+ LOGGER.trace("isProjectionPkOnly: live-out variable {} is not PK-derived; bailing", v);
+ return false;
+ }
+ }
+
+ // A WHERE below the LIMIT that needs a non-PK record field (e.g. an INCLUDE-field filter such as
+ // WHERE m.year > 2000) cannot be served by the index-only plan: the record variable is dead, so
+ // VectorIndexAccessMethod#neutralizeDanglingExpressions collapses the predicate to select(missing)
+ // and drops every row. The liveOut scan above only covers the projection above the LIMIT, so a
+ // SELECT below the LIMIT is invisible to it. Require every SELECT-condition variable in the subtree
+ // to be PK-safe; otherwise fall back to lookup-and-rerank, which materializes the record and lets
+ // PushFilterIntoVectorSearchRule push the INCLUDE filter into the vector cursor correctly.
+ Set<LogicalVariable> filterVars = new HashSet<>();
+ collectSelectConditionVars(subTree.getRoot(), filterVars);
+ for (LogicalVariable v : filterVars) {
+ if (!isVarPkSafe(v, bindings, pkVars, recordVars, pkFieldNames, visiting)) {
+ LOGGER.trace("isProjectionPkOnly: WHERE condition var {} is not PK-safe; not index-only", v);
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Collect the variables used in every {@code SELECT} condition in the subtree (below the LIMIT). Used by
+ * {@link #isProjectionPkOnly} to reject the index-only plan when a {@code WHERE} needs a non-PK record
+ * field (e.g. a filter on an INCLUDE field), which the index-only plan cannot serve.
+ */
+ private void collectSelectConditionVars(ILogicalOperator op, Set<LogicalVariable> vars) {
+ if (op == null) {
+ return;
+ }
+ if (op.getOperatorTag() == LogicalOperatorTag.SELECT) {
+ try {
+ VariableUtilities.getUsedVariables(op, vars);
+ } catch (AlgebricksException e) {
+ LOGGER.trace("collectSelectConditionVars: getUsedVariables failed for SELECT", e);
+ }
+ }
+ for (Mutable<ILogicalOperator> input : op.getInputs()) {
+ collectSelectConditionVars(input.getValue(), vars);
+ }
+ }
+
+ /**
+ * Walk {@code op} and its inputs collecting {@code variable → expression} bindings from every
+ * {@code ASSIGN} operator. Used by {@link #isProjectionPkOnly} to trace derived live-out variables
+ * back to their definitions.
+ */
+ private void collectAssignBindings(ILogicalOperator op, Map<LogicalVariable, ILogicalExpression> bindings) {
+ if (op == null) {
+ return;
+ }
+ if (op.getOperatorTag() == LogicalOperatorTag.ASSIGN) {
+ AssignOperator a = (AssignOperator) op;
+ List<LogicalVariable> vars = a.getVariables();
+ List<Mutable<ILogicalExpression>> exprs = a.getExpressions();
+ for (int i = 0; i < vars.size(); i++) {
+ bindings.put(vars.get(i), exprs.get(i).getValue());
+ }
+ }
+ for (Mutable<ILogicalOperator> input : op.getInputs()) {
+ collectAssignBindings(input.getValue(), bindings);
+ }
+ }
+
+ private boolean isVarPkSafe(LogicalVariable v, Map<LogicalVariable, ILogicalExpression> bindings,
+ Set<LogicalVariable> pkVars, Set<LogicalVariable> recordVars, Set<String> pkFieldNames,
+ Set<LogicalVariable> visiting) {
+ if (pkVars.contains(v)) {
+ return true;
+ }
+ if (recordVars.contains(v)) {
+ // Direct use of the dataset record — requires the assembled record. Not safe.
+ return false;
+ }
+ if (!visiting.add(v)) {
+ // Cycle in bindings (shouldn't normally happen). Treat as unsafe.
+ return false;
+ }
+ try {
+ ILogicalExpression e = bindings.get(v);
+ if (e == null) {
+ // Variable produced by something other than an ASSIGN (e.g. a downstream operator we
+ // don't track). Be conservative.
+ return false;
+ }
+ return isExprPkSafe(e, bindings, pkVars, recordVars, pkFieldNames, visiting);
+ } finally {
+ visiting.remove(v);
+ }
+ }
+
+ private boolean isExprPkSafe(ILogicalExpression e, Map<LogicalVariable, ILogicalExpression> bindings,
+ Set<LogicalVariable> pkVars, Set<LogicalVariable> recordVars, Set<String> pkFieldNames,
+ Set<LogicalVariable> visiting) {
+ switch (e.getExpressionTag()) {
+ case CONSTANT:
+ return true;
+ case VARIABLE:
+ return isVarPkSafe(((VariableReferenceExpression) e).getVariableReference(), bindings, pkVars,
+ recordVars, pkFieldNames, visiting);
+ case FUNCTION_CALL: {
+ AbstractFunctionCallExpression fce = (AbstractFunctionCallExpression) e;
+ FunctionIdentifier fid = fce.getFunctionIdentifier();
+ // Special-case field access on the record variable: PK fields only.
+ if (BuiltinFunctions.FIELD_ACCESS_BY_NAME.equals(fid) && fce.getArguments().size() == 2) {
+ ILogicalExpression a0 = fce.getArguments().get(0).getValue();
+ ILogicalExpression a1 = fce.getArguments().get(1).getValue();
+ if (a0.getExpressionTag() == LogicalExpressionTag.VARIABLE
+ && a1.getExpressionTag() == LogicalExpressionTag.CONSTANT) {
+ LogicalVariable target = ((VariableReferenceExpression) a0).getVariableReference();
+ if (recordVars.contains(target)) {
+ String fieldName = extractStringConstant(a1);
+ return fieldName != null && pkFieldNames.contains(fieldName);
+ }
+ }
+ }
+ // For any other function call: all argument expressions must be PK-safe.
+ for (Mutable<ILogicalExpression> arg : fce.getArguments()) {
+ if (!isExprPkSafe(arg.getValue(), bindings, pkVars, recordVars, pkFieldNames, visiting)) {
+ return false;
+ }
+ }
+ return true;
+ }
+ default:
+ return false;
+ }
+ }
+
+ private static String extractStringConstant(ILogicalExpression e) {
+ if (e.getExpressionTag() != LogicalExpressionTag.CONSTANT) {
+ return null;
+ }
+ ConstantExpression ce = (ConstantExpression) e;
+ IAlgebricksConstantValue v = ce.getValue();
+ if (!(v instanceof AsterixConstantValue)) {
+ return null;
+ }
+ IAObject obj = ((AsterixConstantValue) v).getObject();
+ return (obj instanceof AString) ? ((AString) obj).getStringValue() : null;
+ }
+
+ /**
+ * Clears the state for the next optimization attempt.
+ */
+ protected void clear() {
+ limitRef = null;
+ limitOp = null;
+ orderRef = null;
+ orderOp = null;
+ annDistanceExpr = null;
+ typeEnvironment = null;
+ queryDistanceMetric = null;
+ selectOp = null;
+ filterFieldNames = null;
+ aboveLimitOps.clear();
+ subTree.reset();
+ }
+
+ @Override
+ public Map<FunctionIdentifier, List<IAccessMethod>> getAccessMethods() {
+ return accessMethods;
+ }
+}
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/VectorIndexAccessMethod.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/VectorIndexAccessMethod.java
new file mode 100644
index 0000000000..4d27d74063
--- /dev/null
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/VectorIndexAccessMethod.java
@@ -0,0 +1,768 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.optimizer.rules.am;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.asterix.common.annotations.AbstractExpressionAnnotationWithIndexNames;
+import org.apache.asterix.common.annotations.AnnSearchPreferenceAnnotation;
+import org.apache.asterix.common.config.DatasetConfig.IndexType;
+import org.apache.asterix.common.vector.VectorSimilarityMetric;
+import org.apache.asterix.metadata.entities.Dataset;
+import org.apache.asterix.metadata.entities.Index;
+import org.apache.asterix.object.base.AdmObjectNode;
+import org.apache.asterix.om.base.ADouble;
+import org.apache.asterix.om.base.AInt32;
+import org.apache.asterix.om.base.AInt64;
+import org.apache.asterix.om.base.AMissing;
+import org.apache.asterix.om.base.AString;
+import org.apache.asterix.om.base.IAObject;
+import org.apache.asterix.om.constants.AsterixConstantValue;
+import org.apache.asterix.om.functions.BuiltinFunctions;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.BuiltinType;
+import org.apache.asterix.om.types.IAType;
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.common.utils.Pair;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import org.apache.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
+import org.apache.hyracks.algebricks.core.algebra.expressions.IAlgebricksConstantValue;
+import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import org.apache.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractDataSourceOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.DistinctOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
+import org.apache.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
+
+/**
+ * Access method for vector indexes.
+ *
+ * This access method is designed specifically for ORDER BY ANN_DISTANCE() LIMIT k queries.
+ * It does NOT handle SELECT-based optimizations (WHERE clauses with ANN_DISTANCE).
+ *
+ * Example query pattern:
+ * <pre>
+ * SELECT id, title
+ * FROM movie
+ * WHERE year > 2000 -- Handled by BTreeAccessMethod
+ * ORDER BY ANN_DISTANCE(reviewEmbedding, [1.0, 2.0, ...], "Euclidean") -- Handled by VectorIndexAccessMethod
+ * LIMIT 10
+ * </pre>
+ *
+ * - If vector index EXISTS: Optimizer transforms plan to use UNNEST-MAP(vector_index_search)
+ * → Returns candidate tuples (approximate ANN search)
+ * → ORDER BY ANN_DISTANCE computes distances on candidates only
+ * → Faster but approximate results
+ *
+ * - If vector index DOES NOT EXIST: Optimizer leaves plan unchanged (DATASOURCE_SCAN)
+ * → Returns ALL tuples (exhaustive scan)
+ * → ORDER BY ANN_DISTANCE computes distances on all tuples
+ * → Falls back to exact KNN search (slower but exact results)
+ *
+ * The same ORDER BY ANN_DISTANCE operator works in both cases - the optimizer just swaps the data source.
+ *
+ * This is used by IntroduceTopKAccessMethodRule to validate and analyze ANN_DISTANCE function calls
+ * in ORDER BY clauses.
+ */
+public class VectorIndexAccessMethod implements IAccessMethod {
+
+ public static final VectorIndexAccessMethod INSTANCE = new VectorIndexAccessMethod();
+
+ // Optimizable functions for vector index: the concrete distance builtins that ann_distance desugars
+ // into. Only those carrying an AnnSearchPreferenceAnnotation are actually optimized (see the gate in
+ // analyzeFuncExprArgsAndUpdateAnalysisCtx); a plain vector_distance produces the same builtins WITHOUT
+ // the hint and stays an exact full-scan KNN.
+ private static final List<Pair<FunctionIdentifier, Boolean>> FUNC_IDENTIFIERS =
+ Collections.unmodifiableList(Arrays.asList(new Pair<>(BuiltinFunctions.EUCLIDEAN_DISTANCE, true),
+ new Pair<>(BuiltinFunctions.EUCLIDEAN_SQUARED_DISTANCE, true),
+ new Pair<>(BuiltinFunctions.COSINE_DISTANCE, true),
+ new Pair<>(BuiltinFunctions.DOT_DISTANCE, true)));
+
+ @Override
+ public List<Pair<FunctionIdentifier, Boolean>> getOptimizableFunctions() {
+ return FUNC_IDENTIFIERS;
+ }
+
+ @Override
+ public boolean analyzeFuncExprArgsAndUpdateAnalysisCtx(AbstractFunctionCallExpression funcExpr,
+ List<AbstractLogicalOperator> assignsAndUnnests, AccessMethodAnalysisContext analysisCtx,
+ IOptimizationContext context, IVariableTypeEnvironment typeEnvironment) throws AlgebricksException {
+
+ // The desugared ann_distance is a 2-arg distance builtin carrying the ANN hint. Gate on the hint so
+ // a plain vector_distance (same builtin, no hint) is never rewritten into an index search.
+ if (!funcExpr.hasAnnotation(AnnSearchPreferenceAnnotation.class)) {
+ return false;
+ }
+ if (funcExpr.getArguments().size() != 2) {
+ return false;
+ }
+ // Validate arg0 = variable/field reference, arg1 = constant (query vector).
+ // Populates analysisCtx with information needed for index matching.
+ return AccessMethodUtils.analyzeFuncExprArgsForOneConstAndVarAndUpdateAnalysisCtx(funcExpr, analysisCtx,
+ context, typeEnvironment, false);
+ }
+
+ @Override
+ public boolean matchIndexType(IndexType indexType) {
+ return indexType == IndexType.VTREE;
+ }
+
+ @Override
+ public boolean matchAllIndexExprs(Index index) {
+ // Vector indexes only have one field, so this is not applicable
+ return false;
+ }
+
+ @Override
+ public boolean matchPrefixIndexExprs(Index index) {
+ // Vector indexes don't support prefix matching like composite BTree indexes
+ return false;
+ }
+
+ @Override
+ public boolean applySelectPlanTransformation(List<Mutable<ILogicalOperator>> afterSelectRefs,
+ Mutable<ILogicalOperator> selectRef, OptimizableOperatorSubTree subTree, Index chosenIndex,
+ AccessMethodAnalysisContext analysisCtx, IOptimizationContext context) throws AlgebricksException {
+ // NOT IMPLEMENTED: Vector indexes are not used for SELECT-based optimizations
+ // If we wanted to support: WHERE ANN_DISTANCE(...) < threshold
+ // we would implement this method. For now, only ORDER BY + LIMIT is supported.
+ return false;
+ }
+
+ @Override
+ public ILogicalOperator createIndexSearchPlan(List<Mutable<ILogicalOperator>> afterTopOpRefs,
+ Mutable<ILogicalOperator> topOpRef, Mutable<ILogicalExpression> conditionRef,
+ List<Mutable<ILogicalOperator>> assignBeforeTheOpRefs, OptimizableOperatorSubTree indexSubTree,
+ OptimizableOperatorSubTree probeSubTree, Index chosenIndex, AccessMethodAnalysisContext analysisCtx,
+ boolean retainInput, boolean retainNull, boolean requiresBroadcast, IOptimizationContext context,
+ LogicalVariable newMissingNullPlaceHolderForLOJ, IAlgebricksConstantValue leftOuterMissingValue,
+ List<Pair<LogicalVariable, List<ILogicalExpression>>> optimizableDisjunctionConditions)
+ throws AlgebricksException {
+ // NOT IMPLEMENTED: Used for SELECT-based plan transformation
+ // Vector index top-k search plan is created in IntroduceTopKAccessMethodRule
+ return null;
+ }
+
+ @Override
+ public boolean applyJoinPlanTransformation(List<Mutable<ILogicalOperator>> afterJoinRefs,
+ Mutable<ILogicalOperator> joinRef, OptimizableOperatorSubTree leftSubTree,
+ OptimizableOperatorSubTree rightSubTree, Index chosenIndex, AccessMethodAnalysisContext analysisCtx,
+ IOptimizationContext context, boolean isLeftOuterJoin, boolean isLeftOuterJoinWithSpecialGroupBy,
+ IAlgebricksConstantValue leftOuterMissingValue) throws AlgebricksException {
+ // NOT IMPLEMENTED: Vector indexes are not used for join optimization
+ return false;
+ }
+
+ /**
+ * Creates vector index search plan for ORDER BY ANN_DISTANCE() queries.
+ *
+ * This method creates a two-stage index search plan:
+ * 1. Vector index search: Returns (vector_embedding, pk) for candidate tuples
+ * 2. Primary index lookup: Uses PKs to fetch full records
+ *
+ * Transformation:
+ * LIMIT k → ORDER BY ANN_DISTANCE(vectorField, qvec, metric) → ... → DATASOURCE_SCAN
+ * Into:
+ * LIMIT k → ORDER BY ANN_DISTANCE(vectorField, qvec, metric) → PRIMARY_INDEX_UNNEST(pk) → VECTOR_INDEX_UNNEST
+ *
+ * Data flow:
+ * - VECTOR_INDEX_UNNEST: Returns top-k candidates from index (vector_embedding + pk)
+ * - PRIMARY_INDEX_UNNEST: Uses PK to fetch full record with all fields
+ * - ORDER BY: Computes exact distances on full records
+ * - LIMIT: Extracts final top-k results
+ *
+ * TODO: Add index-only plan optimization (skip primary lookup when only PK is needed)
+ *
+ * @param limitRef Reference to LIMIT operator
+ * @param orderRef Reference to ORDER operator
+ * @param annDistanceExpr The ANN_DISTANCE function expression from ORDER BY
+ * @param subTree The subtree containing the data source
+ * @param chosenIndex The vector index to use
+ * @param analysisCtx Analysis context with index information
+ * @param context Optimization context
+ * @return The transformed plan with vector index search + primary lookup, or null if transformation fails
+ */
+ public ILogicalOperator createIndexSearchPlan(Mutable<ILogicalOperator> limitRef,
+ Mutable<ILogicalOperator> orderRef, AbstractFunctionCallExpression annDistanceExpr,
+ OptimizableOperatorSubTree subTree, Index chosenIndex, AccessMethodAnalysisContext analysisCtx,
+ IOptimizationContext context, SelectOperator selectOp, boolean indexOnly,
+ List<AbstractLogicalOperator> aboveLimitOps) throws AlgebricksException {
+
+ // Get dataset metadata
+ Dataset dataset = subTree.getDataset();
+ ARecordType recordType = subTree.getRecordType();
+ ARecordType metaRecordType = subTree.getMetaRecordType();
+ AbstractDataSourceOperator dataSourceOp = (AbstractDataSourceOperator) subTree.getDataSourceRef().getValue();
+
+ // The desugared ann_distance call is <distance-builtin>(vectorField, queryVector) carrying an
+ // AnnSearchPreferenceAnnotation with {metric, min_probe_fraction, k_multiplier}.
+ // arg0 = vectorField (variable reference), arg1 = queryVector (constant or variable).
+ AnnSearchPreferenceAnnotation annHint = annDistanceExpr.getAnnotation(AnnSearchPreferenceAnnotation.class);
+ ILogicalExpression queryVectorExpr = annDistanceExpr.getArguments().get(1).getValue();
+
+ // Extract k value from LIMIT operator
+ LimitOperator limitOp = (LimitOperator) limitRef.getValue();
+ ILogicalExpression kValueExpr = limitOp.getMaxObjects().getValue();
+
+ // Create variables to hold query parameters in the positional layout the runtime search expects:
+ // [query_vector, k_value, distance_metric, min_probe_fraction, k_multiplier].
+ ArrayList<LogicalVariable> queryVarList = new ArrayList<>();
+ ArrayList<Mutable<ILogicalExpression>> queryExprList = new ArrayList<>();
+
+ // Add query vector variable
+ LogicalVariable queryVectorVar = context.newVar();
+ queryVarList.add(queryVectorVar);
+ queryExprList.add(new MutableObject<>(queryVectorExpr.cloneExpression()));
+
+ // Add k value variable
+ LogicalVariable kValueVar = context.newVar();
+ queryVarList.add(kValueVar);
+ queryExprList.add(new MutableObject<>(kValueExpr.cloneExpression()));
+
+ // Add distance metric variable (slot 2). The runtime scan never reads it (the effective metric is
+ // baked into the index metadata); it is retained for the positional contract and rebuilt from the
+ // ANN hint's metric string.
+ LogicalVariable distanceMetricVar = context.newVar();
+ queryVarList.add(distanceMetricVar);
+ queryExprList.add(new MutableObject<>(
+ new ConstantExpression(new AsterixConstantValue(new AString(annHint.getMetric())))));
+
+ // Add min_probe_fraction variable (slot 3) from the ANN hint (default/validation applied at rewrite).
+ // Fraction of leaf clusters to probe (0.0-1.0); nprobe = max(1, floor(totalLeafClusters * fraction)).
+ LogicalVariable minProbeFractionVar = context.newVar();
+ queryVarList.add(minProbeFractionVar);
+ queryExprList.add(new MutableObject<>(
+ new ConstantExpression(new AsterixConstantValue(new ADouble(annHint.getMinProbeFraction())))));
+
+ // Add k_multiplier variable (slot 4) from the ANN hint. K * kMultiplier candidates are collected
+ // for reranking.
+ LogicalVariable kMultiplierVar = context.newVar();
+ queryVarList.add(kMultiplierVar);
+ queryExprList.add(new MutableObject<>(
+ new ConstantExpression(new AsterixConstantValue(new AInt32(annHint.getKMultiplier())))));
+
+ // Create ASSIGN operator to hold query parameters
+ AssignOperator assignSearchKeys = new AssignOperator(queryVarList, queryExprList);
+ assignSearchKeys.setSourceLocation(dataSourceOp.getSourceLocation());
+ assignSearchKeys.getInputs().add(
+ new MutableObject<>(OperatorManipulationUtil.deepCopy(dataSourceOp.getInputs().get(0).getValue())));
+ assignSearchKeys.setExecutionMode(dataSourceOp.getExecutionMode());
+ context.computeAndSetTypeEnvironmentForOperator(assignSearchKeys);
+
+ // Create VectorJobGenParams to pass parameters to Hyracks runtime
+ VectorJobGenParams jobGenParams = new VectorJobGenParams(chosenIndex.getIndexName(), IndexType.VTREE,
+ chosenIndex.getDatabaseName(), dataset.getDataverseName(), dataset.getDatasetName(), false, // retainInput - not needed for simple case
+ false // requiresBroadcast
+ );
+ jobGenParams.setQueryVarList(queryVarList);
+
+ // Index-only opportunity detected upstream in IntroduceTopKAccessMethodRule (PK-only
+ // projection above LIMIT). When set, the secondary UnnestMap emits an extra $$dist field, the
+ // ORDER BY rebinds to it, and the primary BTree lookup + rerank are skipped (see branch below).
+ boolean isIndexOnlyPlan = indexOnly;
+ jobGenParams.setIndexOnly(isIndexOnlyPlan);
+
+ // Create UNNEST-MAP operator for vector index search
+ // This returns: <pk> from the index (vector embeddings skipped to save memory)
+ ILogicalOperator secondaryIndexUnnestOp = AccessMethodUtils.createSecondaryIndexUnnestMap(dataset, recordType,
+ metaRecordType, chosenIndex, assignSearchKeys, jobGenParams, context, false, false, null);
+
+ // INCLUDE-field filter pushdown is intentionally deferred to PushFilterIntoVectorSearchRule (a
+ // physical rewrite): the field-access predicate references the record variable, which is not in this
+ // unnest's input type environment. Just register the variables the vector index search produces.
+ context.computeAndSetTypeEnvironmentForOperator(secondaryIndexUnnestOp);
+
+ if (isIndexOnlyPlan) {
+ // ============================================================================
+ // INDEX-ONLY BRANCH
+ // ============================================================================
+ // The projection above LIMIT only references PK columns (verified by
+ // IntroduceTopKAccessMethodRule.isProjectionPkOnly). We can therefore:
+ // 1. Add a new $$dist ADOUBLE output variable to the secondary UnnestMap so the runtime
+ // emits [ pk_0, ..., pk_{n-1}, dist ] per candidate.
+ // 2. Rewrite the upstream ORDER BY ann_distance(...) to ORDER BY $$dist (a cheap scalar
+ // sort, no record assembly, no field-access, no rerank).
+ // 3. Substitute old PK variables (from the original DataSourceScan) with the new PK
+ // variables produced by the secondary UnnestMap — the optimizer is about to replace
+ // the DataSourceScan, so any operator that still references the old PK vars would
+ // dangle.
+ // 4. Rewrite field-access($$rec, "<pk_field>") expressions to VarRef($$pk_new) so ASSIGNs
+ // like $$id := field-access($$rec, "id") become $$id := $$pk_new.
+ // 5. Skip createRestOfIndexSearchPlan — no primary BTree lookup, no record assembly. The
+ // runtime reads D(q,x) from the cursor and appends it.
+ if (!(secondaryIndexUnnestOp instanceof UnnestMapOperator)) {
+ // Unexpected shape — fall through to the legacy lookup-and-rerank plan below.
+ isIndexOnlyPlan = false;
+ jobGenParams.setIndexOnly(false);
+ } else {
+ UnnestMapOperator unnestMap = (UnnestMapOperator) secondaryIndexUnnestOp;
+ LogicalVariable distVar = context.newVar();
+ unnestMap.getVariables().add(distVar);
+ unnestMap.getVariableTypes().add(BuiltinType.ADOUBLE);
+
+ // Rewrite ORDER BY: replace the ann_distance call with VarRef($$dist). The earlier
+ // pattern-match guarantees orderOp has exactly one expression (the ann_distance call);
+ // keep the same direction.
+ OrderOperator orderOp = (OrderOperator) orderRef.getValue();
+ if (orderOp.getOrderExpressions().size() != 1) {
+ // Unexpected ORDER shape — roll back the distVar addition before falling through.
+ unnestMap.getVariables().remove(unnestMap.getVariables().size() - 1);
+ unnestMap.getVariableTypes().remove(unnestMap.getVariableTypes().size() - 1);
+ isIndexOnlyPlan = false;
+ jobGenParams.setIndexOnly(false);
+ } else {
+ VariableReferenceExpression distVarRef = new VariableReferenceExpression(distVar);
+ distVarRef.setSourceLocation(orderOp.getSourceLocation());
+ orderOp.getOrderExpressions().get(0).second.setValue(distVarRef);
+ }
+
+ if (isIndexOnlyPlan) {
+ // Build the variable substitution map: old PK vars → new PK vars. The secondary
+ // UnnestMap allocates fresh context.newVar() PK variables, so anything that
+ // referenced the old PK variable from the DataSourceScan needs to be redirected.
+ List<LogicalVariable> oldVars = dataSourceOp.getVariables();
+ int numPK = dataset.getPrimaryKeys().size();
+ List<LogicalVariable> newPkVars = unnestMap.getVariables().subList(0, numPK);
+ Map<LogicalVariable, LogicalVariable> pkSubstitution = new java.util.HashMap<>();
+ for (int i = 0; i < numPK; i++) {
+ pkSubstitution.put(oldVars.get(i), newPkVars.get(i));
+ }
+ // The record variable (and meta record var, if present) — we rewrite
+ // field-access($$rec, "<pk_field>") to VarRef($$pk_new) below.
+ LogicalVariable oldRecVar = numPK < oldVars.size() ? oldVars.get(numPK) : null;
+
+ // Map PK field name -> new PK variable.
+ Map<String, LogicalVariable> pkFieldToNewVar = new java.util.HashMap<>();
+ List<List<String>> pkPaths = dataset.getPrimaryKeys();
+ for (int i = 0; i < numPK; i++) {
+ List<String> p = pkPaths.get(i);
+ if (p != null && p.size() == 1) {
+ pkFieldToNewVar.put(p.get(0), newPkVars.get(i));
+ }
+ }
+
+ // The rewrites below must cover the operators ABOVE the LIMIT too — the result
+ // projection (e.g. SELECT VALUE m.idx) lives above LIMIT and references the old
+ // record var via field-access($$rec, "idx"). Walking only from limitRef would leave
+ // it dangling ("Could not infer type"). aboveLimitOps[0] is the outermost ancestor
+ // (e.g. DistributeResult); walking from it reaches the projection, the LIMIT, the
+ // ORDER, and the ASSIGNs below in a single descent.
+ ILogicalOperator rewriteRoot = (aboveLimitOps != null && !aboveLimitOps.isEmpty())
+ ? aboveLimitOps.get(0) : (ILogicalOperator) limitRef.getValue();
+
+ // 1. Substitute old PK var refs throughout the plan (above and below LIMIT).
+ org.apache.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities
+ .substituteVariablesInDescendantsAndSelf(rewriteRoot, pkSubstitution, context);
+
+ // 2. Rewrite field-access on the old record var to direct PK VarRefs (e.g. the
+ // SELECT VALUE m.idx projection above LIMIT).
+ if (oldRecVar != null) {
+ rewriteRecordFieldAccessToPk(rewriteRoot, oldRecVar, pkFieldToNewVar);
+ }
+
+ // 3. Any remaining ASSIGN expression that still references oldRecVar (transitively) —
+ // for example $$x := $$m.getField("embedding") hoisted by the SQL++ compiler out of
+ // ann_distance(m.embedding, ...) — is now structurally orphaned: no operator above
+ // LIMIT consumes it (we wouldn't have entered the index-only branch otherwise) but the
+ // type system still walks it. Replace each such expression with MISSING so it types
+ // cleanly. None of these values is ever read at runtime.
+ if (oldRecVar != null) {
+ neutralizeDanglingExpressions(rewriteRoot, oldRecVar);
+ }
+
+ // Recompute type env bottom-up over every operator whose schema/expressions changed:
+ // the new secondary UnnestMap, the LIMIT chain, then each above-LIMIT ancestor from the
+ // one closest to LIMIT up to the root (aboveLimitOps is ordered root-first).
+ context.computeAndSetTypeEnvironmentForOperator(secondaryIndexUnnestOp);
+ org.apache.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil.typeOpRec(limitRef, context);
+ if (aboveLimitOps != null) {
+ for (int ai = aboveLimitOps.size() - 1; ai >= 0; ai--) {
+ context.computeAndSetTypeEnvironmentForOperator(aboveLimitOps.get(ai));
+ }
+ }
+
+ // Cross-pollination dedup (index-only branch): when cross_pollination_m > 1 the secondary
+ // cursor emits up to M (pk..., dist) copies per record. The primary-lookup path dedups via a
+ // DistinctOperator above its primary UNNEST-MAP (see createRestOfIndexSearchPlan below), but the
+ // index-only branch skips that path, so it must splice its OWN Distinct — keyed on the new
+ // secondary-UnnestMap PK vars — directly above the secondary UNNEST-MAP and below ORDER BY/LIMIT,
+ // so LIMIT applies to DISTINCT PKs. DistinctOperator propagates all input vars, so $$dist
+ // survives for the ORDER BY $$dist above it. For M == 1 we return the bare UNNEST-MAP so the
+ // plan stays byte-identical to the legacy path.
+ if (extractCrossPollinationM(chosenIndex) > 1 && numPK > 0) {
+ List<Mutable<ILogicalExpression>> dedupExprs = new ArrayList<>(numPK);
+ for (LogicalVariable pkVar : newPkVars) {
+ VariableReferenceExpression pkRef = new VariableReferenceExpression(pkVar);
+ pkRef.setSourceLocation(secondaryIndexUnnestOp.getSourceLocation());
+ dedupExprs.add(new MutableObject<>(pkRef));
+ }
+ DistinctOperator dedupOp = new DistinctOperator(dedupExprs);
+ dedupOp.setSourceLocation(secondaryIndexUnnestOp.getSourceLocation());
+ dedupOp.getInputs().add(new MutableObject<>(secondaryIndexUnnestOp));
+ dedupOp.setExecutionMode(secondaryIndexUnnestOp.getExecutionMode());
+ context.computeAndSetTypeEnvironmentForOperator(dedupOp);
+ return dedupOp;
+ }
+ return secondaryIndexUnnestOp;
+ }
+ }
+ }
+
+ // Add primary index lookup to get full record
+ // This uses the PKs returned from vector index to fetch complete records
+ ILogicalOperator primaryIndexUnnestOp = AccessMethodUtils.createRestOfIndexSearchPlan(null, // afterTopOpRefs - not needed for ORDER BY case
+ null, // topOpRef - not needed for ORDER BY case
+ null, // conditionRef - no WHERE condition to push down
+ null, // assignsBeforeTopOpRef - query params already in assignSearchKeys
+ dataSourceOp, dataset, recordType, metaRecordType, secondaryIndexUnnestOp, // inputOp: vector index search results
+ context, true, // sortPrimaryKeys
+ false, // retainInput
+ false, // retainMissing
+ false, // requiresBroadcast
+ chosenIndex, // secondaryIndex
+ analysisCtx, subTree, // indexSubTree
+ null, // probeSubTree - not a join
+ null, // newMissingPlaceHolderForLOJ
+ null, // leftOuterMissingValue
+ false, // anyRealTypeConvertedToIntegerType
+ null // optimizableDisjunctionConditions — vector index doesn't use OR-disjunction CBO
+ );
+
+ // Cross-pollination dedup: each base record surfaces as up to M (distance, centroidId, pk) copies from
+ // the vector cursor when cross_pollination_m > 1. Splice a DistinctOperator keyed on the PK variables
+ // ABOVE the primary-index lookup. We can't put it BETWEEN the two UNNEST-MAPs because
+ // createRestOfIndexSearchPlan internally casts its inputOp to AbstractUnnestMapOperator. Cost is one
+ // primary-key lookup per duplicate, bounded by M * nprobe_candidates — cheap in practice. For M == 1
+ // we skip dedup so the plan is byte-identical to the legacy path.
+ //
+ // Dedup keys are the data-source operator's PK vars (head of dataSourceOp.getVariables()), NOT the
+ // secondary UNNEST-MAP's PK vars: createRestOfIndexSearchPlan emits the primary UNNEST-MAP using the
+ // data source's variable IDs, so those are the vars actually in scope above primaryIndexUnnestOp. Using
+ // the secondary's PK vars instead causes "Could not infer type for variable" because they are only
+ // INPUT search keys to the primary unnest, not part of its output type environment.
+ ILogicalOperator topOfIndexPlan = primaryIndexUnnestOp;
+ int crossPollinationM = extractCrossPollinationM(chosenIndex);
+ if (crossPollinationM > 1) {
+ List<LogicalVariable> dsVars = dataSourceOp.getVariables();
+ int numPkVars = dataset.getPrimaryKeys().size();
+ if (numPkVars > 0 && dsVars.size() >= numPkVars) {
+ List<LogicalVariable> dedupPkVars = new ArrayList<>(dsVars.subList(0, numPkVars));
+ List<Mutable<ILogicalExpression>> distinctExprs = new ArrayList<>(dedupPkVars.size());
+ for (LogicalVariable pkVar : dedupPkVars) {
+ VariableReferenceExpression pkRef = new VariableReferenceExpression(pkVar);
+ pkRef.setSourceLocation(primaryIndexUnnestOp.getSourceLocation());
+ distinctExprs.add(new MutableObject<>(pkRef));
+ }
+ DistinctOperator dedupOp = new DistinctOperator(distinctExprs);
+ dedupOp.setSourceLocation(primaryIndexUnnestOp.getSourceLocation());
+ dedupOp.getInputs().add(new MutableObject<>(primaryIndexUnnestOp));
+ dedupOp.setExecutionMode(primaryIndexUnnestOp.getExecutionMode());
+ context.computeAndSetTypeEnvironmentForOperator(dedupOp);
+ topOfIndexPlan = dedupOp;
+ }
+ }
+
+ return topOfIndexPlan;
+ }
+
+ /**
+ * Reads {@code cross_pollination_m} from the chosen vector index's WITH-object. Returns 1 (no
+ * cross-pollination) if the index is not a {@code VectorIndexDetails} or the field is absent.
+ */
+ private static int extractCrossPollinationM(Index chosenIndex) {
+ if (chosenIndex == null) {
+ return 1;
+ }
+ Index.IIndexDetails details = chosenIndex.getIndexDetails();
+ if (!(details instanceof Index.VectorIndexDetails)) {
+ return 1;
+ }
+ AdmObjectNode withObj = ((Index.VectorIndexDetails) details).getWithObjectNode();
+ if (withObj == null) {
+ return 1;
+ }
+ return Math.max(1, withObj.getOptionalInt("cross_pollination_m", 1));
+ }
+
+ /**
+ * Checks if this ann_distance expression can use the given vector index (by field-name match).
+ * Query/index distance-metric compatibility is handled separately in
+ * {@link IntroduceTopKAccessMethodRule#chooseVectorIndex}.
+ *
+ * @param index The vector index to check
+ * @param optFuncExpr The optimizable function expression
+ * @param checkApplicableOnly Whether to only check applicability
+ * @return true if the index can be used, false otherwise
+ */
+ @Override
+ public boolean exprIsOptimizable(Index index, IOptimizableFuncExpr optFuncExpr, boolean checkApplicableOnly)
+ throws AlgebricksException {
+ if (index.getIndexType() != IndexType.VTREE) {
+ return false;
+ }
+ // Get the field name from the ann_distance call (arg0).
+ List<String> fieldName = optFuncExpr.getFieldName(0);
+ if (fieldName == null || fieldName.isEmpty()) {
+ return false;
+ }
+ // A vector index has exactly one key field; the query field must match it.
+ Index.VectorIndexDetails vectorDetails = (Index.VectorIndexDetails) index.getIndexDetails();
+ List<List<String>> indexKeyFieldNames = vectorDetails.getKeyFieldNames();
+ if (indexKeyFieldNames.size() != 1) {
+ return false;
+ }
+ return indexKeyFieldNames.get(0).equals(fieldName);
+ }
+
+ @Override
+ public AbstractExpressionAnnotationWithIndexNames getSecondaryIndexAnnotation(IOptimizableFuncExpr optFuncExpr) {
+ // Not used for vector indexes
+ return null;
+ }
+
+ @Override
+ public String getName() {
+ return "VECTOR_INDEX";
+ }
+
+ @Override
+ public boolean acceptsFunction(AbstractFunctionCallExpression functionExpr, Index index, IAType indexedFieldType,
+ boolean defaultNull, boolean finalStep) throws AlgebricksException {
+ // Check if this function can be optimized with this index type
+
+ // Vector fields should be arrays of numbers
+ ATypeTag typeTag = indexedFieldType.getTypeTag();
+ return typeTag == ATypeTag.ARRAY || typeTag == ATypeTag.MULTISET;
+ }
+
+ @Override
+ public int compareTo(IAccessMethod o) {
+ return this.getName().compareTo(o.getName());
+ }
+
+ /**
+ * Normalizes a distance metric string to its canonical form.
+ * Handles aliases and case-insensitive matching.
+ *
+ * Supported metrics and aliases:
+ * - "euclidean", "l2" → "euclidean"
+ * - "euclidean_squared", "l2_squared" → "euclidean_squared"
+ * - "cosine", "cosine similarity" → "cosine"
+ * - "dot" → "dot"
+ *
+ * @param metric The distance metric string (may be null or empty)
+ * @return Normalized canonical metric name, or "euclidean" as default
+ */
+ public static String normalizeDistanceMetric(String metric) {
+ if (metric == null || metric.trim().isEmpty()) {
+ return VectorSimilarityMetric.EUCLIDEAN.canonical(); // Default metric
+ }
+ VectorSimilarityMetric resolved = VectorSimilarityMetric.fromAlias(metric);
+ // Unknown metric: return normalized as-is (the distance function handles/rejects it).
+ return (resolved != null) ? resolved.canonical() : metric.toLowerCase().trim();
+ }
+
+ /**
+ * Extracts and normalizes the distance metric from a vector index.
+ *
+ * @param index The vector index
+ * @return Normalized distance metric string, or "euclidean" as default
+ */
+ public static String getIndexDistanceMetric(Index index) {
+ if (index.getIndexType() != IndexType.VTREE) {
+ return ""; // Default for non-vector indexes
+ }
+
+ Index.VectorIndexDetails vectorDetails = (Index.VectorIndexDetails) index.getIndexDetails();
+ AdmObjectNode withObjectNode = vectorDetails.getWithObjectNode();
+
+ String indexMetric = (withObjectNode != null) ? withObjectNode.getOptionalString("similarity", "") : "";
+
+ return normalizeDistanceMetric(indexMetric);
+ }
+
+ /**
+ * Ensures an integer constant expression uses AInt32 instead of AInt64.
+ *
+ * SQL++ parser creates AInt64 (8 bytes) for all integer literals, but the runtime
+ * uses IntegerPointable.getInteger() which reads only 4 bytes. Reading the first
+ * 4 bytes of an 8-byte big-endian AInt64 for small values yields 0.
+ *
+ * This method converts AInt64 constants to AInt32 at compile time so the runtime
+ * can correctly read them with IntegerPointable.
+ *
+ * @param expr The expression to check and potentially convert
+ * @return The original expression if not an AInt64 constant, or a new AInt32 constant expression
+ */
+
+ /**
+ * Index-only support: walk {@code op} and its descendants and replace every
+ * {@code field-access-by-name(VarRef(recVar), "fieldName")} where {@code fieldName} is a PK column
+ * with a direct {@code VarRef(pkVar)}. Used to repair ASSIGNs like
+ * {@code $$id := field-access($$rec, "id")} after the DataSourceScan has been replaced by the
+ * secondary UnnestMap (which no longer produces the record variable).
+ */
+ private static void rewriteRecordFieldAccessToPk(ILogicalOperator op, LogicalVariable recVar,
+ Map<String, LogicalVariable> pkFieldToVar) {
+ if (op == null) {
+ return;
+ }
+ if (op.getOperatorTag() == LogicalOperatorTag.ASSIGN) {
+ AssignOperator a = (AssignOperator) op;
+ for (Mutable<ILogicalExpression> exprRef : a.getExpressions()) {
+ rewriteRecordFieldAccessExpr(exprRef, recVar, pkFieldToVar);
+ }
+ }
+ // Other operator types could in principle hold expressions referencing $$rec (e.g. SELECT,
+ // ORDER, ASSIGNs further down). Walk all inputs and apply uniformly.
+ for (Mutable<ILogicalOperator> input : op.getInputs()) {
+ rewriteRecordFieldAccessToPk(input.getValue(), recVar, pkFieldToVar);
+ }
+ }
+
+ private static void rewriteRecordFieldAccessExpr(Mutable<ILogicalExpression> exprRef, LogicalVariable recVar,
+ Map<String, LogicalVariable> pkFieldToVar) {
+ ILogicalExpression e = exprRef.getValue();
+ if (e.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
+ return;
+ }
+ AbstractFunctionCallExpression fce = (AbstractFunctionCallExpression) e;
+ if (BuiltinFunctions.FIELD_ACCESS_BY_NAME.equals(fce.getFunctionIdentifier())
+ && fce.getArguments().size() == 2) {
+ ILogicalExpression a0 = fce.getArguments().get(0).getValue();
+ ILogicalExpression a1 = fce.getArguments().get(1).getValue();
+ if (a0.getExpressionTag() == LogicalExpressionTag.VARIABLE
+ && a1.getExpressionTag() == LogicalExpressionTag.CONSTANT) {
+ LogicalVariable v0 = ((VariableReferenceExpression) a0).getVariableReference();
+ if (recVar.equals(v0)) {
+ String fieldName = extractStringFromConstant(a1);
+ if (fieldName != null && pkFieldToVar.containsKey(fieldName)) {
+ VariableReferenceExpression pkRef =
+ new VariableReferenceExpression(pkFieldToVar.get(fieldName));
+ pkRef.setSourceLocation(e.getSourceLocation());
+ exprRef.setValue(pkRef);
+ return;
+ }
+ }
+ }
+ }
+ for (Mutable<ILogicalExpression> arg : fce.getArguments()) {
+ rewriteRecordFieldAccessExpr(arg, recVar, pkFieldToVar);
+ }
+ }
+
+ /**
+ * Replace any ASSIGN expression whose subtree (transitively) mentions a dead variable with a MISSING
+ * constant. Seeded with the now-gone dataset record variable; the closure grows to include every
+ * variable whose defining ASSIGN we neutralize, so chains such as
+ * {@code $$237 := field-access($$rec, "embedding")} → {@code $$dist := ann_distance($$237, ...)} are
+ * fully repaired (the second ASSIGN references {@code $$237}, not {@code $$rec} directly, and would
+ * otherwise dangle and fail type inference). The live-out check guarantees nothing above LIMIT depends
+ * on these values, but the type system still walks them.
+ */
+ private static void neutralizeDanglingExpressions(ILogicalOperator root, LogicalVariable oldRecVar) {
+ Set<LogicalVariable> dead = new java.util.HashSet<>();
+ dead.add(oldRecVar);
+ // Iterate to a fixpoint: each pass may neutralize an ASSIGN whose variable then feeds the next.
+ while (neutralizeDeadPass(root, dead)) {
+ // keep going until no new variable becomes dead
+ }
+ }
+
+ private static boolean neutralizeDeadPass(ILogicalOperator op, Set<LogicalVariable> dead) {
+ if (op == null) {
+ return false;
+ }
+ boolean changed = false;
+ if (op.getOperatorTag() == LogicalOperatorTag.ASSIGN) {
+ AssignOperator a = (AssignOperator) op;
+ List<LogicalVariable> vars = a.getVariables();
+ List<Mutable<ILogicalExpression>> exprs = a.getExpressions();
+ for (int i = 0; i < exprs.size(); i++) {
+ Mutable<ILogicalExpression> exprRef = exprs.get(i);
+ ILogicalExpression e = exprRef.getValue();
+ if (e.getExpressionTag() != LogicalExpressionTag.CONSTANT && expressionReferencesAny(e, dead)) {
+ ConstantExpression missing = new ConstantExpression(new AsterixConstantValue(AMissing.MISSING));
+ missing.setSourceLocation(e.getSourceLocation());
+ exprRef.setValue(missing);
+ if (dead.add(vars.get(i))) {
+ changed = true;
+ }
+ }
+ }
+ }
+ for (Mutable<ILogicalOperator> input : op.getInputs()) {
+ changed |= neutralizeDeadPass(input.getValue(), dead);
+ }
+ return changed;
+ }
+
+ private static boolean expressionReferencesAny(ILogicalExpression e, Set<LogicalVariable> vars) {
+ if (e == null) {
+ return false;
+ }
+ if (e.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
+ return vars.contains(((VariableReferenceExpression) e).getVariableReference());
+ }
+ if (e.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
+ for (Mutable<ILogicalExpression> arg : ((AbstractFunctionCallExpression) e).getArguments()) {
+ if (expressionReferencesAny(arg.getValue(), vars)) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ private static String extractStringFromConstant(ILogicalExpression e) {
+ if (e.getExpressionTag() != LogicalExpressionTag.CONSTANT) {
+ return null;
+ }
+ ConstantExpression ce = (ConstantExpression) e;
+ IAlgebricksConstantValue v = ce.getValue();
+ if (!(v instanceof AsterixConstantValue)) {
+ return null;
+ }
+ IAObject obj = ((AsterixConstantValue) v).getObject();
+ return (obj instanceof AString) ? ((AString) obj).getStringValue() : null;
+ }
+
+}
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/VectorJobGenParams.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/VectorJobGenParams.java
new file mode 100644
index 0000000000..73ae60c7d8
--- /dev/null
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/VectorJobGenParams.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.optimizer.rules.am;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.asterix.common.config.DatasetConfig.IndexType;
+import org.apache.asterix.common.metadata.DataverseName;
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
+
+/**
+ * Helper class for reading and writing job-gen parameters for vector index access methods.
+ *
+ * Passes information from the optimizer (Algebricks layer) to the physical runtime
+ * (Hyracks layer) for vector index ANN search.
+ *
+ * Parameters passed via queryVarList (variables assigned in an ASSIGN above the UNNEST-MAP):
+ * [0] = query vector (e.g., from constant [1.0, 2.0, ...])
+ * [1] = k value (from LIMIT k)
+ * [2] = distance metric (e.g., "euclidean")
+ * [3] = min_probe_fraction (ANN_DISTANCE arg 3, default 0.1)
+ * [4] = k_multiplier (ANN_DISTANCE arg 4, default 1)
+ */
+public class VectorJobGenParams extends AccessMethodJobGenParams {
+
+ protected List<LogicalVariable> queryVarList;
+
+ /**
+ * Index-only flag for ANN top-K queries.
+ *
+ * When {@code true}, the runtime emits a per-candidate distance field alongside the primary
+ * key(s), and the optimizer skips the downstream primary BTree lookup + rerank ASSIGN. Used when
+ * the projection above {@code LIMIT k → ORDER BY ann_distance(...)} references only PK columns.
+ * When {@code false} (the default), the legacy lookup-and-rerank plan is produced and the runtime
+ * emits only primary key(s).
+ */
+ protected boolean indexOnly;
+
+ public VectorJobGenParams() {
+ }
+
+ public VectorJobGenParams(String indexName, IndexType indexType, String databaseName, DataverseName dataverseName,
+ String datasetName, boolean retainInput, boolean requiresBroadcast) {
+ super(indexName, indexType, databaseName, dataverseName, datasetName, retainInput, requiresBroadcast);
+ }
+
+ @Override
+ public void writeToFuncArgs(List<Mutable<ILogicalExpression>> funcArgs) {
+ super.writeToFuncArgs(funcArgs);
+ // indexOnly slot lives between the base params and the variable list — readers can locate it
+ // positionally at funcArgs[super.getNumParams()] without knowing the varlist length.
+ funcArgs.add(new MutableObject<>(AccessMethodUtils.createBooleanConstant(indexOnly)));
+ writeVarList(queryVarList, funcArgs);
+ }
+
+ @Override
+ public void readFromFuncArgs(List<Mutable<ILogicalExpression>> funcArgs) throws AlgebricksException {
+ super.readFromFuncArgs(funcArgs);
+ int index = super.getNumParams();
+ indexOnly = AccessMethodUtils.getBooleanConstant(funcArgs.get(index));
+ index++;
+ queryVarList = new ArrayList<>();
+ readVarList(funcArgs, index, queryVarList);
+ }
+
+ public void setQueryVarList(List<LogicalVariable> queryVarList) {
+ this.queryVarList = queryVarList;
+ }
+
+ public List<LogicalVariable> getQueryVarList() {
+ return queryVarList;
+ }
+
+ public void setIndexOnly(boolean indexOnly) {
+ this.indexOnly = indexOnly;
+ }
+
+ public boolean isIndexOnly() {
+ return indexOnly;
+ }
+}
\ No newline at end of file
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/util/ValidateUtil.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/util/ValidateUtil.java
index 333b82fe6a..593a1bf251 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/util/ValidateUtil.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/util/ValidateUtil.java
@@ -261,6 +261,7 @@ public class ValidateUtil {
switch (indexType) {
case ARRAY:
case BTREE:
+ case VTREE:
switch (fieldType.getTypeTag()) {
case TINYINT:
case SMALLINT:
@@ -278,8 +279,8 @@ public class ValidateUtil {
case DAYTIMEDURATION:
break;
case ANY:
- if (indexType == IndexType.BTREE) {
- // ANY is only allowed for normal secondary indexes
+ if (indexType == IndexType.BTREE || indexType == IndexType.VTREE) {
+ // ANY is allowed for BTREE and VECTOR (include fields) indexes
break;
}
default:
diff --git a/asterixdb/asterix-app/data/vector/glove-sub.adm b/asterixdb/asterix-app/data/vector/glove-sub.adm
new file mode 100644
index 0000000000..ce219e4d41
--- /dev/null
+++ b/asterixdb/asterix-app/data/vector/glove-sub.adm
@@ -0,0 +1,150 @@
+{"idx": 0, "label": "g0", "embedding": [-0.11332999914884567, 0.4840199947357178, 0.09077099710702896, -0.22439000010490417, 0.034205999225378036, -0.5583099722862244, 0.04184899851679802, -0.5357300043106079, 0.1880899965763092, -0.5872200131416321, 0.015313000418245792, -0.014554999768733978, 0.8084200024604797, -0.0385189987719059, 0.7534800171852112, 0.705020010471344, -0.17862999439239502, 0.3222000002861023, 0.6757500171661377, 0.6719800233840942, 0.26043999195098877, 0.41870000958 [...]
+{"idx": 1, "label": "g1", "embedding": [-0.9954400062561035, -2.3650999069213867, -0.24332000315189362, -1.032099962234497, 0.42052000761032104, -1.1816999912261963, -0.16450999677181244, -1.6829999685287476, 0.4967299997806549, -0.27257999777793884, -0.025397000834345818, 0.3418799936771393, 1.552299976348877, 1.3531999588012695, 0.33296999335289, -0.005667699966579676, -0.765250027179718, 0.4958699941635132, 1.2210999727249146, 0.8339400291442871, -0.20031000673770905, -0.5965700149536 [...]
+{"idx": 2, "label": "g2", "embedding": [-0.1018500030040741, 0.5981699824333191, 0.20281000435352325, 0.2524299919605255, -1.0583000183105469, -0.33024999499320984, 1.5149999856948853, -0.3101300001144409, -0.5248100161552429, 0.6327300071716309, 0.579479992389679, 1.4845000505447388, 0.7416300177574158, 0.359360009431839, 0.33215001225471497, 0.22267000377178192, -0.13083000481128693, -0.9412800073623657, -0.10467000305652618, -0.9260299801826477, 0.8826299905776978, 0.9027799963951111, [...]
+{"idx": 3, "label": "g3", "embedding": [-0.7327799797058105, -0.4715299904346466, 1.3312000036239624, 0.94514000415802, 0.3919300138950348, -0.1551699936389923, 0.7700099945068359, -0.4857099950313568, -0.2548600137233734, -0.5117200016975403, -0.6129599809646606, 0.8504999876022339, 1.2229000329971313, -0.7339199781417847, 0.20598000288009644, -0.32853999733924866, 0.4632500112056732, 1.6895999908447266, 1.0917999744415283, -0.07455500215291977, -0.10862000286579132, 0.02061099931597709 [...]
+{"idx": 4, "label": "g4", "embedding": [-0.21326999366283417, 0.6362199783325195, -0.5030400156974792, 0.17258000373840332, -0.5377299785614014, -0.5156599879264832, -0.49985000491142273, -1.0233999490737915, -0.3592199981212616, -1.399399995803833, 0.4357700049877167, 0.3476099967956543, 1.2699999809265137, 0.49540001153945923, -0.5752099752426147, -0.2080100029706955, -0.3730100095272064, 0.6117500066757202, 0.3132700026035309, 0.1779700070619583, 0.5214700102806091, -0.513270020484924 [...]
+{"idx": 5, "label": "g5", "embedding": [0.0442890003323555, 0.3195500075817108, 0.1504400074481964, -0.5662299990653992, -0.10915999859571457, -0.7914199829101562, 0.1951500028371811, 0.4062199890613556, -0.0005696499720215797, -0.006272300146520138, 0.16854000091552734, -0.030765000730752945, 0.6301100254058838, -0.10275000333786011, 0.038217999041080475, 0.4878399968147278, 0.1617799997329712, 0.7818300127983093, 0.33133000135421753, 0.4836699962615967, 0.1265600025653839, 0.0733319967 [...]
+{"idx": 6, "label": "g6", "embedding": [-0.14972999691963196, -0.07629500329494476, -0.36278000473976135, -0.2722899913787842, 0.7445999979972839, 0.10750000178813934, -0.4644399881362915, -0.23910999298095703, -0.06740300357341766, -0.5654699802398682, 0.29137998819351196, -0.13415999710559845, 0.2727699875831604, 0.19517000019550323, 0.3986800014972687, -0.3901999890804291, 0.6355400085449219, 0.652970016002655, 0.36847999691963196, -0.4257799983024597, -0.20889000594615936, -0.1995899 [...]
+{"idx": 7, "label": "g7", "embedding": [-0.1512800008058548, 0.40674999356269836, -0.01043500006198883, -0.2879999876022339, 1.1121000051498413, 0.3300600051879883, -0.2022400051355362, -0.572920024394989, 0.5303500294685364, -0.4060400128364563, -0.3432199954986572, 0.5129899978637695, 0.839460015296936, 0.053488001227378845, -0.3990199863910675, 0.7181199789047241, -0.3826799988746643, -0.762220025062561, 1.2730000019073486, -0.10858000069856644, 0.3646700084209442, -0.326339989900589, [...]
+{"idx": 8, "label": "g8", "embedding": [-1.032099962234497, 0.6323000192642212, -0.06442400068044662, -0.365200012922287, -0.3932099938392639, -0.3331100046634674, -0.22578999400138855, -0.34099000692367554, 0.0867059975862503, -0.12419000267982483, 0.36913999915122986, -0.19154000282287598, 1.183899998664856, -0.43255001306533813, 0.2128800004720688, -0.6383200287818909, 0.9547100067138672, 0.6857100129127502, 0.2628999948501587, 0.268779993057251, -0.058531999588012695, 0.8156899809837 [...]
+{"idx": 9, "label": "g9", "embedding": [-0.31189998984336853, -0.017993999645113945, -0.955560028553009, -0.2546199858188629, 0.3377699851989746, 0.63714998960495, -0.03886299952864647, 0.11513999849557877, -0.049150001257658005, -0.60139000415802, -0.05385199934244156, 0.49226999282836914, -2.186800003051758, 0.5229499936103821, -0.20976999402046204, 0.05969399958848953, -0.3080799877643585, 1.2232999801635742, 0.1712999939918518, -0.025250999256968498, -0.36294999718666077, -0.46761000 [...]
+{"idx": 10, "label": "g10", "embedding": [-0.4217199981212616, 0.746649980545044, -0.06778299808502197, -0.32982999086380005, 0.35936999320983887, -1.2430000305175781, -0.4123699963092804, -0.007579199969768524, 0.39087000489234924, -0.8938800096511841, 0.5170800089836121, -0.5129299759864807, -0.026806000620126724, 0.32221001386642456, 0.4603300094604492, -0.2990100085735321, -0.10225000232458115, -0.05715399980545044, 0.6668199896812439, 0.5902000069618225, -0.991569995880127, -0.68143 [...]
+{"idx": 11, "label": "g11", "embedding": [-0.3357599973678589, -0.49160000681877136, 0.1019899994134903, -0.07278700172901154, 0.3195900022983551, -0.10023000091314316, 0.23212000727653503, -0.21830999851226807, 0.17951999604701996, 0.3683899939060211, -0.4205400049686432, 0.7581400275230408, 0.6485099792480469, 0.3249500095844269, 0.08272700011730194, -0.22693000733852386, -0.11348000168800354, -0.03925900161266327, -0.3512499928474426, 0.10677000135183334, -0.1028200015425682, 0.674740 [...]
+{"idx": 12, "label": "g12", "embedding": [-0.7998300194740295, -0.2878200113773346, -0.4292300045490265, 0.421640008687973, -0.8674799799919128, 0.43786999583244324, 0.1358799934387207, -0.39146000146865845, -0.2961199879646301, 0.1254899948835373, 0.15328000485897064, 0.5549899935722351, 1.142899990081787, -0.19989000260829926, -0.1480800062417984, 0.015356999821960926, -0.4491199851036072, -0.2954699993133545, -0.21028999984264374, -0.19627000391483307, 2.1524999141693115, -0.635529994 [...]
+{"idx": 13, "label": "g13", "embedding": [-0.8731499910354614, 1.340499997138977, 0.030750000849366188, -0.5211600065231323, 0.3784399926662445, 0.24288000166416168, -0.24853000044822693, 0.6257699728012085, 0.9373499751091003, 0.5658199787139893, 0.5647100210189819, -0.8752999901771545, 1.4740999937057495, -0.6851900219917297, -0.18967999517917633, 1.8596999645233154, 0.007765099871903658, 0.02598400041460991, 0.6421099901199341, -0.27406999468803406, 0.46919000148773193, 1.117200016975 [...]
+{"idx": 14, "label": "g14", "embedding": [0.30862000584602356, -1.0233999490737915, 1.4838000535964966, 0.07450000196695328, 0.8006700277328491, -0.8500800132751465, 0.5228099822998047, -0.3631899952888489, -0.541450023651123, 0.004685999825596809, -0.39149999618530273, -0.524649977684021, 1.238800048828125, 0.09259100258350372, 0.22171999514102936, -0.2942799925804138, -0.014449999667704105, 0.4922800064086914, -0.3984000086784363, 0.7957500219345093, -0.45045000314712524, 0.23543000221 [...]
+{"idx": 15, "label": "g15", "embedding": [0.05908999964594841, -0.3169899880886078, 0.1062999963760376, -0.24728000164031982, -0.19345000386238098, -0.4133700132369995, -0.42261001467704773, -0.5369700193405151, 0.4964100122451782, -0.9191200137138367, -0.1278800070285797, -0.021821999922394753, 0.7031999826431274, -0.3510400056838989, 0.5665500164031982, 0.0734969973564148, 0.6448799967765808, 0.33430999517440796, 0.07254800200462341, 0.4603100121021271, -0.0518609993159771, 0.418000012 [...]
+{"idx": 16, "label": "g16", "embedding": [0.34894999861717224, 0.6516100168228149, 0.2460000067949295, 0.15271000564098358, 0.24451999366283417, 0.5330899953842163, -0.3600899875164032, -0.25470998883247375, 0.0037897001020610332, 0.6471899747848511, -0.5817300081253052, 0.20444999635219574, 0.5394300222396851, -0.003464299952611327, -0.5898299813270569, -0.3027400076389313, 0.1712699979543686, 0.10277999937534332, 0.5051199793815613, 0.8713799715042114, 0.3058600127696991, 0.78825002908 [...]
+{"idx": 17, "label": "g17", "embedding": [-0.8896200060844421, 1.319100022315979, -1.3942999839782715, 0.3517000079154968, 0.6401100158691406, 0.02285500057041645, 0.5746399760246277, 0.37852001190185547, -0.34964999556541443, -0.27017998695373535, 0.572409987449646, 1.1438000202178955, 1.2934999465942383, -1.3077000379562378, -0.4833599925041199, -0.3419699966907501, -0.4439600110054016, 0.6764600276947021, 0.6163300275802612, -0.07813999801874161, 0.8564299941062927, -0.219109997153282 [...]
+{"idx": 18, "label": "g18", "embedding": [-0.3232400119304657, 0.21987000107765198, -0.1162400022149086, -0.04013799875974655, 0.192780002951622, -0.10480999946594238, -0.23712000250816345, 0.013011000119149685, -0.2126300036907196, 0.36059001088142395, -0.9582300186157227, 0.15534000098705292, 0.29300999641418457, 0.35585999488830566, 0.567799985408783, -0.4161899983882904, -0.08893100172281265, 0.09468500316143036, 0.1345899999141693, 0.07093600183725357, 0.7816399931907654, 0.29159998 [...]
+{"idx": 19, "label": "g19", "embedding": [0.5490800142288208, 0.619700014591217, 0.054719001054763794, -0.6769300103187561, 0.22980999946594238, -0.2264299988746643, -0.5612300038337708, -0.0035353999119251966, -0.5778099894523621, -0.3057900071144104, -0.3813000023365021, 0.6792700290679932, 0.23549999296665192, -0.37724000215530396, 0.5453000068664551, 0.7810400128364563, -0.4546099901199341, 0.2843100130558014, 0.6827300190925598, 0.19936999678611755, -0.23904000222682953, 0.500429987 [...]
+{"idx": 20, "label": "g20", "embedding": [-0.48065000772476196, -0.8492000102996826, 0.3344799876213074, 0.09907100349664688, -1.320199966430664, 0.5963699817657471, -0.8638299703598022, -0.8763499855995178, 0.35155001282691956, -0.10619000345468521, -0.14061999320983887, -1.118899941444397, 0.6980400085449219, 0.5503100156784058, 0.15971000492572784, 0.10768000036478043, 0.14934000372886658, -0.009641099721193314, -0.1344500035047531, -0.7870000004768372, -1.0503000020980835, -0.0661519 [...]
+{"idx": 21, "label": "g21", "embedding": [0.3977299928665161, -0.5203400254249573, 0.06794600188732147, 0.3369700014591217, 0.4679499864578247, 0.4055899977684021, -0.011443999595940113, -0.07925599813461304, 0.5503900051116943, -0.31481000781059265, -0.27684998512268066, 0.17371000349521637, 0.77742999792099, -0.21743999421596527, 0.9764000177383423, 0.17452000081539154, 0.6503199934959412, -0.799589991569519, 0.15031999349594116, 0.03328299894928932, 0.2195100039243698, 0.4700799882411 [...]
+{"idx": 22, "label": "g22", "embedding": [0.4564000070095062, 0.3203200101852417, 0.09364300221204758, 0.33597999811172485, 0.9031100273132324, -0.29923999309539795, 0.2976999878883362, -0.2156199961900711, -0.19056999683380127, -0.5572699904441833, -1.0607000589370728, -0.7610499858856201, -0.060982998460531235, -0.347460001707077, 0.16958999633789062, 0.15639999508857727, 0.060343001037836075, 0.45138999819755554, 0.1822900027036667, -0.10971999913454056, -0.08999700099229813, -0.07355 [...]
+{"idx": 23, "label": "g23", "embedding": [0.24899999797344208, -0.19292999804019928, -0.4058400094509125, 0.043928999453783035, 0.33004000782966614, -0.9496399760246277, 1.0882999897003174, -0.3273000121116638, -0.2827700078487396, 0.05308900028467178, -0.23169000446796417, 0.4003300070762634, 1.738700032234192, 0.15423999726772308, -0.3556399941444397, -0.5190200209617615, -0.14667999744415283, -0.08728499710559845, 0.24191999435424805, 0.13492999970912933, 0.5179299712181091, 0.4297600 [...]
+{"idx": 24, "label": "g24", "embedding": [-0.20159000158309937, 0.5486400127410889, 0.32993000745773315, 0.8741999864578247, 0.687690019607544, -1.348099946975708, -0.24854999780654907, -0.18603000044822693, -0.4433499872684479, 1.3549000024795532, 0.5073400139808655, -0.9973099827766418, 0.8145599961280823, 1.1756000518798828, 1.2768000364303589, 0.8478000164031982, -0.300929993391037, 0.007591300178319216, -0.9162899851799011, -0.36267998814582825, -0.2634600102901459, 0.19953000545501 [...]
+{"idx": 25, "label": "g25", "embedding": [-0.2529500126838684, 0.9820700287818909, -0.09511999785900116, 1.1953999996185303, -0.32054001092910767, 0.8721699714660645, -0.22511999309062958, 0.2428400069475174, 0.6561300158500671, -0.5548099875450134, 0.14225000143051147, -0.4024200141429901, 0.2812199890613556, 0.3861599862575531, -0.3614499866962433, -1.2785999774932861, 0.15241999924182892, -0.11337000131607056, 0.14706000685691833, -0.1861799955368042, -0.6276599764823914, -0.355210006 [...]
+{"idx": 26, "label": "g26", "embedding": [-0.64860999584198, 0.691349983215332, -0.30066001415252686, 0.8554800152778625, -0.736739993095398, 0.1259399950504303, 0.6028199791908264, -2.0662999153137207, 1.0003999471664429, -0.8313500285148621, 0.31929001212120056, 0.21310999989509583, -0.1877799928188324, -0.31582000851631165, 0.220660001039505, 0.18061000108718872, -0.5306599736213684, 0.7144100069999695, -0.5092999935150146, 0.08624999970197678, -0.42337000370025635, 0.3825699985027313 [...]
+{"idx": 27, "label": "g27", "embedding": [-0.21879999339580536, -0.040699999779462814, -0.21973000466823578, -0.4453999996185303, -0.035833001136779785, -0.7388399839401245, -0.11040999740362167, 0.051642000675201416, 0.14172999560832977, -0.3401300013065338, -0.5761100053787231, -0.3672400116920471, 0.7608699798583984, -0.30570998787879944, -0.0690850019454956, -0.27379000186920166, -0.5375300049781799, 0.09094300121068954, -0.35390999913215637, 0.2399899959564209, -0.05724800005555153, [...]
+{"idx": 28, "label": "g28", "embedding": [0.11834999918937683, -0.3661800026893616, 0.7508999705314636, -0.23367999494075775, -1.4242000579833984, -0.014050000347197056, -1.4769999980926514, -0.14959999918937683, 0.6211000084877014, 0.13064000010490417, 0.2977699935436249, -0.11043000221252441, 0.34536001086235046, 0.41190001368522644, 0.37821000814437866, 1.2914999723434448, 0.304390013217926, 0.41999000310897827, 0.6318100094795227, -0.04901200160384178, -0.5241900086402893, -0.8261299 [...]
+{"idx": 29, "label": "g29", "embedding": [1.163599967956543, -0.2569800019264221, -1.0429999828338623, -0.34442999958992004, 0.48465999960899353, -0.37716999650001526, 0.19457000494003296, 0.5989099740982056, -0.4851900041103363, -0.6138200163841248, -0.5672299861907959, 0.06072299927473068, 0.4002299904823303, -0.3499999940395355, -0.4939599931240082, 0.21347999572753906, -0.10806000232696533, 0.6700699925422668, 0.31650999188423157, 0.013647000305354595, 0.3141700029373169, 0.051608998 [...]
+{"idx": 30, "label": "g30", "embedding": [0.21267999708652496, -1.736799955368042, -0.05222000181674957, -0.3513300120830536, -0.8264899849891663, -0.30842000246047974, 1.089900016784668, 0.3548099994659424, 0.1877399981021881, 0.039379000663757324, -0.38670000433921814, 1.2239999771118164, 1.7330000400543213, 0.1287200003862381, 0.03779999911785126, 0.4850800037384033, 0.9470700025558472, -0.267769992351532, 0.04563099890947342, 1.2821999788284302, -0.4616599977016449, 0.537460029125213 [...]
+{"idx": 31, "label": "g31", "embedding": [0.48921999335289, 0.36153000593185425, 0.47839000821113586, -0.146479994058609, 0.45868998765945435, -1.051300048828125, 0.1746699959039688, -0.7318900227546692, 0.30959999561309814, -0.4403800070285797, -0.21480000019073486, 0.17410999536514282, 0.6032000184059143, 0.15107999742031097, 0.7562199831008911, 0.7778599858283997, 0.17247000336647034, 0.6188600063323975, 0.35187000036239624, 0.10073000192642212, -0.052705999463796616, 0.53756999969482 [...]
+{"idx": 32, "label": "g32", "embedding": [-0.438289999961853, 0.14991000294685364, -0.2734900116920471, 0.13328999280929565, -0.6128100156784058, -0.07742100208997726, 0.1312199980020523, -0.11472000181674957, -0.4326600134372711, -0.4156300127506256, -0.5318300127983093, 0.5691400170326233, 0.3641600012779236, 0.4985100030899048, 0.08088000118732452, 0.08023899793624878, -0.36298999190330505, 1.2877000570297241, 0.6221399903297424, -0.21774999797344208, 0.35679998993873596, -0.390639990 [...]
+{"idx": 33, "label": "g33", "embedding": [-0.07257799804210663, -0.5223600268363953, 0.40540000796318054, -0.5117400288581848, 0.605459988117218, 0.4149099886417389, -0.9828299880027771, 0.5555999875068665, 0.9393399953842163, 0.23555000126361847, 0.5264899730682373, -0.8421599864959717, 0.18957999348640442, 0.7473700046539307, -0.4869900047779083, 0.03550200164318085, 0.16676999628543854, -0.7025600075721741, 0.9570000171661377, 0.1492300033569336, 0.46136000752449036, -0.40703999996185 [...]
+{"idx": 34, "label": "g34", "embedding": [0.22021999955177307, 0.4941900074481964, -0.36155998706817627, 0.8771399855613708, -1.1367000341415405, 0.31957000494003296, -0.6191200017929077, -0.5494599938392639, 0.3152199983596802, 0.31784000992774963, 0.5906699895858765, -0.16459999978542328, 1.1629999876022339, 0.960129976272583, -0.025171000510454178, 0.520389974117279, 0.1747100055217743, 0.24714000523090363, -0.5170199871063232, 0.1734600067138672, 0.9432899951934814, 0.266039997339248 [...]
+{"idx": 35, "label": "g35", "embedding": [0.024776000529527664, 0.02165899984538555, -0.6235700249671936, 0.08851999789476395, -0.828249990940094, -0.09118100255727768, -0.056008998304605484, 0.4518899917602539, 0.8898400068283081, -0.4961099922657013, -0.054836999624967575, -1.3276000022888184, 1.2524000406265259, 0.6682000160217285, -0.11326000094413757, -0.06703799962997437, 0.5730699896812439, -0.21859000623226166, 0.8131999969482422, 0.43705999851226807, 0.4777800142765045, 0.465739 [...]
+{"idx": 36, "label": "g36", "embedding": [-0.32743000984191895, 0.22885000705718994, -0.3452399969100952, -0.2164199948310852, -0.38436999917030334, -0.8015400171279907, -0.12713000178337097, -0.939079999923706, -0.7942399978637695, -0.02999899908900261, -0.43790000677108765, 0.5426300168037415, 1.0497000217437744, -0.6945800185203552, -0.3196699917316437, 0.46786001324653625, -0.9679200053215027, 0.47172001004219055, 0.4643000066280365, 0.8057500123977661, 1.8946000337600708, -0.5641400 [...]
+{"idx": 37, "label": "g37", "embedding": [-0.1926800012588501, 0.08184800297021866, 0.149399995803833, 0.04068699851632118, -0.6490399837493896, -0.22912999987602234, 0.15908999741077423, -0.706089973449707, 0.38955000042915344, -1.385200023651123, 0.027060000225901604, -0.3541699945926666, 0.36601001024246216, -0.02441900037229061, -0.3315500020980835, 0.5291100144386292, 0.6401900053024292, 0.5124099850654602, 0.2771100103855133, -0.36653000116348267, -0.31887000799179077, 0.2857800126 [...]
+{"idx": 38, "label": "g38", "embedding": [-0.6083700060844421, 0.05270199850201607, 0.3085800111293793, -0.7173299789428711, 0.43705999851226807, -0.0953579992055893, -0.06724099814891815, -0.07109499722719193, 0.3053399920463562, -0.26611000299453735, -0.19217999279499054, 0.8325499892234802, 0.2104800045490265, -0.01252800039947033, -0.2343199998140335, 0.6181700229644775, -0.34689998626708984, -0.028883999213576317, 1.1377999782562256, 0.4240399897098541, -0.681879997253418, 0.4244099 [...]
+{"idx": 39, "label": "g39", "embedding": [0.1921200007200241, 0.9983299970626831, -0.07083000242710114, 0.4170700013637543, -0.3693000078201294, -0.33428001403808594, -0.8764399886131287, -1.0920000076293945, 0.20544999837875366, -0.3578299880027771, 0.12976999580860138, -0.18342000246047974, 1.5937999486923218, -0.3397800028324127, -0.19598999619483948, 0.3274500072002411, 0.9023900032043457, 0.28356999158859253, -0.43244999647140503, -0.7025399804115295, -0.4195699989795685, 0.50581997 [...]
+{"idx": 40, "label": "g40", "embedding": [-0.1444299966096878, 0.4807099997997284, 0.10154999792575836, 5.0676000682869926e-05, 0.18367999792099, -0.17148999869823456, -0.6125400066375732, 0.12732000648975372, -0.2844499945640564, -0.3917900025844574, -0.14549000561237335, 0.13867999613285065, 0.1765100061893463, 0.3587400019168854, 0.5389900207519531, 0.09483999758958817, -0.17876000702381134, 0.6178399920463562, 0.3103399872779846, -0.17089000344276428, 0.16642999649047852, -0.39261999 [...]
+{"idx": 41, "label": "g41", "embedding": [0.08284500241279602, 1.0318000316619873, -0.4555400013923645, 0.43540000915527344, -0.8288099765777588, -0.26993998885154724, 0.15288999676704407, -0.3914699852466583, -0.4748699963092804, -0.42544999718666077, 0.27445000410079956, 0.22987000644207, 1.2171000242233276, -0.6189699769020081, -0.5447700023651123, 0.0488319993019104, -0.39555999636650085, -0.49424999952316284, 0.48131000995635986, -0.10642000287771225, 0.5440899729728699, -0.49564000 [...]
+{"idx": 42, "label": "g42", "embedding": [0.12612999975681305, 0.0824899971485138, 0.22119000554084778, 0.2872200012207031, 0.20151999592781067, -0.40031999349594116, -0.13423000276088715, 0.006257799919694662, 0.3649899959564209, -0.3471499979496002, 0.20667999982833862, -0.03430600091814995, 0.335970014333725, -0.3540700078010559, 0.25505000352859497, -0.15695999562740326, 0.3508799970149994, -0.03353499993681908, 0.929610013961792, 0.6516900062561035, -0.27070000767707825, -0.12240000 [...]
+{"idx": 43, "label": "g43", "embedding": [1.2454999685287476, -0.021640000864863396, 0.5502700209617615, -0.20830999314785004, -0.6682699918746948, -0.3816699981689453, -0.3501400053501129, 0.06339500099420547, 0.9432399868965149, 0.6345499753952026, 0.04104600101709366, 0.5166400074958801, 1.229200005531311, -0.4431900084018707, 0.34345000982284546, 0.8400099873542786, 0.28883999586105347, 0.7492600083351135, 1.417799949645996, -0.6369699835777283, 0.5253499746322632, -0.453220009803772 [...]
+{"idx": 44, "label": "g44", "embedding": [-0.642300009727478, -0.6753100156784058, 0.5371400117874146, 0.27630001306533813, 0.14169000089168549, 0.2698799967765808, 0.6386500000953674, -1.15910005569458, 0.7204200029373169, 0.2970399856567383, 0.28233999013900757, 0.4613899886608124, 0.829010009765625, -0.6851099729537964, -0.17680999636650085, 0.6917700171470642, 0.28306999802589417, -0.6771000027656555, 0.20906999707221985, 0.6492300033569336, -1.0638999938964844, 0.5077099800109863, - [...]
+{"idx": 45, "label": "g45", "embedding": [-0.47102999687194824, -1.0111000537872314, 0.7321400046348572, 0.29370999336242676, 0.23128999769687653, -0.30309000611305237, -0.8102099895477295, -0.1072700023651123, -0.5694500207901001, -0.41536998748779297, -0.30386000871658325, -0.6556900143623352, 0.2636699974536896, 0.24945999681949615, 0.31593000888824463, 0.3241499960422516, 0.7543200254440308, 0.030935000628232956, 0.6448799967765808, -0.4823099970817566, 0.4284299910068512, -0.9890900 [...]
+{"idx": 46, "label": "g46", "embedding": [-0.14324000477790833, 0.0863099992275238, 0.012329000048339367, 0.13812999427318573, -0.5530300140380859, 0.08045800030231476, -0.4364500045776367, -0.7063999772071838, -0.32806000113487244, -0.9381200075149536, 0.10459999740123749, 0.10563000291585922, 0.7929800152778625, -0.41947001218795776, -0.12592999637126923, 0.5551900267601013, 0.6934999823570251, 0.06064800173044205, -0.13118000328540802, -0.5022100210189819, -0.3714100122451782, -0.1591 [...]
+{"idx": 47, "label": "g47", "embedding": [-0.019726000726222992, 0.21708999574184418, -0.19001999497413635, -0.24874000251293182, 0.03453399986028671, -0.8881099820137024, -0.7959100008010864, 0.9803400039672852, 0.303739994764328, -0.5329399704933167, 0.3103199899196625, -0.16381999850273132, 0.4450399875640869, -0.3679099977016449, 0.32510000467300415, 0.24446000158786774, 0.17403000593185425, 0.5348100066184998, 0.35565000772476196, 0.8061299920082092, -0.01362099964171648, 0.16768999 [...]
+{"idx": 48, "label": "g48", "embedding": [0.43498000502586365, -0.045830998569726944, 1.0625, 0.2640399932861328, -1.3828999996185303, 0.5435600280761719, -0.21251000463962555, -0.04313899949193001, 0.08318299800157547, 0.538129985332489, 0.23222999274730682, 0.6165500283241272, -0.4470300078392029, 0.9929800033569336, 1.6025999784469604, -0.1465200036764145, 0.6428700089454651, 0.3915799856185913, -0.6461899876594543, 0.3610300123691559, 0.8277599811553955, 0.37411999702453613, 0.471760 [...]
+{"idx": 49, "label": "g49", "embedding": [-0.011769000440835953, 0.812690019607544, 0.24447999894618988, 0.4759500026702881, -0.2860899865627289, 0.041770998388528824, 0.2850100100040436, 0.04731399938464165, -0.09838700294494629, 0.33320000767707825, -0.2828499972820282, -0.23467999696731567, 0.809939980506897, 0.4238100051879883, 0.5958999991416931, -0.4294700026512146, 0.9358599781990051, 0.08961900323629379, 1.2383999824523926, 0.10362999886274338, 0.5309699773788452, 0.1377699971199 [...]
+{"idx": 50, "label": "g50", "embedding": [0.10832999646663666, 0.5019099712371826, -0.1542000025510788, -0.010696999728679657, 0.012741999700665474, -0.4731900095939636, -0.4670799970626831, 0.13342000544071198, -0.2341099977493286, -1.2782000303268433, 0.36114999651908875, 0.25960999727249146, 1.1155999898910522, -0.2267799973487854, 0.20613999664783478, 0.0495540015399456, -0.0005289100226946175, 0.03158099949359894, -0.1072700023651123, -0.08161400258541107, 0.25694000720977783, 0.074 [...]
+{"idx": 51, "label": "g51", "embedding": [-0.5392699837684631, 0.7053099870681763, -0.9789400100708008, 0.14270000159740448, -0.26982998847961426, 0.5738000273704529, -0.49277999997138977, -0.4940600097179413, 0.08167000114917755, -1.364799976348877, 0.47929999232292175, 0.40334001183509827, 0.18018999695777893, 0.8844299912452698, 0.3506599962711334, -0.585669994354248, -0.10390999913215637, 0.6567400097846985, -0.8018900156021118, 1.5399999618530273, -1.1203999519348145, -1.54990005493 [...]
+{"idx": 52, "label": "g52", "embedding": [0.2780799865722656, -0.4660100042819977, -0.5021600127220154, 0.4784199893474579, -0.45219001173973083, -1.246899962425232, 0.24442000687122345, 0.05337899923324585, 0.24164000153541565, 0.038047999143600464, -0.7742599844932556, -0.20945000648498535, 0.5477100014686584, -0.3818100094795227, -0.7074599862098694, -0.19964000582695007, -0.11732999980449677, -0.8685899972915649, 0.8111400008201599, -0.7702900171279907, 0.24981999397277832, 0.2704699 [...]
+{"idx": 53, "label": "g53", "embedding": [-0.26058998703956604, -0.43483999371528625, 0.2515299916267395, -0.41530001163482666, 0.22205999493598938, -0.1900400072336197, -0.5377900004386902, -0.7824100255966187, -0.8745700120925903, -0.5128800272941589, 0.21227000653743744, -0.19818000495433807, -0.8422099947929382, 1.243399977684021, 0.06831300258636475, 0.6949899792671204, 0.025955000892281532, 0.21699999272823334, -0.3340499997138977, 0.11520999670028687, -0.876479983329773, -0.133300 [...]
+{"idx": 54, "label": "g54", "embedding": [-1.0413999557495117, 0.9106500148773193, -0.0761450007557869, 0.6129400134086609, 0.3843100070953369, 0.3572100102901459, 0.1701900064945221, 0.1290999948978424, -0.1897599995136261, -0.3170199990272522, 0.16784000396728516, 0.9888799786567688, 0.3260599970817566, 0.049897000193595886, 0.2063799947500229, -0.420960009098053, 0.25071001052856445, 0.2658500075340271, 0.5546799898147583, 0.5601500272750854, -0.036247000098228455, 0.19621999561786652 [...]
+{"idx": 55, "label": "g55", "embedding": [-0.14357000589370728, -0.10869000107049942, 0.5486599802970886, 0.032290998846292496, -0.26868000626564026, -0.894569993019104, 0.578719973564148, 0.542930006980896, 0.09358199685811996, -0.4446699917316437, -0.9280499815940857, 0.0856660008430481, -0.32844001054763794, -0.25718000531196594, -0.11197999864816666, -0.05808199942111969, 0.09225700050592422, 0.2857300043106079, -0.8083199858665466, 0.43678000569343567, 0.47773000597953796, 0.0717779 [...]
+{"idx": 56, "label": "g56", "embedding": [0.49237000942230225, -0.3524700105190277, -0.4672600030899048, -0.9678599834442139, -0.8714600205421448, -0.5192099809646606, -0.36083999276161194, 0.48256000876426697, -0.028843000531196594, -1.4381999969482422, 0.24560999870300293, 0.9779599905014038, 0.6777399778366089, 0.2738800048828125, -1.333400011062622, 0.6725299954414368, 1.1734000444412231, -0.5631099939346313, -0.5023999810218811, -0.453249990940094, 0.768809974193573, 0.3365899920463 [...]
+{"idx": 57, "label": "g57", "embedding": [0.034212999045848846, 0.01938600093126297, -0.7487199902534485, 0.27147001028060913, 0.028811000287532806, -0.22842000424861908, -0.24705000221729279, -0.257099986076355, -0.3372800052165985, -0.05656599998474121, -0.2508299946784973, 0.7368599772453308, 1.4000999927520752, 0.1296200007200241, 0.4564799964427948, -0.2840000092983246, -0.6379299759864807, -0.6166099905967712, 0.46693000197410583, 0.8815600275993347, 1.1832000017166138, 0.072683997 [...]
+{"idx": 58, "label": "g58", "embedding": [-0.04742399975657463, 0.5625799894332886, 0.20381000638008118, 0.164900004863739, 0.6223199963569641, -0.4634700119495392, -0.3470500111579895, -0.526419997215271, 0.7222499847412109, -0.40400999784469604, -0.03829000145196915, 0.05062999948859215, 0.9284800291061401, 0.23770000040531158, -0.3409300148487091, -0.1871200054883957, -0.11552999913692474, 0.45649001002311707, 0.7785500288009644, 0.7643399834632874, 0.19089999794960022, 0.694530010223 [...]
+{"idx": 59, "label": "g59", "embedding": [0.11339999735355377, -0.5279099941253662, -0.4805999994277954, -0.4566499888896942, -0.2037999927997589, -0.738510012626648, 0.9934800267219543, -0.5300099849700928, 0.1906999945640564, 0.20778000354766846, -0.19006000459194183, 0.14119000732898712, 0.6121600270271301, -0.35938000679016113, 0.014155999757349491, -0.3355099856853485, -0.5240799784660339, 0.6955999732017517, 0.11029999703168869, 0.4474000036716461, -0.014967000111937523, 0.65188002 [...]
+{"idx": 60, "label": "g60", "embedding": [-0.7445200085639954, -0.31738999485969543, 0.21295000612735748, 0.20542000234127045, -0.4993799924850464, -0.8579800128936768, 0.6659899950027466, -0.9425600171089172, -0.6273800134658813, 0.2637700140476227, -0.42781001329421997, -0.1204100027680397, 0.5882499814033508, -0.12759999930858612, -0.831089973449707, 0.7348700165748596, -0.46053001284599304, -0.12190999835729599, -0.608959972858429, -0.11029999703168869, 0.24269999563694, 0.4761799871 [...]
+{"idx": 61, "label": "g61", "embedding": [0.9605299830436707, 0.9356200098991394, 0.3412100076675415, 0.15806999802589417, 0.6259599924087524, -0.7709800004959106, -0.1133200004696846, 0.5618799924850464, -0.7061300277709961, -0.688979983329773, 0.7479000091552734, -0.9713299870491028, 0.47091999650001526, 0.1791599988937378, -1.124400019645691, 0.3318600058555603, -0.461650013923645, 1.5306999683380127, 1.0128999948501587, -1.5595999956130981, 0.9226400256156921, -0.007705000229179859, [...]
+{"idx": 62, "label": "g62", "embedding": [-0.2912600040435791, 0.2563599944114685, -0.1549299955368042, 0.8459399938583374, 0.18203000724315643, -0.4424299895763397, 0.47593000531196594, 0.5378699898719788, 0.004321199841797352, -0.6086900234222412, 0.1804099977016449, 0.16559000313282013, 0.8078399896621704, -0.029478000476956367, -0.5568100214004517, -0.6352099776268005, -0.13043999671936035, 0.4018099904060364, 0.7930300235748291, 0.012524000369012356, 0.4106299877166748, -0.690559983 [...]
+{"idx": 63, "label": "g63", "embedding": [-0.8589699864387512, 1.2822999954223633, 0.21514999866485596, -0.18828000128269196, 0.6105200052261353, -0.5473200082778931, -0.6200000047683716, -0.10367000102996826, -0.2943199872970581, -1.4266999959945679, -0.16051000356674194, 1.0515999794006348, 1.1500999927520752, -0.6604899764060974, 0.11383000016212463, -1.0546000003814697, -0.01282500009983778, 0.6930699944496155, -0.23808999359607697, 0.7759400010108948, 0.3574399948120117, -1.02799999 [...]
+{"idx": 64, "label": "g64", "embedding": [-0.4939900040626526, 0.6980699896812439, 0.021740999072790146, -0.2745800018310547, 0.39559999108314514, -0.3716599941253662, -0.6347000002861023, 0.35343000292778015, 0.15137000381946564, -0.5430300235748291, 0.015456999652087688, 0.2809399962425232, 0.4892300069332123, 0.30768001079559326, 0.2613599896430969, -0.41826000809669495, -0.1685200035572052, 0.22506999969482422, 0.06461399793624878, 0.7683200240135193, 0.10193999856710434, 0.008710600 [...]
+{"idx": 65, "label": "g65", "embedding": [-0.9450899958610535, -0.4909299910068512, -0.14395999908447266, 0.5112699866294861, 0.20841999351978302, -0.08440300077199936, -0.166360005736351, -0.9146999716758728, -0.17669999599456787, -0.7808099985122681, 0.38741999864578247, -0.19220000505447388, 0.05955100059509277, -0.23469999432563782, 0.4002299904823303, 0.17103999853134155, 0.2575399875640869, -0.10034999996423721, 0.0019950000569224358, 0.3147299885749817, 0.27955999970436096, -0.656 [...]
+{"idx": 66, "label": "g66", "embedding": [-0.1550000011920929, 0.09213399887084961, 0.010462000034749508, 0.25255998969078064, 0.06543300300836563, -0.1325100064277649, 0.19168999791145325, -0.6840100288391113, 0.20927000045776367, -0.162990003824234, -0.657260000705719, 0.5776200294494629, 1.128100037574768, -0.6762700080871582, 0.28321999311447144, -0.031505998224020004, 0.16741999983787537, -0.976170003414154, 0.9276999831199646, 0.28578001260757446, -0.13946999609470367, 0.3995099961 [...]
+{"idx": 67, "label": "g67", "embedding": [-0.7093899846076965, 0.28248998522758484, 0.05785999819636345, 0.013181000016629696, 0.5198799967765808, -0.3122499883174896, -0.18651999533176422, -0.21279999613761902, -0.4716300070285797, -0.6845600008964539, -0.009391799569129944, -0.4719099998474121, 0.5545499920845032, 0.12904000282287598, 0.3756699860095978, 0.035225000232458115, 0.8376100063323975, 0.25666001439094543, 0.23292000591754913, -0.44029998779296875, -0.8160399794578552, -0.271 [...]
+{"idx": 68, "label": "g68", "embedding": [-0.19550999999046326, -0.4379799962043762, -0.2951200008392334, -1.0326000452041626, 0.18046000599861145, 0.7461699843406677, 0.13504000008106232, -1.2706999778747559, 0.2757999897003174, 0.16410000622272491, 0.736299991607666, 0.09115900099277496, 0.6711699962615967, -0.278329998254776, -1.353700041770935, 0.35218000411987305, -0.7723299860954285, 1.0688999891281128, -0.31033000349998474, 0.32653000950813293, 0.7323799729347229, -0.4025799930095 [...]
+{"idx": 69, "label": "g69", "embedding": [-0.9203699827194214, 0.43887999653816223, -0.00989769957959652, -0.6502400040626526, 0.43696001172065735, 0.017784999683499336, 0.34863999485969543, 0.22958000004291534, -0.11462000012397766, -0.08312699943780899, 0.18783000111579895, 0.11234000325202942, 1.0154999494552612, 0.13500000536441803, 0.2934800088405609, 0.7190700173377991, 0.4310699999332428, 0.06319999694824219, -0.2109600007534027, 0.14103999733924866, 0.14385999739170074, -0.320309 [...]
+{"idx": 70, "label": "g70", "embedding": [0.20112000405788422, -1.3752000331878662, -0.3933199942111969, -0.3831700086593628, 0.4458500146865845, -0.6328799724578857, -0.3005099892616272, -0.42493000626564026, 0.3379899859428406, 0.31529000401496887, -0.5264599919319153, 0.3756200075149536, 0.8601700067520142, 0.16699999570846558, -0.10409999638795853, 0.6082500219345093, 0.32433998584747314, 0.08452799916267395, 0.6502199769020081, 0.17566999793052673, 0.3423300087451935, -0.05491999909 [...]
+{"idx": 71, "label": "g71", "embedding": [0.0664220005273819, 0.8593999743461609, -0.4683400094509125, 0.5816900134086609, -0.631820023059845, -0.3452500104904175, -0.8882099986076355, 0.08683999627828598, 1.2792999744415283, 0.6272299885749817, -0.2887899875640869, 0.3847599923610687, 1.0713000297546387, -0.4017699956893921, 0.24235999584197998, 1.374899983406067, 0.7621600031852722, 0.3062700033187866, -0.5021799802780151, 0.9575600028038025, -0.6202800273895264, -0.21083000302314758, [...]
+{"idx": 72, "label": "g72", "embedding": [0.302480012178421, -0.0735429972410202, 0.5044199824333191, -0.42906999588012695, -0.19620999693870544, -0.7178900241851807, -1.1013000011444092, -0.5698099732398987, 0.516480028629303, -0.7135900259017944, -0.0988209992647171, -0.8527699708938599, 0.6444900035858154, 0.2961699962615967, -0.16200999915599823, -0.3819800019264221, 0.31532999873161316, -0.16572999954223633, 0.5053399801254272, -0.158610001206398, 1.248900055885315, -1.0480999946594 [...]
+{"idx": 73, "label": "g73", "embedding": [-0.16392000019550323, 0.6445800065994263, 0.38185998797416687, 0.43257999420166016, 0.4170199930667877, 0.27237001061439514, 0.3767699897289276, -0.401419997215271, -0.00513519998639822, -1.4577000141143799, -0.6513299942016602, -0.6492599844932556, 0.3569900095462799, -1.1683000326156616, 0.6256499886512756, -0.32978999614715576, 0.3343000113964081, -0.16884000599384308, -0.1451999992132187, 0.15376000106334686, -0.08439300209283829, -0.21999999 [...]
+{"idx": 74, "label": "g74", "embedding": [0.9921200275421143, -0.14122000336647034, -0.1553100049495697, 0.9791200160980225, -0.3359600007534027, -0.2966899871826172, -0.37384000420570374, -0.1898999959230423, -0.7235699892044067, -0.12223999947309494, -0.1637600064277649, 0.0034564000088721514, 0.1520099937915802, 0.013082999736070633, -0.6069300174713135, -0.6243100166320801, 0.8159700036048889, 0.7066299915313721, 0.8148499727249146, -0.07883799821138382, 0.8087599873542786, 0.3456499 [...]
+{"idx": 75, "label": "g75", "embedding": [-0.4094499945640564, -0.36640000343322754, 1.3040000200271606, -0.4849799871444702, -0.17802000045776367, 0.12190999835729599, -0.12592999637126923, 0.754040002822876, -0.12383999675512314, 0.12851999700069427, -0.6528199911117554, 0.18895000219345093, 0.03031099960207939, -0.7195299863815308, 0.6208900213241577, -0.18806999921798706, -0.9408000111579895, -0.1324699968099594, -0.3549000024795532, 0.17869000136852264, 0.010885999538004398, 0.80786 [...]
+{"idx": 76, "label": "g76", "embedding": [1.0082000494003296, 0.5732300281524658, 0.1990099996328354, 0.7153000235557556, -0.39653998613357544, -0.3394800126552582, 0.1666799932718277, -0.9866399765014648, 0.8583700060844421, -0.5960800051689148, 0.05460600182414055, -0.4237000048160553, 0.7521600127220154, 1.0326999425888062, 0.2186799943447113, -0.6450499892234802, 0.5295500159263611, -0.20160000026226044, 0.24893000721931458, 0.6543599963188171, 0.10864999890327454, 0.9885299801826477 [...]
+{"idx": 77, "label": "g77", "embedding": [-0.3508700132369995, -0.20138999819755554, 0.13381999731063843, -0.3304400146007538, 0.43048998713493347, -1.034500002861023, -0.01584699936211109, -0.23529000580310822, 0.06237500160932541, -0.2720400094985962, -0.8351799845695496, 1.7836999893188477, 0.39169999957084656, -0.2576099932193756, -0.5991399884223938, 0.010881000198423862, -0.057819001376628876, 0.28317999839782715, -0.08228400349617004, -1.3550000190734863, 1.704300045967102, -0.387 [...]
+{"idx": 78, "label": "g78", "embedding": [-1.3077000379562378, 0.26467999815940857, -0.6551100015640259, -0.26353999972343445, 0.05403599888086319, -0.2568399906158447, -0.4775199890136719, -0.19467000663280487, -0.15109999477863312, -0.6140999794006348, 0.02301100082695484, 0.683459997177124, 1.6958999633789062, -0.9808099865913391, -0.41310998797416687, -0.2253900021314621, -0.19612999260425568, 0.7994099855422974, 0.7685199975967407, 0.39860999584198, 0.034327998757362366, -0.64902001 [...]
+{"idx": 79, "label": "g79", "embedding": [-0.13087999820709229, 0.4274199903011322, -0.01672299951314926, 0.23681999742984772, 0.5766800045967102, -0.45267999172210693, 0.0764629989862442, 0.2027599960565567, 0.26278001070022583, 0.2613700032234192, -0.6703699827194214, -0.04736899957060814, 0.6998199820518494, -0.33605000376701355, 0.6860299706459045, -0.03583899885416031, 0.7944700121879578, 0.056292999535799026, -0.12382999807596207, -0.0745680034160614, 0.01933700032532215, -0.244749 [...]
+{"idx": 80, "label": "g80", "embedding": [-0.2457900047302246, 0.7005100250244141, 0.03643399849534035, -0.4279699921607971, -0.9627500176429749, -0.5962700247764587, -0.4630100131034851, -0.37303000688552856, 0.10824999958276749, -0.6436600089073181, 0.5416300296783447, 0.060371000319719315, 1.187999963760376, 0.915090024471283, 0.09828200191259384, 0.09859099984169006, 0.06721899658441544, 0.594569981098175, -0.5083799958229065, 0.31391000747680664, 1.1002999544143677, -0.0226709991693 [...]
+{"idx": 81, "label": "g81", "embedding": [-0.1476600021123886, 1.3950999975204468, -0.5217599868774414, 0.6402000188827515, -0.03622300177812576, -0.3104200065135956, 0.5978299975395203, -0.38550999760627747, 0.5852000117301941, -0.020948000252246857, 0.1089399978518486, -0.1087300032377243, 0.45381999015808105, -0.08197899907827377, -0.8657900094985962, 0.004029700066894293, 0.27039000391960144, 0.40915998816490173, 0.2782999873161316, -0.020771000534296036, 0.6548399925231934, -0.14811 [...]
+{"idx": 82, "label": "g82", "embedding": [0.8138099908828735, -0.14393000304698944, -0.6517400145530701, -0.47088000178337097, 0.9315599799156189, -0.2915099859237671, -0.3395000100135803, -0.9026299715042114, 0.08527500182390213, -0.18320000171661377, -0.5170999765396118, -0.05467100068926811, 1.4882999658584595, -0.03544899821281433, 0.08612900227308273, 0.41916999220848083, -0.2400200068950653, -1.1638000011444092, -0.1525000035762787, 0.9530500173568726, -0.1390099972486496, -0.23442 [...]
+{"idx": 83, "label": "g83", "embedding": [0.8879299759864807, -0.5458099842071533, -0.47269999980926514, -0.29225000739097595, 0.4260199964046478, -0.6212599873542786, 0.016140999272465706, -0.23512999713420868, 0.26534000039100647, 0.1328199952840805, 0.4505299925804138, 0.39708998799324036, 1.5817999839782715, -0.4686700105667114, 0.2067199945449829, 0.16453999280929565, -0.6857399940490723, -0.051316998898983, 0.9264500141143799, 0.5543400049209595, 1.1188000440597534, -0.803810000419 [...]
+{"idx": 84, "label": "g84", "embedding": [0.3969399929046631, -0.9262099862098694, 0.5752500295639038, -0.6112899780273438, -0.21734000742435455, -0.35653001070022583, 0.2442300021648407, -0.244609996676445, -0.4046599864959717, -0.14202000200748444, 0.4782100021839142, 0.46568000316619873, 1.340399980545044, 0.132860004901886, -0.024073999375104904, -0.6559399962425232, 0.23343999683856964, 0.3047100007534027, 0.923009991645813, -0.0015625000232830644, -0.09698399901390076, 0.1080899983 [...]
+{"idx": 85, "label": "g85", "embedding": [-0.025498000904917717, -0.24706000089645386, 0.13075000047683716, -0.47433000802993774, 0.06787800043821335, 0.03673499822616577, 0.02357099950313568, 0.40501999855041504, 0.38534998893737793, -0.2108599990606308, 0.03777699917554855, -0.039101000875234604, 0.5878700017929077, 0.30827000737190247, 0.11432000249624252, 0.08502999693155289, -0.15737000107765198, -0.23015999794006348, -0.30640000104904175, 0.04065300151705742, 0.6667199730873108, -0 [...]
+{"idx": 86, "label": "g86", "embedding": [-0.006662100087851286, -0.22450999915599823, 0.13619999587535858, -0.3637999892234802, -0.004939599893987179, -0.487529993057251, -0.4699999988079071, -1.7581000328063965, -0.15594999492168427, -0.3564099967479706, 0.5303900241851807, -0.07579299807548523, 0.9148100018501282, -0.3749200105667114, 0.11309000104665756, 0.2193399965763092, -0.3723199963569641, 0.19109000265598297, 0.7689599990844727, -0.5649999976158142, 0.198730006814003, 0.8824499 [...]
+{"idx": 87, "label": "g87", "embedding": [-0.7001100182533264, -0.3167000114917755, -0.2779400050640106, 0.7636799812316895, -0.11627999693155289, -0.02142699994146824, 0.11027000099420547, -0.28141000866889954, 0.19520999491214752, 0.09945099800825119, -0.12655000388622284, 0.29385000467300415, 0.26638999581336975, 0.47154000401496887, 0.20999999344348907, -0.3099299967288971, 0.2604599893093109, -0.08364800363779068, -0.2722800076007843, 0.168380007147789, -0.14158999919891357, 0.24504 [...]
+{"idx": 88, "label": "g88", "embedding": [0.043021999299526215, -0.3146199882030487, 0.10510999709367752, 0.9526399970054626, 0.3332099914550781, 0.5120999813079834, -0.2606000006198883, -0.002983999904245138, 0.08391900360584259, -0.5678499937057495, 0.20343999564647675, 0.16366000473499298, 0.182109996676445, -0.4965299963951111, 0.39017000794410706, 0.4129500091075897, -0.009559700265526772, 0.3671500086784363, 0.1980700045824051, 0.6410599946975708, -0.2517299950122833, -0.1405999958 [...]
+{"idx": 89, "label": "g89", "embedding": [-0.1755799949169159, 0.01618500053882599, -0.047276999801397324, -0.41091999411582947, -0.23871000111103058, -0.08105900138616562, -0.6455100178718567, -0.5270500183105469, -0.47832998633384705, 0.09685800224542618, -0.5610499978065491, -0.12873999774456024, 0.24618999660015106, -0.5184299945831299, -0.7353000044822693, 0.6161199808120728, 0.2806900143623352, -0.7706900238990784, -0.12563000619411469, 0.5094500184059143, 0.29587000608444214, -0.3 [...]
+{"idx": 90, "label": "g90", "embedding": [0.8724499940872192, 0.0147280003875494, -0.9970300197601318, 0.564079999923706, 0.14809000492095947, 0.32934999465942383, 0.28071001172065735, 0.06083400174975395, 0.5148699879646301, -0.40536999702453613, -0.6098999977111816, 0.46511000394821167, -1.232100009918213, -0.4638200104236603, 0.15703999996185303, 0.8745599985122681, 0.27195999026298523, 0.75941002368927, -0.8813199996948242, 1.3466999530792236, 0.6134300231933594, -0.5657399892807007, [...]
+{"idx": 91, "label": "g91", "embedding": [-0.5555199980735779, -0.0020055000204592943, -0.4150800108909607, -0.7089700102806091, -0.2979399859905243, 0.6755499839782715, -0.46074000000953674, -0.7182599902153015, -0.3583199977874756, -0.6308199763298035, -0.2718000113964081, -0.8354600071907043, 1.243499994277954, 0.23115000128746033, -0.0043298001401126385, 0.3075000047683716, 0.871999979019165, 0.5194600224494934, 1.6090999841690063, 0.656029999256134, -0.6878399848937988, -0.819040000 [...]
+{"idx": 92, "label": "g92", "embedding": [-0.16401000320911407, 0.4663099944591522, -0.1408499926328659, -0.216839998960495, -0.4498800039291382, 0.5552800297737122, -0.34411001205444336, 0.6316499710083008, -0.72052001953125, 0.3711099922657013, -0.2336599975824356, -0.6647800207138062, 0.7487300038337708, 0.8371899724006653, -0.06107399985194206, 0.07713600248098373, 0.18448999524116516, 0.7997300028800964, 0.6698499917984009, -0.009549099951982498, -0.6052799820899963, -0.894129991531 [...]
+{"idx": 93, "label": "g93", "embedding": [-0.40261000394821167, 0.20699000358581543, -0.19314999878406525, -0.2653299868106842, 0.025774000212550163, -0.46094998717308044, -0.3151699900627136, 0.026384999975562096, 0.08973100036382675, -0.27052998542785645, -0.35242000222206116, 0.31358999013900757, 0.14094999432563782, 0.27167001366615295, -0.05468500033020973, -0.2614000141620636, -0.015344999730587006, 0.19171999394893646, 0.22362999618053436, 0.4096199870109558, -0.1290999948978424, [...]
+{"idx": 94, "label": "g94", "embedding": [-0.24052999913692474, -0.14656999707221985, 0.17624999582767487, -0.14760999381542206, 0.255950003862381, -0.023740999400615692, -0.0944409966468811, -0.483379989862442, -0.5690100193023682, -0.09296000003814697, -0.6135500073432922, -0.29495999217033386, 1.2770999670028687, 0.09192699939012527, -0.2891499996185303, -0.013260000385344028, 0.13868999481201172, 0.31817999482154846, -0.7911800146102905, 0.8638100028038025, -0.3248499929904938, 0.800 [...]
+{"idx": 95, "label": "g95", "embedding": [0.010874000377953053, -0.39274999499320984, -0.005528099834918976, 0.9860299825668335, -0.07492200285196304, -0.3372099995613098, 0.10420999675989151, -0.1222200021147728, 0.02779500000178814, -0.6469799876213074, -0.20058000087738037, 0.18339000642299652, 0.46456998586654663, -0.26840999722480774, -0.019938999786973, -0.769819974899292, -0.13163000345230103, -0.4057599902153015, -0.16076000034809113, -0.2048099935054779, -0.40981999039649963, -0 [...]
+{"idx": 96, "label": "g96", "embedding": [0.056963000446558, -0.9329500198364258, -0.4763000011444092, -0.3778199851512909, -0.447160005569458, -0.5362399816513062, -0.3205600082874298, 0.1144300028681755, -0.7117300033569336, -0.7029600143432617, -0.15671999752521515, 0.04176099970936775, 1.5770000219345093, -0.2150299996137619, -1.0611000061035156, 0.28582999110221863, -0.3788500130176544, -0.22367000579833984, 0.7121000289916992, 0.7596399784088135, 0.8296700119972229, 0.2596200108528 [...]
+{"idx": 97, "label": "g97", "embedding": [-0.2437400072813034, -0.36362001299858093, 0.3118399977684021, 0.04752400144934654, 0.5801399946212769, 0.6593899726867676, 0.16408999264240265, 0.017136000096797943, -0.6012799739837646, -0.6343700289726257, -0.13344000279903412, 0.1886799931526184, 0.36812999844551086, 0.7944499850273132, -0.7320799827575684, 0.6557400226593018, 0.9177600145339966, 1.524899959564209, 0.9521300196647644, -0.14646999537944794, 0.9839800000190735, 0.64828997850418 [...]
+{"idx": 98, "label": "g98", "embedding": [0.18161000311374664, 0.8037400245666504, -1.1974999904632568, -0.7356399893760681, 0.5636199712753296, -0.6061400175094604, 0.9584900140762329, -1.2695000171661377, -0.09241999685764313, -1.129699945449829, 0.3070099949836731, 0.342629998922348, 1.6678999662399292, -0.27904999256134033, -1.2187000513076782, 0.13576999306678772, 0.09292600303888321, 0.46990999579429626, -0.17010000348091125, -0.045928001403808594, 0.4853299856185913, 0.28764998912 [...]
+{"idx": 99, "label": "g99", "embedding": [-0.21095000207424164, -0.08745100349187851, -0.003206999972462654, 0.031373001635074615, -0.04294800013303757, -0.6172299981117249, 0.015386000275611877, -1.2400000095367432, -0.2379399985074997, -0.3305700123310089, -0.367000013589859, -0.19076000154018402, 0.5313100218772888, 0.241239994764328, 0.21834999322891235, 0.16301999986171722, 0.18942999839782715, -0.07541400194168091, 0.7553099989891052, -0.508870005607605, 0.5769500136375427, -0.0934 [...]
+{"idx": 100, "label": "g100", "embedding": [0.3128899931907654, 0.20973999798297882, 0.16022999584674835, -0.4705899953842163, -0.2107200026512146, -0.4030799865722656, 0.019804999232292175, -0.26010000705718994, -0.3343600034713745, -0.3512600064277649, 0.605400025844574, 0.6102799773216248, 0.3259600102901459, -0.24470999836921692, 1.6480000019073486, -0.3088499903678894, 0.5696099996566772, 0.09213300049304962, 0.46108001470565796, -0.12421999871730804, 0.7457699775695801, -0.55690002 [...]
+{"idx": 101, "label": "g101", "embedding": [1.3384000062942505, 0.2471500039100647, 0.3755300045013428, -0.07449900358915329, 0.7402600049972534, -0.4499000012874603, 0.5731499791145325, 0.49928000569343567, 0.5410199761390686, 0.5457500219345093, -0.05370499938726425, -0.2078700065612793, 1.0116000175476074, -0.19066999852657318, 0.9728800058364868, -0.08343199640512466, 0.009707000106573105, 0.2645699977874756, -0.09776599705219269, 0.09562499821186066, 0.30674999952316284, 0.516700029 [...]
+{"idx": 102, "label": "g102", "embedding": [-0.015003999695181847, 0.03275499865412712, -0.5974400043487549, 0.616349995136261, -0.5980299711227417, 0.30726000666618347, 0.32462000846862793, 0.21140000224113464, 0.3666900098323822, -0.12065000087022781, 0.2291799932718277, 0.03798900172114372, 0.45511001348495483, 0.6057400107383728, 0.027580000460147858, -0.08982399851083755, 0.49994000792503357, 0.05341099947690964, -0.34435999393463135, 0.38043999671936035, 0.22901999950408936, -0.064 [...]
+{"idx": 103, "label": "g103", "embedding": [0.9272900223731995, -1.31850004196167, 0.04327499866485596, -0.7228299975395203, -0.09933000057935715, 0.6745799779891968, -0.7559400200843811, -0.7266799807548523, 0.6418899893760681, -0.7971400022506714, -0.36010000109672546, -0.2741999924182892, 0.4572399854660034, -0.4548400044441223, -0.3904699981212616, 1.0252000093460083, 1.0343999862670898, -0.5085200071334839, 0.8222600221633911, 0.44808000326156616, 0.08178199827671051, 0.951399981975 [...]
+{"idx": 104, "label": "g104", "embedding": [0.7973300218582153, -0.46386998891830444, 0.050443001091480255, -1.5010000467300415, -1.218400001525879, -0.966949999332428, 0.424019992351532, -0.5084999799728394, -0.3371100127696991, -0.5878099799156189, -0.31617000699043274, 0.3870199918746948, 1.5843000411987305, 0.07109100371599197, -0.119889996945858, -0.37689998745918274, -0.7514899969100952, -0.1868000030517578, 0.43035998940467834, 0.794700026512146, 0.9689300060272217, 0.430189996957 [...]
+{"idx": 105, "label": "g105", "embedding": [-0.28174999356269836, 0.31602001190185547, 0.8410000205039978, -0.2801100015640259, 0.10621000081300735, 0.6535300016403198, -0.9720399975776672, -0.328110009431839, -0.08442000299692154, -0.8792799711227417, -0.03808100149035454, 0.20161999762058258, -0.9826899766921997, 0.322519987821579, 0.20757000148296356, -0.9970600008964539, -0.2037699967622757, 0.7028300166130066, -0.7155100107192993, -0.4076499938964844, 0.01359300035983324, 0.80172002 [...]
+{"idx": 106, "label": "g106", "embedding": [-0.6133800148963928, -0.6976100206375122, -0.3151699900627136, -0.07619599997997284, 0.03519299998879433, -0.25126001238822937, -0.15665000677108765, 0.023809000849723816, 0.24525000154972076, -0.3667599856853485, -0.5184699892997742, -0.15561999380588531, -0.04108700156211853, -0.28685998916625977, -0.14988000690937042, 0.06964199990034103, 0.012772000394761562, 0.6436300277709961, -0.5826399922370911, 0.4555799961090088, -0.04605399817228317, [...]
+{"idx": 107, "label": "g107", "embedding": [-0.47672000527381897, -0.21243999898433685, -0.6767299771308899, 0.6903600096702576, -0.27772998809814453, -0.15448999404907227, -0.23367999494075775, -0.16137999296188354, -0.9212899804115295, -0.47310999035835266, -0.2955699861049652, 0.5686799883842468, 0.5792999863624573, -1.1531000137329102, 0.4583599865436554, -0.5381699800491333, -1.0120999813079834, 0.28459998965263367, 0.1769299954175949, 0.712119996547699, 1.452299952507019, -0.389919 [...]
+{"idx": 108, "label": "g108", "embedding": [-0.9983699917793274, -0.9579700231552124, 0.2141599953174591, -0.23593999445438385, -0.4210300147533417, -0.02841299958527088, -0.39493000507354736, -0.21527999639511108, -0.1156499981880188, -0.6222400069236755, -0.4348900020122528, -0.1610500067472458, 0.5467399954795837, -0.10461000353097916, -0.4324899911880493, -0.476500004529953, -1.0180000066757202, -0.14969000220298767, -0.5049600005149841, 0.2041500061750412, -0.025388000532984734, 1.1 [...]
+{"idx": 109, "label": "g109", "embedding": [-0.48050999641418457, -0.02827399969100952, 0.09089499711990356, 0.9603700041770935, -0.3614499866962433, 0.05548499897122383, -0.21448999643325806, 0.30169999599456787, -0.13371999561786652, 0.28769999742507935, 0.20141999423503876, 0.8894199728965759, -0.045253999531269073, -0.6736000180244446, 0.4171000123023987, -1.249400019645691, -0.3667699992656708, 1.1476000547409058, -0.26528000831604004, 0.8728899955749512, 0.12191999703645706, -0.758 [...]
+{"idx": 110, "label": "g110", "embedding": [1.5996999740600586, 0.7344300150871277, -0.28727999329566956, -0.10106000304222107, -0.6177800297737122, 0.33052998781204224, -0.8693299889564514, -0.723330020904541, -0.6405500173568726, -0.7233099937438965, -0.3443000018596649, -0.5172299742698669, 0.8435199856758118, 0.5410000085830688, -0.038019001483917236, -0.2727600038051605, 0.07162600010633469, 0.6826300024986267, 0.426800012588501, -0.18522000312805176, -0.14027999341487885, 0.0491649 [...]
+{"idx": 111, "label": "g111", "embedding": [-0.38148999214172363, -0.4228399991989136, 0.326090008020401, 0.22164000570774078, 0.1780800074338913, -0.42642998695373535, 0.39271000027656555, 0.6979399919509888, 0.607509970664978, -0.7005299925804138, -1.1430000066757202, 0.5166000127792358, 0.15883000195026398, -0.9760599732398987, -1.4258999824523926, -0.13444000482559204, -0.02464899979531765, 0.24851000308990479, -0.056710999459028244, -0.18955999612808228, 0.26256000995635986, -0.5209 [...]
+{"idx": 112, "label": "g112", "embedding": [1.0224000215530396, 1.0151000022888184, -0.25760000944137573, 0.03779200091958046, 0.09525000303983688, 0.9046000242233276, -0.07757599651813507, 0.6724100112915039, -0.3013499975204468, -1.0606000423431396, 1.2094999551773071, -0.36882999539375305, 1.0006999969482422, -0.24361999332904816, -0.04912199825048447, -0.18796999752521515, 0.650160014629364, 0.6634899973869324, 0.6765099763870239, 0.27726998925209045, 0.6689299941062927, -0.350360006 [...]
+{"idx": 113, "label": "g113", "embedding": [-0.9063299894332886, 0.5704900026321411, -0.19363999366760254, -0.151869997382164, -0.5119900107383728, -0.7185500264167786, -0.8341500163078308, 0.1382800042629242, 0.6614500284194946, -1.0226999521255493, 0.5444999933242798, -0.49518001079559326, 1.4061000347137451, -0.5984399914741516, 0.03323100134730339, -1.274999976158142, 0.4670200049877167, -0.03448000177741051, -0.33886000514030457, -0.4487600028514862, 0.742110013961792, -0.0241059996 [...]
+{"idx": 114, "label": "g114", "embedding": [0.20302000641822815, 0.9251700043678284, -1.220900058746338, 0.1429399996995926, 0.09129399806261063, -0.182219997048378, 0.8659700155258179, 0.2844800055027008, -0.8539299964904785, -0.8513000011444092, 0.2357099950313568, 0.8680400252342224, 1.8809000253677368, -0.5980799794197083, -0.39364999532699585, -0.30008000135421753, -0.244159996509552, 0.3446100056171417, 0.4580700099468231, 0.4326600134372711, 1.0569000244140625, -0.3370800018310547 [...]
+{"idx": 115, "label": "g115", "embedding": [0.05509699881076813, -0.585860013961792, -1.5226999521255493, -0.6212499737739563, -0.7262600064277649, -0.5992299914360046, -0.5098699927330017, 0.9677500128746033, 0.7734400033950806, 0.4689300060272217, 0.6693400144577026, -0.2639000117778778, 0.08030600100755692, 0.36542001366615295, -0.5249599814414978, -0.24177999794483185, 0.7148200273513794, -1.1952999830245972, 1.3122999668121338, 0.4028399884700775, 0.8920999765396118, -0.465420007705 [...]
+{"idx": 116, "label": "g116", "embedding": [-0.6088899970054626, 0.09882400184869766, 0.026360999792814255, -0.29868000745773315, 0.185479998588562, -0.03606100007891655, -0.0886240005493164, -0.30125001072883606, 0.7089099884033203, -0.7321699857711792, 0.08395499736070633, 0.41356998682022095, 0.7414500117301941, 0.1881999969482422, 0.09386300295591354, -0.03451799973845482, 0.7646499872207642, 0.3616800010204315, -0.5766500234603882, 0.6604200005531311, -0.011958000250160694, 0.206679 [...]
+{"idx": 117, "label": "g117", "embedding": [-0.7206799983978271, 0.06810899823904037, 0.9695900082588196, -0.09285300225019455, 0.1964700073003769, -0.5508700013160706, -0.34332001209259033, -0.47209998965263367, -0.18240000307559967, -0.821150004863739, 0.06412199884653091, -0.34696000814437866, -0.2703000009059906, -0.3308599889278412, 0.17576000094413757, -0.2911899983882904, 0.07332099974155426, 0.1603900045156479, -0.1718199998140335, 0.6376100182533264, -0.11807999759912491, 0.1850 [...]
+{"idx": 118, "label": "g118", "embedding": [-0.15076999366283417, 0.6466799974441528, -0.0032250001095235348, -0.19558000564575195, -0.4756399989128113, -1.0075000524520874, -1.069200038909912, -0.6715599894523621, -0.2043900042772293, -0.09984999895095825, 0.5878999829292297, -0.7588499784469604, -0.36267000436782837, 0.7982100248336792, 0.46693000197410583, -0.4174399971961975, 0.1548900008201599, 0.9767600297927856, -0.10604000091552734, -0.11670000106096268, 0.09942000359296799, 0.06 [...]
+{"idx": 119, "label": "g119", "embedding": [-0.08169499784708023, -0.018662000074982643, 0.36967000365257263, -0.44415000081062317, 0.23017999529838562, -0.7644100189208984, -0.8428000211715698, -0.1180500015616417, 0.37376001477241516, -0.5450400114059448, -0.22821000218391418, -0.7228000164031982, -0.3694100081920624, -0.45552998781204224, 0.28126001358032227, 0.1007699966430664, 0.3154299855232239, 0.00918199960142374, -0.1470700055360794, 0.5947499871253967, -0.3450300097465515, -0.6 [...]
+{"idx": 120, "label": "g120", "embedding": [-0.17023000121116638, -0.16708999872207642, -0.780269980430603, -0.5211499929428101, -0.5729100108146667, 0.4330199956893921, 0.08082299679517746, -0.5848199725151062, 0.20433999598026276, -0.48614999651908875, 0.3762100040912628, -0.14194999635219574, 0.9701700210571289, 0.7340199947357178, 0.714739978313446, 0.5436000227928162, 0.7212200164794922, -0.17583000659942627, 0.4268699884414673, 1.0841000080108643, 0.11422999948263168, 0.19602000713 [...]
+{"idx": 121, "label": "g121", "embedding": [0.3179999887943268, 0.6893600225448608, -0.35600998997688293, 0.12174999713897705, 0.1877399981021881, -0.14191000163555145, -0.2507599890232086, -0.7508400082588196, 0.3722600042819977, 0.7466599941253662, 1.070099949836731, -0.16674000024795532, 0.9905300140380859, -0.35624998807907104, -0.47023001313209534, 0.14414000511169434, 0.958840012550354, -0.021490000188350677, -0.2506900131702423, -0.09617999941110611, -0.76978999376297, -0.37606999 [...]
+{"idx": 122, "label": "g122", "embedding": [-1.7578999996185303, -0.4306100010871887, -0.5463500022888184, 0.11643999814987183, 1.0561000108718872, -0.20338000357151031, 0.5962499976158142, -0.03823599964380264, -0.8037099838256836, 0.21490000188350677, -0.5154200196266174, 0.17966000735759735, 0.5075700283050537, 0.1301800012588501, -0.9663699865341187, 0.35697001218795776, 0.1876700073480606, 0.30414000153541565, 0.2454500049352646, -0.8651400208473206, -0.16053999960422516, -0.6872599 [...]
+{"idx": 123, "label": "g123", "embedding": [0.34937000274658203, -0.5138199925422668, -0.3006100058555603, -0.40178999304771423, 0.859279990196228, -0.10074000060558319, 0.5259799957275391, -0.6560800075531006, -0.03997199982404709, -0.09928800165653229, 0.3233799934387207, 0.1950400024652481, 0.5170599818229675, -0.3255600035190582, 0.13728000223636627, 0.30504000186920166, 0.3035599887371063, -0.045986998826265335, 0.8748700022697449, 0.44064998626708984, -0.19833000004291534, 0.530780 [...]
+{"idx": 124, "label": "g124", "embedding": [0.008454700000584126, 0.6361799836158752, -0.5022199749946594, 0.44207999110221863, -0.34615999460220337, 0.15241999924182892, 0.6386799812316895, 0.14936000108718872, 0.17447000741958618, -0.9456999897956848, 0.7004600167274475, -0.44846999645233154, 0.09447000175714493, 0.733460009098053, -0.555109977722168, 0.9476199746131897, 0.34393998980522156, 0.0061703999526798725, 0.0009129500249400735, 0.15772999823093414, -0.44843998551368713, 1.0872 [...]
+{"idx": 125, "label": "g125", "embedding": [0.13765999674797058, -0.6411600112915039, 1.0159000158309937, -0.34784001111984253, 0.803409993648529, 0.030233999714255333, 0.5632200241088867, -0.47290998697280884, -0.3905799984931946, -0.3940899968147278, -0.052046999335289, 0.9277300238609314, 1.0920000076293945, 0.12681999802589417, -0.4614199995994568, 0.8077300190925598, -0.6839699745178223, -0.4973599910736084, 0.4060800075531006, -0.41071000695228577, -0.547510027885437, 0.12176000326 [...]
+{"idx": 126, "label": "g126", "embedding": [0.9680500030517578, -0.09256400167942047, -0.09895899891853333, 0.5378900170326233, -0.03678100183606148, -0.052848998457193375, -0.21852999925613403, 0.5565599799156189, -0.7385799884796143, 0.23114000260829926, -0.4790799915790558, -0.8362600207328796, 0.64274001121521, -0.6891599893569946, 0.01222699973732233, -0.36309999227523804, 1.281499981880188, -0.9398699998855591, -1.1601999998092651, 0.2797999978065491, -0.32635998725891113, 0.301750 [...]
+{"idx": 127, "label": "g127", "embedding": [-0.9128699898719788, 1.1354999542236328, 0.7994899749755859, 1.839900016784668, -0.9291999936103821, 1.6519999504089355, 0.7082899808883667, -0.045951999723911285, -0.5054200291633606, 0.610230028629303, 0.3357200026512146, 0.516759991645813, 1.001099944114685, -0.20218999683856964, -1.1490000486373901, 0.9556400179862976, 0.32339999079704285, 1.6101000308990479, 0.11688999831676483, 0.5087699890136719, -0.6557999849319458, -0.313620001077652, [...]
+{"idx": 128, "label": "g128", "embedding": [-0.3024100065231323, 0.7818599939346313, -1.2110999822616577, -0.16328999400138855, -0.35214000940322876, 1.1647000312805176, -0.5277500152587891, 0.71288001537323, -0.41604000329971313, -0.8733800053596497, -0.26686999201774597, -1.1095000505447388, 0.06946100294589996, 1.1262999773025513, -0.4379599988460541, -0.3976300060749054, -0.0428909994661808, 0.4073199927806854, -0.04010000079870224, 0.9110699892044067, 0.5125700235366821, -0.45403000 [...]
+{"idx": 129, "label": "g129", "embedding": [-0.5613999962806702, 0.07401199638843536, -0.31327998638153076, -0.09705100208520889, 0.7685800194740295, 0.03956000134348869, -0.5022299885749817, -1.107800006866455, 0.2742699980735779, 0.9796800017356873, 0.5135599970817566, 0.06059399992227554, 0.6426399946212769, 0.7103000283241272, 0.015065000392496586, 0.19970999658107758, 0.6321499943733215, 0.40915000438690186, -0.3044300079345703, -0.3847000002861023, -0.1969199925661087, -0.044992998 [...]
+{"idx": 130, "label": "g130", "embedding": [0.7940899729728699, 0.059126999229192734, -0.0022291000932455063, -1.1095999479293823, -1.4635000228881836, 0.8392699956893921, -0.11756999790668488, 0.1609800010919571, -0.09956400096416473, -0.705299973487854, 1.1790000200271606, -1.2811000347137451, 1.0292999744415283, 0.37303999066352844, -0.41319000720977783, -0.0015045000473037362, -0.15399999916553497, -0.8073599934577942, 0.5027999877929688, 0.2992599904537201, 0.05481100082397461, -0.5 [...]
+{"idx": 131, "label": "g131", "embedding": [-0.33390000462532043, 0.27656999230384827, 0.3421800136566162, 0.24258999526500702, -0.15796999633312225, 0.057686999440193176, -0.01964299939572811, 0.1623699963092804, 0.4395799934864044, -0.42083001136779785, -0.7835299968719482, -0.16272999346256256, 0.5511999726295471, 0.23789000511169434, 0.5254899859428406, 0.2575699985027313, 0.08274299651384354, 0.2092999964952469, -0.0363599993288517, -0.49300000071525574, -0.5344099998474121, 0.39989 [...]
+{"idx": 132, "label": "g132", "embedding": [-0.14762000739574432, 0.12830999493598938, -0.6777899861335754, -0.6652899980545044, 0.384660005569458, 0.2780599892139435, -0.8922100067138672, -0.3184399902820587, -0.21845999360084534, 0.09360100328922272, 0.42059001326560974, -0.3218899965286255, 0.817520022392273, -0.06325499713420868, -0.0059022000059485435, -0.5703499913215637, 0.19156000018119812, 0.32444000244140625, -0.010653999634087086, 0.31064000725746155, -0.293040007352829, -0.34 [...]
+{"idx": 133, "label": "g133", "embedding": [-0.07326100021600723, 0.4062800109386444, -1.0052000284194946, 0.12533999979496002, -0.4171299934387207, 0.40132999420166016, -0.08525700122117996, 0.4866800010204315, -0.385699987411499, 0.010280000045895576, 0.1523600071668625, 0.33952000737190247, 0.986519992351532, 1.1334999799728394, 0.1447100043296814, -0.32662999629974365, 0.3196299970149994, -0.37307000160217285, -0.273470014333725, -0.1257299929857254, -0.47762998938560486, 0.270240008 [...]
+{"idx": 134, "label": "g134", "embedding": [-0.872160017490387, 0.10464999824762344, -0.288100004196167, -0.366890013217926, -0.5118799805641174, -0.7126200199127197, 0.9982399940490723, 0.23226000368595123, 0.010293000377714634, 0.6748200058937073, 0.8623300194740295, -0.12273000180721283, 0.2503499984741211, -1.5303000211715698, -0.9929800033569336, 0.6940900087356567, -0.11512000113725662, -1.2194000482559204, 0.4401800036430359, -0.7448300123214722, -0.5252799987792969, 1.88680005073 [...]
+{"idx": 135, "label": "g135", "embedding": [-0.7032600045204163, 0.02551800012588501, 0.27031999826431274, 0.338019996881485, 0.6267300248146057, -0.40171998739242554, -0.2674799859523773, -0.006410100031644106, -0.320360004901886, -0.6980900168418884, -0.13319000601768494, -0.05379299819469452, 0.21979999542236328, -0.23159000277519226, -0.43459999561309814, 0.8467400074005127, 0.28804001212120056, 0.10837999731302261, -0.5211600065231323, 0.14920000731945038, 0.4809899926185608, 0.9720 [...]
+{"idx": 136, "label": "g136", "embedding": [-0.31244000792503357, -0.4571399986743927, 0.006658900063484907, -0.4872100055217743, 0.9009900093078613, 0.32938000559806824, -0.6573399901390076, -0.6288400292396545, 0.842769980430603, -0.5937299728393555, -0.824400007724762, 0.19840000569820404, 0.6349400281906128, -0.38457998633384705, -0.47683998942375183, 0.6593999862670898, 0.1331000030040741, -0.8454899787902832, -0.10834000259637833, -0.7088500261306763, -0.20777000486850739, -0.62445 [...]
+{"idx": 137, "label": "g137", "embedding": [0.23887999355793, -0.8922200202941895, -0.09903600066900253, -0.28507000207901, -0.4505699872970581, -0.18421000242233276, 0.09373600035905838, 0.24997000396251678, 0.45974001288414, 0.18283000588417053, -0.23469999432563782, -0.5543500185012817, 0.5507500171661377, -0.39162999391555786, 0.12488999962806702, 0.2122800052165985, 0.47828999161720276, 0.29971998929977417, 0.5544899702072144, -0.2362000048160553, 0.10644999891519547, -0.02510499954 [...]
+{"idx": 138, "label": "g138", "embedding": [-0.6562399864196777, 0.8199099898338318, 0.10560999810695648, 0.1973699927330017, 0.18147000670433044, -0.34376001358032227, -0.0764319971203804, -0.7942000031471252, -0.027874000370502472, -0.18390999734401703, -0.1722699999809265, -0.6261600255966187, 1.2187999486923218, 0.17666000127792358, 0.6763799786567688, -0.04269799962639809, 0.4395599961280823, -0.28582000732421875, 0.713919997215271, -0.42715999484062195, -0.7196000218391418, -0.2522 [...]
+{"idx": 139, "label": "g139", "embedding": [-0.017393000423908234, 0.2758899927139282, -0.19378000497817993, 0.6099500060081482, -0.514460027217865, 0.45524999499320984, 0.38670000433921814, 0.18363000452518463, -0.7661100029945374, -0.2932800054550171, -1.125599980354309, 0.814079999923706, 0.9332200288772583, 0.41655001044273376, 0.32276999950408936, 0.3947100043296814, 0.09958100318908691, 0.4702500104904175, -0.0469839982688427, 0.30566999316215515, 1.6239999532699585, -0.11671999841 [...]
+{"idx": 140, "label": "g140", "embedding": [-0.8853899836540222, 0.9707499742507935, 0.8378700017929077, 0.10592000186443329, -0.06834500283002853, -0.738569974899292, -0.2537800073623657, 0.5218899846076965, 0.4124700129032135, -0.18668000400066376, 0.6163600087165833, 0.38047999143600464, 0.9614499807357788, 0.20898999273777008, -0.03928700089454651, -0.436599999666214, 0.9645699858665466, -1.111199975013733, 0.47394999861717224, 0.5367100238800049, -0.9607700109481812, 1.0784000158309 [...]
+{"idx": 141, "label": "g141", "embedding": [-0.5506399869918823, 0.25029999017715454, -0.1913599967956543, 0.20097999274730682, 0.006249099969863892, 0.4926399886608124, 0.6208900213241577, 0.7935199737548828, -0.4267300069332123, 0.8725100159645081, -0.4821400046348572, -0.12240000069141388, -2.449700117111206, 0.38187000155448914, -0.21024000644683838, -0.8023300170898438, -0.7730799913406372, 0.4162200093269348, -0.4572399854660034, -0.45120999217033386, -0.03342200070619583, -0.09505 [...]
+{"idx": 142, "label": "g142", "embedding": [0.5267199873924255, 0.0892300009727478, -0.7953100204467773, 0.9667900204658508, 0.09637399762868881, -0.2130499929189682, 0.9745500087738037, 0.009895400144159794, -0.9909999966621399, -0.5454400181770325, 0.3260599970817566, 0.6233599781990051, 0.9164199829101562, 0.13955999910831451, -0.6488699913024902, -0.17836999893188477, -0.737559974193573, 0.5524399876594543, 0.08234500139951706, 0.08261100202798843, 1.8287999629974365, -0.086724996566 [...]
+{"idx": 143, "label": "g143", "embedding": [-0.10516999661922455, 0.03433600068092346, -0.10790000110864639, -0.7666699886322021, 0.4528000056743622, 0.4412600100040436, 0.46952998638153076, -0.01918799988925457, 0.20291000604629517, -0.14249999821186066, -0.14294999837875366, 0.7099000215530396, 1.1872999668121338, -0.279449999332428, -0.002014300087466836, 0.3665100038051605, -0.31571000814437866, -0.30129000544548035, 1.020900011062622, -0.9133399724960327, -0.22585999965667725, -0.21 [...]
+{"idx": 144, "label": "g144", "embedding": [-0.003826000029221177, -0.8304799795150757, -0.043418001383543015, -0.23711000382900238, -0.7512199878692627, -0.22558000683784485, 0.41165000200271606, -0.06363999843597412, -0.2581700086593628, 0.27858999371528625, -0.11685000360012054, -0.00190250005107373, 0.16282999515533447, -0.2345300018787384, 0.09097199887037277, -0.17552000284194946, 0.31652000546455383, -0.058316998183727264, 0.2809799909591675, 0.3147599995136261, 0.6240599751472473 [...]
+{"idx": 145, "label": "g145", "embedding": [0.08975499868392944, 0.045177001506090164, 0.1498900055885315, -0.8740299940109253, 0.3743700087070465, -0.006831999868154526, -0.6500599980354309, 0.38071000576019287, 1.1748000383377075, 0.7813699841499329, 0.38071998953819275, 0.24873000383377075, -1.1988999843597412, 0.2922300100326538, 0.6367700099945068, -0.59375, 0.26680999994277954, -0.3953700065612793, 0.2849699854850769, 0.0639910027384758, -1.0462000370025635, -0.011208999902009964, [...]
+{"idx": 146, "label": "g146", "embedding": [-0.48548999428749084, -0.08857700228691101, -0.7743399739265442, 0.8776500225067139, 0.3109999895095825, -0.22076000273227692, 0.3354099988937378, 0.2942799925804138, 0.10548000037670135, -0.1847900003194809, 0.007272799964994192, 0.10780999809503555, 1.0608999729156494, 0.41071999073028564, -0.23048000037670135, 0.11846999824047089, -0.37935999035835266, 0.13109999895095825, 0.24946999549865723, -0.134210005402565, 0.2713499963283539, 0.490599 [...]
+{"idx": 147, "label": "g147", "embedding": [0.521809995174408, -1.2925000190734863, 0.8597400188446045, -0.26541998982429504, -0.5039100050926208, -0.4796299934387207, 0.1252799928188324, 0.5757499933242798, -0.891480028629303, 0.07982099801301956, -0.6439700126647949, 1.1446000337600708, 0.9107300043106079, 0.23859000205993652, 0.48054999113082886, 0.23864999413490295, 0.13061000406742096, 0.2902899980545044, 0.3041200041770935, -1.086899995803833, 0.13555000722408295, -0.13154999911785 [...]
+{"idx": 148, "label": "g148", "embedding": [-0.36528000235557556, -0.2199999988079071, 0.3207300007343292, -0.3163999915122986, -0.3168700039386749, 0.6606299877166748, -0.28470999002456665, -0.28464001417160034, 0.8443099856376648, -0.3203999996185303, 0.4658600091934204, 0.04585399851202965, 0.5129600167274475, -0.10053999722003937, 0.7456799745559692, -0.14213000237941742, -0.15725000202655792, -0.004133000038564205, 1.590999960899353, -0.3580400049686432, 0.2185100018978119, 0.724979 [...]
+{"idx": 149, "label": "g149", "embedding": [0.39583998918533325, -0.027223000302910805, 0.19965000450611115, -0.16023999452590942, -0.008614099584519863, -0.2750000059604645, -0.2788200080394745, -0.07225000113248825, -0.5987200140953064, 0.29377999901771545, 0.2211499959230423, 0.15331999957561493, 1.194000005722046, -0.21484999358654022, 0.360289990901947, 0.7149199843406677, -0.16259999573230743, 0.20961999893188477, 0.34505000710487366, 0.39559000730514526, 0.7390499711036682, -0.735 [...]
diff --git a/asterixdb/asterix-app/data/vector/movie-sub.adm b/asterixdb/asterix-app/data/vector/movie-sub.adm
new file mode 100644
index 0000000000..d19699847a
--- /dev/null
+++ b/asterixdb/asterix-app/data/vector/movie-sub.adm
@@ -0,0 +1,100 @@
+{"id": 27205, "title": "Inception", "embedding": [0.0074708107858896255, 0.01605764962732792, -0.02256624959409237, -0.042012039572000504, 0.09289159625768661, 0.027462318539619446, 0.08246946334838867, 0.00459109153598547, 0.047774769365787506, -0.012208864092826843, 0.02889857068657875, -0.010192588903009892, 0.005875263828784227, -0.010019898414611816, -0.014396922662854195, -0.07372946292161942, 0.005000752862542868, 0.05144509673118591, 0.007719757501035929, 0.01593409664928913, 0.0 [...]
+{"id": 157336, "title": "Interstellar", "embedding": [-0.06534450501203537, -0.10422594845294952, 0.021496161818504333, 0.04403414949774742, -0.051738958805799484, -0.09265454113483429, 0.019126025959849358, 0.003753772471100092, -0.028400886803865433, 0.02002832666039467, 0.04667805880308151, -0.03443469479680061, -0.06969400495290756, 0.031318433582782745, -0.09394200891256332, 0.04678508639335632, -0.00684673385694623, -0.030039764940738678, -0.02875950001180172, -0.0724523738026619, [...]
+{"id": 155, "title": "The Dark Knight", "embedding": [0.004041258245706558, -0.018151411786675453, -0.10712436586618423, -0.015724288299679756, -0.012203955091536045, 0.05338031426072121, -0.009702371433377266, 0.022262055426836014, 0.010085919871926308, 0.0032537891529500484, 0.020376531407237053, -0.03608819097280502, -0.004181609023362398, -0.0028944071382284164, 0.01324950996786356, 0.00614866754040122, 0.12382546067237854, 0.06673227995634079, 0.03238803520798683, 0.0295276567339897 [...]
+{"id": 19995, "title": "Avatar", "embedding": [0.01766314171254635, 0.0034245525021106005, 0.042760543525218964, -0.04438752681016922, -0.06908498704433441, 0.01715397834777832, 0.020456260070204735, -0.07457631081342697, 0.011736799031496048, 0.04816338047385216, 0.04214431345462799, -0.07037214189767838, 0.01747010461986065, 0.010077621787786484, -0.026647601276636124, 0.055234987288713455, -0.034295327961444855, -0.005706708878278732, 0.021168183535337448, 0.046531230211257935, -0.025 [...]
+{"id": 24428, "title": "The Avengers", "embedding": [-0.04097116366028786, -0.016031388193368912, -0.02782243676483631, -0.011035189963877201, 0.075645312666893, 0.06583024561405182, 0.021633662283420563, -0.009832819923758507, -0.009151695296168327, 0.019916724413633347, -0.01535104215145111, -0.010393813252449036, 0.020910732448101044, 0.10423088073730469, -0.005330364219844341, -0.013783476315438747, 0.05474638193845749, -0.08134400844573975, -0.0396093986928463, -0.06890099495649338, [...]
+{"id": 293660, "title": "Deadpool", "embedding": [-0.08656146377325058, -0.008958287537097931, -0.0822606310248375, 0.006896054372191429, 0.06375760585069656, -0.0022290190681815147, 0.11754728853702545, -0.02342323213815689, -0.007781205233186483, 0.03417909890413284, 0.07090871036052704, 0.028612298890948296, 0.04337339103221893, 0.10099700838327408, -0.008770574815571308, -0.015380672179162502, 0.04476097598671913, -0.0224334504455328, -0.03241995349526405, 0.06490734964609146, 0.0122 [...]
+{"id": 299536, "title": "Avengers: Infinity War", "embedding": [-0.0626683458685875, 0.010252736508846283, -0.010392706841230392, -0.021597255021333694, -0.00664761196821928, -0.002956453477963805, 0.0033624775242060423, 0.021097682416439056, -0.00023520819377154112, 0.0567607544362545, 0.007625790312886238, -0.05223942548036575, -0.017817294225096703, 0.042817916721105576, -0.0335819274187088, -0.0016447476809844375, 0.039287663996219635, -0.06834211200475693, -0.08146953582763672, 0.00 [...]
+{"id": 550, "title": "Fight Club", "embedding": [-0.03570679947733879, -0.04635647311806679, -0.07093056291341782, 0.04855087772011757, -0.023422883823513985, 0.06103140115737915, 0.07590265572071075, -0.04595165327191353, 0.052109017968177795, -0.010463393293321133, 0.010511365719139576, -0.0414559431374073, 0.048043347895145416, -0.012877294793725014, 0.009251934476196766, 0.0435866042971611, -0.0020317775197327137, 0.0034593092277646065, 0.06093442067503929, -0.026927096769213676, -0. [...]
+{"id": 118340, "title": "Guardians of the Galaxy", "embedding": [-0.06597890704870224, 0.01807456463575363, 0.03017498180270195, -0.03059355542063713, 0.018546979874372482, -0.0479787215590477, 0.026667119935154915, 0.03619087114930153, 0.017570050433278084, -0.02245848812162876, 0.07434871792793274, -0.07051984965801239, -0.009344032034277916, 0.02626677416265011, 0.03515423834323883, -0.041280340403318405, -0.013762333430349827, -0.09044752269983292, 0.0027215222362428904, -0.100910089 [...]
+{"id": 680, "title": "Pulp Fiction", "embedding": [-0.01857922412455082, -0.0614464208483696, -0.07124938815832138, 0.003173143370077014, -0.026651009917259216, 0.07428602129220963, 0.014454759657382965, 0.0009055715636350214, 0.09926825761795044, -0.05026109889149666, 0.050097666680812836, 0.041439659893512726, 0.009396902285516262, 0.04711673781275749, 0.004123533610254526, -0.03571753948926926, 0.03176308423280716, 0.0560125932097435, 0.04764378443360329, 0.009818535298109055, -0.0056 [...]
+{"id": 13, "title": "Forrest Gump", "embedding": [-0.06272141635417938, -0.04941535368561745, 0.009312952868640423, 0.08713451772928238, -0.02156771346926689, 0.10508010536432266, 0.048381101340055466, -0.021298276260495186, -0.006965085864067078, 0.01661154255270958, 0.009577677585184574, 0.047799136489629745, -0.04848393797874451, 0.025230618193745613, -0.028317498043179512, 0.05951330065727234, -0.03575816750526428, 0.005719662643969059, 0.006293277256190777, -0.016469037160277367, -0 [...]
+{"id": 671, "title": "Harry Potter and the Philosopher's Stone", "embedding": [-0.004310962278395891, 0.024611175060272217, -0.021611034870147705, 0.00947610754519701, -0.04439878091216087, -0.03130454197525978, 0.043265823274850845, -0.025848109275102615, -0.012158924713730812, -0.055730875581502914, 0.013196416199207306, 0.06648571789264679, -0.060045044869184494, 0.04438595101237297, -0.016436345875263214, -0.005337127484381199, -0.022954892367124557, 0.04548772796988487, -0.047762885 [...]
+{"id": 1726, "title": "Iron Man", "embedding": [-0.03638796508312225, 0.07648811489343643, -0.0629308670759201, 0.03750128298997879, 0.014906095340847969, 0.004821385256946087, 0.0630541741847992, -0.03824242576956749, -0.09071019291877747, 0.02051815763115883, 0.0005745535017922521, -0.03508780524134636, 0.11199597269296646, 0.059365738183259964, 0.007531233597546816, -0.027557026594877243, 0.07445234805345535, -0.0014264058554545045, 0.025883212685585022, 0.022908223792910576, 0.030469 [...]
+{"id": 68718, "title": "Django Unchained", "embedding": [-0.07389315217733383, 0.019997606053948402, -0.006222703494131565, 0.02941598929464817, -0.03778772056102753, 0.07859908044338226, 0.018465496599674225, -0.06571436673402786, -0.08919283002614975, -0.04348417744040489, 0.05247121304273605, -0.00047271110815927386, 0.00858495943248272, 0.0017511919140815735, -0.08031865954399109, 0.025539642199873924, 0.006961648818105459, 0.09127316623926163, 0.05362964794039726, 0.0136927189305424 [...]
+{"id": 278, "title": "The Shawshank Redemption", "embedding": [-0.04443618655204773, -0.07722356915473938, -0.059657786041498184, -0.009497571736574173, -0.04846232384443283, 0.0319601371884346, 0.1143978089094162, -0.0212872177362442, 0.08214723318815231, -0.04711167886853218, 0.006906110793352127, 0.06397374719381332, 0.041254300624132156, 0.03943658992648125, 0.05858473479747772, 0.003077042754739523, -0.03670600801706314, 0.05146149918437004, -0.01147160679101944, -0.0378456115722656 [...]
+{"id": 299534, "title": "Avengers: Endgame", "embedding": [-0.03737539425492287, -0.03055523708462715, 0.023638814687728882, -0.029691588133573532, 0.06310853362083435, 0.01520749181509018, -0.029576405882835388, 0.0027748881839215755, 0.02907915972173214, 0.05614107474684715, 0.007290770765393972, 0.003568294458091259, -0.07752850651741028, 0.07123617827892303, -0.006204676348716021, 0.0017272710101678967, 0.015426495112478733, -0.08862896263599396, -0.05687204375863075, -0.047851324081 [...]
+{"id": 603, "title": "The Matrix", "embedding": [-0.0028160770889371634, 0.0077405087649822235, -0.14073790609836578, -0.04498438537120819, -0.06568080186843872, -0.008585098199546337, -0.02331066131591797, -0.05018167570233345, 0.044980574399232864, 0.047254882752895355, 0.025360750034451485, 0.051973629742860794, 0.07652897387742996, -0.017427371814846992, -0.026620246469974518, -0.0023783312644809484, -0.05359741672873497, -0.007229626644402742, 0.012381119653582573, -0.01494962163269 [...]
+{"id": 597, "title": "Titanic", "embedding": [-0.02997656539082527, -0.04448118433356285, 0.011988230049610138, 0.06126220524311066, 0.02540775202214718, 0.07408807426691055, -0.05767495185136795, 0.08616764843463898, -0.03312922641634941, -0.06734571605920792, -0.05992697924375534, 0.0699620172381401, 0.012641588225960732, 0.07762962579727173, -0.09053406119346619, 0.029982328414916992, -0.02045504003763199, -0.005096576176583767, -0.006513403262943029, 0.04342617467045784, 0.0728345960 [...]
+{"id": 475557, "title": "Joker", "embedding": [0.05657866597175598, -0.06718983501195908, -0.09452497959136963, 0.014727012254297733, 0.0327584445476532, 0.06311451643705368, 0.05717384070158005, 0.051921434700489044, -0.0682012215256691, -0.017731543630361557, 0.07081980258226395, -0.04679948836565018, -0.032459188252687454, 0.03579109534621239, -0.0226838868111372, 0.041373301297426224, 0.09712151437997818, -0.04409778490662575, 0.037877921015024185, 0.05994211137294769, -0.00244359346 [...]
+{"id": 120, "title": "The Lord of the Rings: The Fellowship of the Ring", "embedding": [-0.015172175131738186, 0.03340897336602211, -0.064532071352005, -0.004289834760129452, 0.03331151604652405, -0.0507594496011734, 0.009655947797000408, -0.014133522287011147, 0.08844179660081863, -0.03799491748213768, -0.06170031055808067, 0.009298590011894703, 0.04547198861837387, -0.05885116010904312, -0.003341031726449728, 0.03759533166885376, -0.01431785523891449, 0.03557853400707245, 0.02219508215 [...]
+{"id": 122, "title": "The Lord of the Rings: The Return of the King", "embedding": [-0.06278040260076523, -0.02287152409553528, -0.037835847586393356, 0.02927432581782341, 0.03881357982754707, -0.04867321252822876, -0.029846815392374992, 0.03544112667441368, 0.07924153655767441, -0.051799360662698746, -0.043216440826654434, 0.0007515315664932132, -0.02006942220032215, -0.06478375196456909, 0.006493435241281986, -0.027274195104837418, -0.033931914716959, 0.083253413438797, -0.046271052211 [...]
+{"id": 11324, "title": "Shutter Island", "embedding": [0.014572212472558022, 0.033345721662044525, -0.057812705636024475, 0.03790946677327156, 0.025981705635786057, -0.001832099398598075, 0.06766992807388306, -0.03254265338182449, -0.005087956786155701, 0.04225853085517883, 0.1072559654712677, 0.05758639797568321, 0.06453187018632889, 0.030017657205462456, 0.027222292497754097, -0.01919124834239483, -0.03213084116578102, 0.011723611503839493, 0.011155478656291962, -0.0013959603384137154, [...]
+{"id": 106646, "title": "The Wolf of Wall Street", "embedding": [-0.0043642171658575535, -0.01976202242076397, -0.059005048125982285, 0.011824638582766056, -0.04800193011760712, 0.030479393899440765, 0.07057233899831772, -0.04912615939974785, -0.033003658056259155, -0.03097473829984665, 0.022989559918642044, 0.0658072829246521, 0.040588926523923874, -0.015488912351429462, -0.0008235153509303927, 0.013236403465270996, -0.003328626975417137, 0.020818110555410385, 0.00806234497576952, 0.022 [...]
+{"id": 99861, "title": "Avengers: Age of Ultron", "embedding": [-0.043345022946596146, -0.030697857961058617, -0.02253352664411068, -0.000724776356946677, 0.05803821608424187, -0.012928077951073647, -0.033224571496248245, 0.034390274435281754, -0.07582584768533707, 0.04631615802645683, 0.016405336558818817, 0.014810804277658463, 0.0658903643488884, 0.026593217626214027, -0.03286760300397873, 0.017738142982125282, 0.01193707063794136, -0.06924112886190414, -0.03895069286227226, -0.0848738 [...]
+{"id": 271110, "title": "Captain America: Civil War", "embedding": [-0.042876947671175, -0.033842772245407104, -0.0640445351600647, -0.02800499089062214, 0.05780667066574097, 0.10616789013147354, -0.011382652446627617, -0.02254411391913891, -0.0709582194685936, 0.055404406040906906, -0.018867889419198036, -0.04490373656153679, 0.07199456542730331, 0.00047938100760802627, 0.0005046256701461971, -0.018389437347650528, 0.010315818712115288, -0.10651063174009323, -0.008391780778765678, -0.00 [...]
+{"id": 49026, "title": "The Dark Knight Rises", "embedding": [-0.0011769175762310624, 0.012997237034142017, -0.07743843644857407, -0.006485170219093561, 0.026061207056045532, 0.04479395970702171, -0.034792058169841766, 0.030706588178873062, 0.03143766522407532, 0.027188533917069435, 0.015578395687043667, -0.10269994288682938, 0.04358584061264992, -0.035344477742910385, 0.03693617507815361, -0.005568368826061487, 0.11452723294496536, 0.05397168546915054, 0.0517735593020916, -0.03011597320 [...]
+{"id": 68721, "title": "Iron Man 3", "embedding": [-0.021945105865597725, -0.051752571016550064, -0.05823031812906265, 0.03665662184357643, 0.015683691948652267, 0.07944850623607635, 0.05390237644314766, -0.11783825606107712, -0.013874678872525692, -0.02164524979889393, 0.010411991737782955, -0.005770211108028889, 0.05689450725913048, 0.04253176599740982, 0.050781819969415665, 0.0042707533575594425, -0.01208619773387909, -0.022750459611415863, 0.024059301242232323, -0.003976788837462664, [...]
+{"id": 284054, "title": "Black Panther", "embedding": [-0.08017772436141968, 0.01438127737492323, -0.0952938050031662, 0.043510038405656815, -0.05720148980617523, -0.0070917983539402485, 0.028058458119630814, -0.014253275468945503, 0.011612782254815102, 0.037296295166015625, -0.014524593949317932, -0.16979049146175385, 0.024841682985424995, 0.019935227930545807, -0.017742453143000603, -0.005695155821740627, 0.052394308149814606, -0.08627642691135406, -0.05288152024149895, -0.057284172624 [...]
+{"id": 284052, "title": "Doctor Strange", "embedding": [-0.05849218741059303, 0.025617964565753937, -0.05030293017625809, -0.002904471242800355, -0.0778435543179512, -0.014157719910144806, 0.07371204346418381, 0.06724614650011063, -0.045766137540340424, 0.019304979592561722, 0.014334427192807198, -0.007986166514456272, -0.029123544692993164, 0.10454495251178741, -0.09361440688371658, -0.0423947349190712, 0.005775875877588987, -0.017865093424916267, -0.03265680745244026, 0.032293945550918 [...]
+{"id": 76341, "title": "Mad Max: Fury Road", "embedding": [-0.04644708335399628, -0.06246284022927284, -0.022679470479488373, 0.008011506870388985, -0.022292986512184143, 0.04925848916172981, -0.04988103359937668, 0.002302692737430334, -0.03442109003663063, 0.009755712933838367, 0.00912798848003149, 0.12169525027275085, 0.038560304790735245, 0.04383234679698944, -0.011536841280758381, 0.06396465003490448, 0.02819627895951271, -0.056348446756601334, -0.0051871598698198795, -0.049856498837 [...]
+{"id": 16869, "title": "Inglourious Basterds", "embedding": [0.06979531794786453, -0.01753338798880577, -0.02742753177881241, -0.04152316600084305, 0.03993147239089012, 0.04366850480437279, 0.04840899631381035, 0.022851036861538887, 0.04472985118627548, 0.03299277648329735, 0.04384380951523781, -0.06323574483394623, 0.07728476822376251, 0.024456044659018517, -0.04100885987281799, -0.006984207313507795, 0.02099069580435753, 0.05270783230662346, -0.07381899654865265, 0.018152475357055664, [...]
+{"id": 70160, "title": "The Hunger Games", "embedding": [0.04818129166960716, 0.010049360804259777, 0.039135754108428955, -0.10426224023103714, 0.01063316036015749, 0.07699301838874817, -0.02260720171034336, -0.027880340814590454, 0.003936226479709148, 0.08977042138576508, -0.09159564226865768, -0.0028479143511503935, -0.0012469145003706217, -0.05186965689063072, -0.021058397367596626, -0.06541262567043304, 0.07154054194688797, -0.018972119316458702, 0.004346252419054508, -0.009703128598 [...]
+{"id": 315635, "title": "Spider-Man: Homecoming", "embedding": [0.006184868514537811, -0.056244101375341415, -0.020211827009916306, -0.053636226803064346, 0.04198543727397919, 0.033429551869630814, -0.010705611668527126, -0.06915342062711716, -0.09016621857881546, 0.009337135590612888, 0.03015141561627388, 0.012065653689205647, -0.009160629473626614, 0.024919012561440468, 0.07018812745809555, 0.055564071983098984, 0.04058666154742241, -0.033312149345874786, -0.004903922323137522, -0.0594 [...]
+{"id": 672, "title": "Harry Potter and the Chamber of Secrets", "embedding": [-0.014999647624790668, 0.05010046809911728, -0.050566017627716064, 0.045800525695085526, -0.0027655907906591892, 0.019157104194164276, -0.023136360570788383, -0.02981649525463581, -0.027701156213879585, -0.07608672976493835, -0.005869999527931213, 0.03513006120920181, -0.06772401928901672, 0.019594166427850723, 0.016317196190357208, -0.03158266097307205, -0.04032051935791969, 0.0563071183860302, -0.023938544094 [...]
+{"id": 283995, "title": "Guardians of the Galaxy Vol. 2", "embedding": [-0.09676863253116608, -0.03540623188018799, 0.020217660814523697, -0.021542133763432503, -0.043292466551065445, -0.01197686605155468, 0.00032528338488191366, 0.04829016700387001, 0.0365675650537014, 0.01756589114665985, 0.07167837768793106, -0.05993199348449707, -0.02205931954085827, 0.01320668775588274, 0.03876717388629913, -0.016803843900561333, 0.008487294428050518, -0.05599866807460785, -0.005918386857956648, -0. [...]
+{"id": 121, "title": "The Lord of the Rings: The Two Towers", "embedding": [-0.0643252283334732, 0.0005818460485897958, -0.03308640047907829, -0.009528656490147114, 0.028881002217531204, -0.0680948793888092, -0.03076072223484516, 0.010040885768830776, 0.06249416619539261, -0.009083712473511696, -0.04190210998058319, -0.019086064770817757, -0.0011160625144839287, -0.04819731041789055, 0.018464630469679832, -0.004996035248041153, -0.047029782086610794, 0.05567512288689613, 0.00448300596326 [...]
+{"id": 1771, "title": "Captain America: The First Avenger", "embedding": [-0.03254826366901398, 0.001690419390797615, -0.04998333379626274, -0.014407879672944546, 0.004675514530390501, 0.041452281177043915, 0.07045108824968338, -0.02538255788385868, -0.03083575889468193, 0.070002481341362, 6.806340388720855e-05, 0.029680337756872177, 0.054063476622104645, 0.1029939129948616, 0.005540268495678902, -0.010217014700174332, 0.009756537154316902, -0.047057777643203735, -0.013618781231343746, 0 [...]
+{"id": 297761, "title": "Suicide Squad", "embedding": [-0.05838587135076523, -0.01577303186058998, -0.12066783756017685, 0.006442574318498373, 0.005804197862744331, 0.015117837116122246, 0.020965740084648132, 0.025133166462183, 0.0324387326836586, 0.09940779209136963, 0.07950036972761154, -0.005544023122638464, 0.02488897182047367, 0.00990734901279211, 0.037693727761507034, -0.038015786558389664, 0.1188342347741127, -0.009676840156316757, -0.07402331382036209, 0.037277013063430786, 0.032 [...]
+{"id": 673, "title": "Harry Potter and the Prisoner of Azkaban", "embedding": [-0.03806249052286148, 0.00267825648188591, -0.076321542263031, 0.004444214515388012, -0.053443167358636856, 0.014457318000495434, 0.06552240997552872, -0.06343034654855728, 0.01638343743979931, -0.07235319912433624, 0.03580006584525108, 0.029433611780405045, -0.0005042359116487205, 0.006601120810955763, -0.007878092117607594, -0.016819404438138008, -0.009203094989061356, 0.044163014739751816, -0.03436123952269 [...]
+{"id": 10195, "title": "Thor", "embedding": [0.03450171276926994, -0.004695482086390257, -0.03480911999940872, 0.018204346299171448, 0.05333780124783516, 0.015362604521214962, -0.033021558076143265, 0.012121404521167278, 0.05844030901789665, 0.023298602551221848, 0.01954863965511322, -0.022562144324183464, 0.06611759960651398, 0.06057608127593994, 0.035564202815294266, 0.01495111733675003, 0.007004535291343927, -0.11614693701267242, 0.006200742442160845, 0.008689259178936481, 0.061993420 [...]
+{"id": 10138, "title": "Iron Man 2", "embedding": [-0.04551311209797859, -0.03125835210084915, -0.028038576245307922, 0.040265120565891266, 0.06184738874435425, -0.012453166767954826, 0.06512451171875, 0.03753463178873062, -0.07918945699930191, -0.033380039036273956, 0.04113905504345894, 0.01898784190416336, 0.059713199734687805, 0.036368146538734436, 0.019276244565844536, -0.04814422130584717, 0.01523805782198906, -0.06964264065027237, -0.019804060459136963, -0.018641728907823563, 0.083 [...]
+{"id": 272, "title": "Batman Begins", "embedding": [0.01433988194912672, 0.0200861394405365, -0.06374096870422363, 0.05025302991271019, 0.03766079619526863, 0.06995461881160736, 0.05926275998353958, 0.011279495432972908, -0.013028833083808422, 0.015021700412034988, -0.001705941278487444, -0.03806041553616524, 0.03153965622186661, 0.00029668057686649263, -0.016269871965050697, 0.03765707463026047, 0.06844647228717804, 0.005289161577820778, 0.030697552487254143, 0.0686345100402832, 0.01938 [...]
+{"id": 284053, "title": "Thor: Ragnarok", "embedding": [0.04029818996787071, 0.025158368051052094, -0.08037068694829941, 0.043259091675281525, 0.04605811834335327, 0.06229126825928688, -0.029750099405646324, 0.009168850257992744, 0.04624176025390625, 0.013056992553174496, 0.01086234301328659, -0.03860953822731972, 0.013696896843612194, 0.05666402727365494, 0.06617400795221329, 0.005941550247371197, -0.0038494248874485493, -0.06681407243013382, 0.0018162510823458433, -0.020580869168043137 [...]
+{"id": 150540, "title": "Inside Out", "embedding": [0.07317458838224411, -0.08159848302602768, 0.0304412841796875, 0.03471730649471283, 0.09993824362754822, 0.00634497357532382, 0.04425068572163582, -0.0045237671583890915, 0.054934337735176086, -0.05812288075685501, 0.06428617238998413, 0.033158931881189346, 0.02053409069776535, -0.05456886440515518, 0.03882719203829765, 0.03893648087978363, -0.029790882021188736, -0.028492458164691925, 0.015683496370911598, -0.026187662035226822, 0.0006 [...]
+{"id": 135397, "title": "Jurassic World", "embedding": [0.02296176366508007, -0.04424763098359108, 0.021321214735507965, -0.049901556223630905, 0.05620419979095459, -0.017927169799804688, -0.05325569584965706, -0.025131452828645706, 0.012875654734671116, 0.004293501842767, -0.026011820882558823, -0.03139873966574669, -0.0211848933249712, 0.08655509352684021, -0.009496294893324375, 0.004229498561471701, -0.009334053844213486, 0.02096673659980297, 0.11660638451576233, -0.060625266283750534 [...]
+{"id": 807, "title": "Se7en", "embedding": [-0.018101120367646217, -0.03138101473450661, -0.10344145447015762, 0.0007243403815664351, 0.0622238963842392, 0.05124741420149803, 0.04063230752944946, 0.004309635143727064, 0.03991357982158661, -0.005707306321710348, 0.04460717737674713, -0.0423458106815815, 0.009900463744997978, 0.009234543889760971, -0.04507831484079361, -0.06332919001579285, 0.003897372866049409, 0.016248919069767, 0.021264920011162758, 0.04074608162045479, -0.0278563369065 [...]
+{"id": 674, "title": "Harry Potter and the Goblet of Fire", "embedding": [0.03381262347102165, 0.032300446182489395, -0.0650620386004448, -0.015910839661955833, -0.04495080187916756, 0.011171354912221432, 0.060283321887254715, -0.006100211758166552, -0.0034317041281610727, -0.08415137976408005, -0.003579711774364114, 0.00034754330408759415, -0.06078067049384117, 0.050428107380867004, -0.0009544481290504336, -0.03683627396821976, -0.07196120917797089, 0.0563570074737072, 0.010831305757164 [...]
+{"id": 22, "title": "Pirates of the Caribbean: The Curse of the Black Pearl", "embedding": [-0.05079403147101402, 0.05741175636649132, 0.025038601830601692, 0.026198716834187508, -0.03682859241962433, 0.0779748186469078, 0.033852145075798035, -0.017603429034352303, 0.05219816416501999, -0.030039258301258087, -0.016578596085309982, 0.050445154309272766, 0.03194941580295563, -0.02678213082253933, -0.0902436301112175, 0.04433367773890495, 0.06929787248373032, 0.022117074579000473, 0.0552487 [...]
+{"id": 12445, "title": "Harry Potter and the Deathly Hallows: Part 2", "embedding": [0.009017950855195522, 0.044816724956035614, -0.0552731454372406, -0.029764775186777115, -0.06448084861040115, -0.012011530809104443, -0.032952528446912766, -0.03374378755688667, -0.054699789732694626, -0.04687419906258583, 0.0017401662189513445, 0.01954731158912182, -0.05793130025267601, 0.013653114438056946, -0.03779446706175804, -0.009558659978210926, -0.05370376631617546, 0.10426963120698929, -0.05688 [...]
+{"id": 11, "title": "Star Wars", "embedding": [-0.058117035776376724, -0.08882070332765579, 0.050920892506837845, -0.08037518709897995, 0.002265981864184141, 0.0538981594145298, 0.030179738998413086, 0.0258457213640213, -4.715936302090995e-05, 0.05668722093105316, 0.09014308452606201, -0.02473401091992855, -0.01691576838493347, -0.03771836683154106, 0.0310821570456028, -0.027579838410019875, 0.07687146216630936, -0.0013295498210936785, -0.024648215621709824, 0.0011921115219593048, -0.003 [...]
+{"id": 297762, "title": "Wonder Woman", "embedding": [0.020905500277876854, -0.06931184977293015, -0.060087427496910095, 0.03813605755567551, 0.0037176141049712896, 0.012173243798315525, 0.031112559139728546, -0.026622256264090538, -0.0012483835453167558, 0.0674724206328392, 0.0431634783744812, 0.013846995308995247, 0.00399596244096756, -0.05435517057776451, -0.009197928011417389, 0.06724769622087479, 0.02966764010488987, -0.09587094187736511, -0.06495712697505951, 0.028028683736920357, [...]
+{"id": 14160, "title": "Up", "embedding": [0.058823224157094955, 0.07000577449798584, 0.002982702571898699, 0.049274664372205734, 0.03832549974322319, 0.06482917815446854, 0.013422511518001556, 0.029734183102846146, -0.05579390749335289, 0.018850643187761307, 0.028189921751618385, -0.017601123079657555, 0.020231110975146294, 0.09089585393667221, -0.006086149252951145, 0.054593704640865326, -0.11813984811306, -0.020075444132089615, 0.019318081438541412, -0.030417922884225845, -0.027543816 [...]
+{"id": 102899, "title": "Ant-Man", "embedding": [-0.05110345035791397, 0.09822431206703186, -0.029226018115878105, -0.0236904788762331, 0.04182213172316551, 0.06946063041687012, 0.05918211117386818, 0.07666667550802231, -0.09760385751724243, 0.11897042393684387, 0.02424883283674717, 0.012535805813968182, -0.03809845447540283, 0.08313023298978806, 0.05960449203848839, -0.013396711088716984, -0.02045816369354725, 0.030728809535503387, 0.016246242448687553, -0.06407105177640915, 0.044353649 [...]
+{"id": 238, "title": "The Godfather", "embedding": [-0.09126710891723633, 0.029030196368694305, -0.04827561601996422, 0.02310327999293804, -0.021734442561864853, 0.017657995223999023, -0.05277174711227417, 0.06226642057299614, 0.04039600491523743, -0.028297746554017067, 0.09687675535678864, -0.0010668081231415272, 0.008083607070147991, 0.05273638293147087, -0.028561457991600037, -0.05899593234062195, 0.01836533099412918, 0.07782316207885742, -0.034777212888002396, 0.02831134758889675, -0 [...]
+{"id": 286217, "title": "The Martian", "embedding": [0.026125192642211914, -0.012241343967616558, -0.017232459038496017, 0.006539338268339634, 0.05140437185764313, -0.061466291546821594, 0.027422906830906868, 0.01843581348657608, -0.014102336950600147, 0.0707668662071228, 0.059430789202451706, -0.0240425243973732, 0.0017062033293768764, 0.017987392842769623, -0.020955707877874374, -0.009186207316815853, -0.02703995257616043, 0.011012361384928226, -0.023543287068605423, -0.048180270940065 [...]
+{"id": 105, "title": "Back to the Future", "embedding": [-0.05549710616469383, 0.038878727704286575, 0.0374625138938427, -0.024808701127767563, 0.017521819099783897, 0.06447122246026993, 0.04199356958270073, -0.013753412291407585, 0.007373750675469637, -0.07676244527101517, 0.09301812201738358, 0.05566616356372833, 0.012679995968937874, -0.021785488352179527, 0.0019015753641724586, 0.05480116605758667, -0.0016222402919083834, -0.00035364972427487373, -0.017307862639427185, -0.02924104221 [...]
+{"id": 140607, "title": "Star Wars: The Force Awakens", "embedding": [-0.050199009478092194, -0.07228825986385345, 0.0627390667796135, -0.025148963555693626, -0.035429779440164566, 0.06784360855817795, 0.021374328061938286, 0.009614815004169941, 0.006270411890000105, 0.06997546553611755, 0.0661277025938034, -0.04798780381679535, 0.012592149898409843, 0.022093890234827995, 0.014208323322236538, -0.05470571294426918, 0.07249686121940613, -0.022169096395373344, 0.09160052239894867, -0.02012 [...]
+{"id": 634649, "title": "Spider-Man: No Way Home", "embedding": [-0.018705172464251518, -0.11091990023851395, -0.02366623841226101, -0.010420918464660645, 0.014173404313623905, 0.032668374478816986, -0.02142530307173729, -0.04205275699496269, -0.10038405656814575, -0.03735936060547829, -0.012290551327168941, 0.010843620635569096, -0.09062232077121735, 0.05170830711722374, 0.04362392798066139, 0.029163828119635582, 0.04360692575573921, 0.006402702536433935, 0.03898996487259865, 0.01221219 [...]
+{"id": 675, "title": "Harry Potter and the Order of the Phoenix", "embedding": [0.04593940079212189, 0.035836782306432724, 0.005796133540570736, 0.03515617176890373, -0.047864433377981186, 0.00629555294290185, -0.017578506842255592, -0.0903693437576294, 0.015687651932239532, -0.02120983973145485, -0.011341736651957035, 0.08215470612049103, -0.055219318717718124, -0.019937753677368164, -0.0372769832611084, -0.04311634600162506, -0.03860625997185707, 0.028784358873963356, -0.04430797323584 [...]
+{"id": 767, "title": "Harry Potter and the Half-Blood Prince", "embedding": [-0.013098429888486862, -0.011644849553704262, 0.03090396709740162, -0.029340682551264763, -0.03562231361865997, 0.06989579647779465, -0.040611591190099716, 0.02578333206474781, -0.005297970958054066, -0.05933833122253418, -0.04850978031754494, 0.02858242206275463, -0.038819022476673126, -0.03542131930589676, -0.03772462531924248, -0.016364810988307, -0.029361462220549583, 0.0346999317407608, -0.07051033526659012 [...]
+{"id": 263115, "title": "Logan", "embedding": [-0.05309786647558212, -0.04322520270943642, -0.0004943471285514534, -0.0070024519227445126, 0.07189923524856567, 0.012520969845354557, 0.04592137038707733, 0.008731616660952568, 0.044503990560770035, -0.01061149500310421, 0.06773307919502258, 0.008678262121975422, 0.0052770404145121574, 0.06641780585050583, -0.03800927847623825, 0.06146298721432686, 0.04425743222236633, -0.07391910254955292, -0.07029985636472702, 0.015035392716526985, 0.0275 [...]
+{"id": 12, "title": "Finding Nemo", "embedding": [-0.04001476988196373, 0.03410449996590614, 0.008842245675623417, 0.010853670537471771, -0.017306871712207794, -0.04911038279533386, 0.05889072269201279, -0.018874604254961014, -0.05096428096294403, -0.03315533325076103, 0.10556567460298538, -0.145391047000885, -0.0351373516023159, 0.12934136390686035, 0.013211474753916264, 0.026013491675257683, 0.07003407925367355, -0.003834909526631236, -0.01833522506058216, -0.014171093702316284, -0.014 [...]
+{"id": 346364, "title": "It", "embedding": [0.07859914004802704, 0.04599446430802345, -0.03724341467022896, 0.052365485578775406, -0.01987523026764393, 0.048829998821020126, 0.05335255712270737, 0.005758315324783325, -0.04249350726604462, 0.013904971070587635, 0.0571107380092144, -0.06555519253015518, 0.09088549017906189, -0.015390101820230484, -0.06629519909620285, 0.009181139059364796, 0.04084306210279465, -0.03364633396267891, 0.03233650326728821, -0.03445315733551979, -0.045703612267 [...]
+{"id": 245891, "title": "John Wick", "embedding": [-0.054457347840070724, -0.044149249792099, -0.07531348615884781, 0.023396480828523636, -0.0056140529923141, 0.007606424391269684, 0.03282170370221138, -0.007599454373121262, -0.01824430376291275, 0.04670891910791397, 0.09432099759578705, 0.03170289099216461, -0.0007847442175261676, -0.03558895364403725, -0.01221549604088068, -0.02280263602733612, -0.04095758870244026, 0.0557907298207283, 0.06754103302955627, -0.11623256653547287, -0.1344 [...]
+{"id": 259316, "title": "Fantastic Beasts and Where to Find Them", "embedding": [0.02568931132555008, -0.052565526217222214, -0.011051997542381287, 0.021451612934470177, 0.03776365891098976, 0.02838297188282013, 0.04930759221315384, -0.0007882473873905838, -0.07042387127876282, 0.0013537703780457377, -0.07080002129077911, 0.08276048302650452, -0.04698754474520683, 0.11010473966598511, -0.04448317736387253, -0.008598119020462036, 0.06856967508792877, -0.07559265941381454, -0.0517634600400 [...]
+{"id": 12444, "title": "Harry Potter and the Deathly Hallows: Part 1", "embedding": [0.011458467692136765, -0.019496891647577286, -0.04214363545179367, -0.005693697836250067, -0.03258978947997093, 0.009652134962379932, -0.022033218294382095, -0.048435717821121216, -0.07883691787719727, -0.07039013504981995, -0.007316902745515108, 0.009548951871693134, -0.012389849871397018, 0.013294161297380924, -0.043288301676511765, -0.0340726263821125, -0.09139873087406158, -0.0026692848186939955, -0. [...]
+{"id": 354912, "title": "Coco", "embedding": [-0.05874214693903923, 0.04673571512103081, 0.014797688461840153, -0.020434970036149025, 0.0016735619865357876, 0.014462132938206196, 0.014911499805748463, -0.008068286813795567, 0.016525520011782646, -0.05517715960741043, 0.09251219779253006, -0.05607956647872925, -0.021548718214035034, -0.014290105551481247, 0.03563665226101875, 0.0032199700362980366, -0.058109551668167114, 0.05423077940940857, 0.021854804828763008, -0.0012470717774704099, - [...]
+{"id": 100402, "title": "Captain America: The Winter Soldier", "embedding": [-0.05140504986047745, -0.007658518385142088, -0.046987906098365784, 0.02135455049574375, 0.018752770498394966, 0.09726662933826447, -0.020275067538022995, -0.033396001905202866, -0.02523801289498806, 0.08113531023263931, -0.011705569922924042, -0.00479517038911581, 0.047842446714639664, 0.0791793093085289, -0.0245791207998991, -0.036454107612371445, 0.026632336899638176, -0.09425895661115646, -0.0166306439787149 [...]
+{"id": 557, "title": "Spider-Man", "embedding": [-0.027792159467935562, -0.041730962693691254, -0.028021788224577904, 0.00791282206773758, -0.039661113172769547, -0.049346547573804855, 0.045964308083057404, -0.028487499803304672, -0.11810295283794403, -0.006798533257097006, 0.009405533783137798, 0.015601561404764652, -0.08560992032289505, 0.08587516099214554, 0.026406846940517426, 0.09459100663661957, 0.10576439648866653, 0.001137149753049016, -0.007989644072949886, -0.03525402396917343, [...]
+{"id": 10681, "title": "WALL\u00b7E", "embedding": [0.0012638706248253584, 0.055271346122026443, 0.013401464559137821, -0.019032634794712067, 0.04348989576101303, -0.051223963499069214, -0.031726691871881485, 0.048479706048965454, -0.014234731905162334, -0.023176897317171097, 0.004269957542419434, -0.023986803367733955, 0.0004280318971723318, -0.01271870732307434, 0.01416214182972908, -0.047857578843832016, -0.016353126615285873, -0.045709218829870224, -0.013486107811331749, -0.000225078 [...]
+{"id": 210577, "title": "Gone Girl", "embedding": [-0.012413338758051395, -0.030242351815104485, -0.004260395187884569, 0.1151597872376442, 0.029669534415006638, 0.04870489612221718, 0.09979654103517532, -0.03842330724000931, 0.04332946613430977, -0.0030888444744050503, 0.11556887626647949, 0.07081630825996399, 0.04519985616207123, -0.05380101874470711, 0.03347242623567581, 0.020619550719857216, 0.06451675295829773, -0.05433443561196327, -0.02095685712993145, -0.02323629893362522, -0.087 [...]
+{"id": 49051, "title": "The Hobbit: An Unexpected Journey", "embedding": [-0.028684116899967194, 0.04316803440451622, 0.0015892833471298218, -0.005603437311947346, 0.008067688904702663, -0.047503646463155746, 0.04082544147968292, -0.00017958870739676058, -0.00168235891032964, -0.03007255122065544, -0.05449039861559868, -0.06453240662813187, -0.020916422829031944, -0.036869946867227554, 0.007518123835325241, 0.06271586567163467, -0.005468927323818207, 0.06873864680528641, 0.05518141761422 [...]
+{"id": 585, "title": "Monsters, Inc.", "embedding": [-0.07679598778486252, -0.0415806919336319, -0.03388657420873642, 0.015938730910420418, -0.01833062805235386, -0.018714696168899536, 0.07844556123018265, -0.03745182976126671, -0.013800705783069134, -0.1393078863620758, 0.014576243236660957, -0.06402015686035156, 0.06293763965368271, 0.030055588111281395, -0.07673340290784836, 0.048651229590177536, 0.07460758090019226, 0.0770493820309639, -0.010783534497022629, -0.07506740093231201, 0.0 [...]
+{"id": 862, "title": "Toy Story", "embedding": [0.026368461549282074, -0.052618686109781265, 0.10287690907716751, -0.03195498511195183, 0.005204619839787483, 0.029093734920024872, 0.08998388051986694, 0.013737736269831657, -0.036896850913763046, 0.04709690064191818, 0.05979277938604355, 0.04271151125431061, 0.033457983285188675, 0.04771064221858978, 0.12374305725097656, 0.061044350266456604, -0.008474180474877357, -0.0386018306016922, -0.018921323120594025, -0.04770775884389877, 0.040266 [...]
+{"id": 281957, "title": "The Revenant", "embedding": [0.007314587477594614, 0.01070375181734562, -0.02723795548081398, 0.05895683914422989, 0.04032042622566223, -0.004420428536832333, 0.00975798536092043, -0.047139257192611694, -0.06656769663095474, 0.10456088930368423, -0.01995091885328293, 0.05792193487286568, 0.014033673331141472, -0.0179925374686718, -0.05529487133026123, 0.024182597175240517, -0.014400326646864414, 0.07102932035923004, 0.10708839446306229, 1.9015002180822194e-05, -0 [...]
+{"id": 209112, "title": "Batman v Superman: Dawn of Justice", "embedding": [0.01429351232945919, 0.038186706602573395, -0.09056881070137024, -0.007160362787544727, 0.048628441989421844, 0.06437921524047852, -0.023962071165442467, 0.04767466336488724, -0.004091229755431414, 0.057960133999586105, 0.005716864485293627, 0.029190322384238243, -0.015620812773704529, 0.030653007328510284, 0.0011786888353526592, -0.008586393669247627, 0.12717989087104797, 0.010125520639121532, -0.015929155051708 [...]
+{"id": 8587, "title": "The Lion King", "embedding": [-0.042611606419086456, 0.09020733088254929, -0.049823395907878876, 0.06532750278711319, 0.0038679135031998158, 0.020214278250932693, 0.016943061724305153, -0.015806002542376518, 0.009359177201986313, -0.0022281035780906677, -0.00272317579947412, -0.05157922953367233, -0.01790592260658741, -0.03495318442583084, -0.018392615020275116, 0.003105579409748316, -0.03099551796913147, 0.053177449852228165, -0.06627470999956131, 0.01908602006733 [...]
+{"id": 98, "title": "Gladiator", "embedding": [-0.024041233584284782, 0.019591238349676132, -0.04701283946633339, -0.034109245985746384, -0.037647150456905365, 0.013913562521338463, 0.03787076845765114, 0.012182198464870453, -0.04565729573369026, 0.01832296885550022, 0.03658098727464676, -0.04304948076605797, 0.03816790133714676, 0.023571787402033806, -0.021777858957648277, -0.0650225430727005, 0.03772304952144623, 0.09945109486579895, 0.024172203615307808, 0.010962300933897495, 0.085593 [...]
+{"id": 37165, "title": "The Truman Show", "embedding": [0.012169480323791504, -0.005584804341197014, -0.02818993851542473, -0.027734300121665, -0.009464467875659466, 0.03853045031428337, 0.05328330025076866, -0.035484421998262405, -0.021185733377933502, -0.05962765961885452, 0.015493339858949184, 0.009457156993448734, 0.043253093957901, -0.04970606788992882, -0.06409657746553421, -0.02938443049788475, 0.02895074151456356, 0.011057434603571892, 0.00729631120339036, 0.005353315733373165, 0 [...]
+{"id": 9806, "title": "The Incredibles", "embedding": [-0.10560645163059235, -0.017792750149965286, -0.07191100716590881, -0.021299561485648155, 0.009106580168008804, -0.018967213109135628, 0.03198812156915665, 0.025541476905345917, -0.03910789638757706, -0.004031355958431959, 0.07816268503665924, 0.022615868598222733, 0.013398822396993637, 0.02183610014617443, 0.05429774522781372, -0.015039294958114624, 0.029924117028713226, -0.047834258526563644, -0.003479755949229002, -0.0294212382286 [...]
+{"id": 329865, "title": "Arrival", "embedding": [0.046453461050987244, -0.023131482303142548, 0.04432970657944679, 0.09898009896278381, 0.013241558335721493, -0.01060732826590538, 0.08829981088638306, -0.00748261995613575, 0.03643760457634926, 0.0026960638351738453, 0.014398854225873947, -0.05443349853157997, 0.00902703870087862, -0.006887700874358416, 0.021014440804719925, 0.04113639518618584, 0.0048333327285945415, -0.07276608049869537, -0.004301534965634346, -0.059946298599243164, -0. [...]
+{"id": 496243, "title": "Parasite", "embedding": [-0.009070903062820435, 0.002473260276019573, 0.005807062610983849, 0.02178065851330757, 0.036210402846336365, -0.00393364392220974, 0.08720894902944565, -0.027741633355617523, -0.02387961372733116, -0.021388567984104156, 0.10804615169763565, -0.09006252884864807, 0.035512518137693405, -0.011959598399698734, -0.01525227352976799, -0.01581142097711563, 0.011411534622311592, 0.028697259724140167, -0.02247272990643978, -0.051299743354320526, [...]
+{"id": 76338, "title": "Thor: The Dark World", "embedding": [0.052301883697509766, -0.01181087363511324, -0.026442160829901695, 0.04445544630289078, 0.054176561534404755, 0.01803107187151909, -0.08397219330072403, 0.02593930996954441, 0.06652085483074188, 0.023673847317695618, -0.0032015880569815636, -0.07442131638526917, 0.04249005392193794, 0.0644516870379448, 0.07492498308420181, 0.009228861890733242, 0.02683667466044426, -0.09018068015575409, 0.007939786650240421, 0.00039893868961371 [...]
+{"id": 1930, "title": "The Amazing Spider-Man", "embedding": [-0.11374334245920181, -0.049004584550857544, -0.018875760957598686, -0.019773555919528008, 0.039708781987428665, -0.0346502847969532, 0.0663532093167305, 0.052873119711875916, -0.07867374271154404, 0.008817912079393864, 0.023425474762916565, 0.049307577311992645, -0.06731827557086945, 0.048616599291563034, 0.029239267110824585, 0.03539915755391121, 0.055573511868715286, 0.019912952557206154, -0.06460762768983841, -0.0045960992 [...]
+{"id": 381288, "title": "Split", "embedding": [-0.020874792709946632, -0.016871638596057892, 0.04736033454537392, -0.04083715379238129, 0.032567474991083145, -0.015680652111768723, 0.029374392703175545, 0.03596406802535057, 0.11026362329721451, 0.01431228220462799, 0.07520441710948944, -0.03605025261640549, 0.007995213381946087, -0.03570408374071121, 0.056762583553791046, 0.009053153917193413, -0.008817161433398724, 0.012747911736369133, -0.10116066783666611, -0.02237868122756481, -0.010 [...]
+{"id": 383498, "title": "Deadpool 2", "embedding": [-0.0685655027627945, -0.04899615794420242, -0.0782598927617073, -0.045225948095321655, 0.021190766245126724, 0.03097030706703663, 0.036507923156023026, 0.011458072811365128, -0.026070522144436836, 0.020530439913272858, 0.07351677119731903, -0.025313755497336388, 0.04200528934597969, 0.10662844032049179, -0.0007262912695296109, 0.010723267681896687, 0.10132196545600891, 0.011730861850082874, 0.002926276996731758, -0.0030530572403222322, [...]
+{"id": 694, "title": "The Shining", "embedding": [-0.03342973440885544, -0.09817937761545181, -0.01101651694625616, 0.05874437838792801, -0.014939568936824799, 0.020230012014508247, 0.0417526513338089, -0.003060161368921399, 0.043032240122556686, -0.047325678169727325, 0.06071232631802559, 0.016568565741181374, 0.05845436826348305, 0.04691417142748833, 0.0004743229947052896, 0.04748334363102913, -0.0030540006700903177, 0.009799693711102009, 0.04230186343193054, 0.02408985048532486, 0.068 [...]
+{"id": 24, "title": "Kill Bill: Vol. 1", "embedding": [-0.016811173409223557, 0.01066307257860899, -0.015960395336151123, -0.01598634384572506, 0.00026303555932827294, 0.01822868175804615, 0.04157162457704544, -0.02535676769912243, 0.00508655421435833, 0.02080320194363594, 0.06483714282512665, -0.03079509176313877, 0.007967549376189709, -0.02816014736890793, 0.048693135380744934, -0.006400151178240776, 0.09556758403778076, 0.042037587612867355, -0.028019724413752556, 0.041613347828388214 [...]
+{"id": 101299, "title": "The Hunger Games: Catching Fire", "embedding": [0.016784679144620895, 0.02035447023808956, 0.04537967965006828, -0.0786382406949997, -0.04940226301550865, 0.07626224309206009, 0.023837966844439507, -0.022000908851623535, -0.0006792951026000082, 0.09559538960456848, -0.09275048971176147, 0.01919195055961609, 0.015586047433316708, -0.054850440472364426, 0.0006503427284769714, -0.09309649467468262, 0.020472567528486252, 0.005623451434075832, 0.01988174021244049, -0. [...]
+{"id": 77338, "title": "The Intouchables", "embedding": [-0.045633576810359955, 0.03261955454945564, -0.01849350519478321, 0.04755295440554619, -0.0564618818461895, -0.0023295481223613024, 0.053866446018218994, 0.03097657486796379, -0.06331828981637955, 0.024508781731128693, 0.04761336371302605, 0.03295626491308212, 0.009555208496749401, 0.05225319042801857, -0.03173273056745529, -0.04149746894836426, 0.026351384818553925, -0.045032799243927, -0.028721889480948448, 0.08927350491285324, 0 [...]
+{"id": 419430, "title": "Get Out", "embedding": [-0.026014242321252823, 0.004203171469271183, -0.003645543474704027, 0.0347798615694046, 0.08860265463590622, 0.03118642047047615, 0.06765234470367432, -0.018935421481728554, 0.04387231916189194, -0.06740224361419678, -0.011777084320783615, -0.018754418939352036, -0.04726358875632286, -0.04632189869880676, 0.040372349321842194, 0.029221173375844955, -0.0012579400790855289, -0.006710530258715153, -0.07922791689634323, 0.009531560353934765, - [...]
+{"id": 205596, "title": "The Imitation Game", "embedding": [-0.07890724390745163, -0.0025403364561498165, -0.08653900027275085, -0.11750341206789017, -0.04600244760513306, 0.04826211556792259, 0.08409564197063446, -0.03290579468011856, -0.00857294350862503, 0.0002590588410384953, 0.0011208541691303253, 0.034542471170425415, 0.08572745323181152, 0.024016575887799263, -0.027384763583540916, 0.037074800580739975, -0.020946070551872253, -0.07059873640537262, 0.015253954567015171, -0.15816125 [...]
+{"id": 424694, "title": "Bohemian Rhapsody", "embedding": [0.08544781059026718, -0.0610961988568306, -0.016456982120871544, 0.010058973915874958, 0.045356445014476776, 0.07851432263851166, 0.0359586700797081, 0.028729083016514778, -0.0009055360569618642, -0.018302982673048973, -0.04762532189488411, -0.030460648238658905, -0.012873210944235325, -0.0390482023358345, 0.044884901493787766, 0.07177498191595078, -0.004482332617044449, 0.0004163120756857097, -0.005409551318734884, 0.00904903188 [...]
+{"id": 497, "title": "The Green Mile", "embedding": [-0.0013680416159331799, 0.07420185953378677, -0.04049579054117203, -0.047814492136240005, -0.024518707767128944, 0.06344965100288391, 0.02368856593966484, 0.0235735010355711, 0.015413693152368069, 0.012902135960757732, 0.09752783179283142, 0.029224371537566185, -0.004996132105588913, -0.014145193621516228, -0.013104998506605625, -0.00788294617086649, 3.113040293101221e-05, 0.04661109298467636, 0.0036079464480280876, -0.0274894014000892 [...]
+{"id": 198663, "title": "The Maze Runner", "embedding": [0.0023529352620244026, 0.03208910673856735, -0.0500289723277092, 0.045938484370708466, 0.008456805720925331, 0.05391019955277443, -0.03334832191467285, -0.08433830738067627, -0.002074233489111066, -0.03148806840181351, 0.055328119546175, 0.06483067572116852, 0.0704907700419426, 0.0209477748721838, 0.007671857252717018, 0.026649637147784233, -0.028726080432534218, 0.02217939682304859, 0.017385700717568398, -0.10392478108406067, -0.0 [...]
+{"id": 18785, "title": "The Hangover", "embedding": [0.008607619442045689, -0.03156144544482231, -0.04395720362663269, 0.05945231020450592, 0.01963655836880207, 0.06616266816854477, 0.042817406356334686, -0.05422278493642807, 0.01873021200299263, -0.08876138180494308, -0.025066133588552475, -0.00111877650488168, -0.02785518951714039, -0.08866365998983383, -0.0037648126017302275, -0.0583023801445961, 0.0061644590459764, 0.019666511565446854, 0.030200175940990448, 0.012999513186514378, -0. [...]
+{"id": 1891, "title": "The Empire Strikes Back", "embedding": [0.030738849192857742, -0.10612605512142181, 0.0939580500125885, -0.054520655423402786, 0.03552529960870743, 0.08531146496534348, -0.021701538935303688, 0.010059627704322338, 0.02887873537838459, 0.0642378181219101, 0.006218160502612591, 0.008606748655438423, -0.011942895129323006, -0.02632787637412548, 0.004492840729653835, -0.03864063322544098, 0.03571499511599541, 0.01914934813976288, 0.03942131623625755, -0.033530216664075 [...]
+{"id": 808, "title": "Shrek", "embedding": [-0.06589801609516144, 0.0037185337860137224, 0.0798800066113472, 0.017452174797654152, 0.01306501217186451, 0.026504501700401306, 0.11025229841470718, 0.012843780219554901, 0.02522112987935543, -0.06140200048685074, -0.04502357542514801, -0.05155058577656746, 0.008185586892068386, -0.009276801720261574, 0.03926260396838188, -0.04742228612303734, 0.039282482117414474, 0.07863940298557281, -0.009469449520111084, -0.03666459023952484, -0.024881483 [...]
+{"id": 2062, "title": "Ratatouille", "embedding": [0.05374803766608238, -0.0320957787334919, 0.02227843552827835, -0.02942039631307125, -0.08578368276357651, -0.021003101021051407, 0.07780759036540985, -0.05383847653865814, -0.009078508242964745, -0.10377077758312225, -0.018148628994822502, -0.030843209475278854, -0.11209429055452347, -0.005632371641695499, -0.059322286397218704, -0.13053923845291138, 0.06839916855096817, 0.01438064780086279, 0.013232148252427578, -0.03421657532453537, - [...]
+{"id": 207703, "title": "Kingsman: The Secret Service", "embedding": [-0.1354236900806427, -0.032262347638607025, -0.1327115297317505, -0.01709236577153206, -0.002059899503365159, -0.017872130498290062, 0.08357103168964386, -0.03202075511217117, -0.03441489860415459, 0.046769823879003525, 0.0016979339998215437, 0.0326041653752327, 0.08357105404138565, 0.0016741000581532717, 0.0225553959608078, -0.03890632838010788, 0.05923599377274513, -0.03057952970266342, -0.06340523809194565, -0.09068 [...]
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DatasetRewriter.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DatasetRewriter.java
index 4a1e02d47e..6543601027 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DatasetRewriter.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DatasetRewriter.java
@@ -46,6 +46,7 @@ import org.apache.asterix.om.functions.IFunctionToDataSourceRewriter;
import org.apache.asterix.om.typecomputer.base.IResultTypeComputer;
import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.om.utils.ConstantExpressionUtil;
import org.apache.asterix.optimizer.rules.UnnestToDataScanRule;
import org.apache.asterix.optimizer.rules.util.EquivalenceClassUtils;
import org.apache.commons.lang3.mutable.Mutable;
@@ -84,6 +85,13 @@ public class DatasetRewriter implements IFunctionToDataSourceRewriter, IResultTy
MetadataProvider metadataProvider = (MetadataProvider) context.getMetadataProvider();
Dataset dataset = fetchDataset(metadataProvider, f);
+ // Arg layout: [..., arg4 = isSample (Boolean), arg5 = sample index name (String) when isSample is true].
+ Boolean isSample = f.getArguments().size() > 4
+ ? ConstantExpressionUtil.getBooleanConstant(f.getArguments().get(4).get()) : null;
+ String sampleIndexName = null;
+ if (isSample != null && isSample && f.getArguments().size() > 5) {
+ sampleIndexName = FunctionUtil.getStringConstant(f.getArguments().get(5));
+ }
List<LogicalVariable> variables = new ArrayList<>();
switch (dataset.getDatasetType()) {
case INTERNAL:
@@ -104,7 +112,12 @@ public class DatasetRewriter implements IFunctionToDataSourceRewriter, IResultTy
DataSourceId dsid =
new DataSourceId(dataset.getDatabaseName(), dataset.getDataverseName(), dataset.getDatasetName());
- DataSource dataSource = metadataProvider.findDataSource(dsid);
+ DataSource dataSource;
+ if (sampleIndexName != null) {
+ dataSource = metadataProvider.findSampleDataSource(dsid, sampleIndexName);
+ } else {
+ dataSource = metadataProvider.findDataSource(dsid);
+ }
boolean hasMeta = dataSource.hasMeta();
if (hasMeta) {
variables.add(context.newVar());
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/io/PersistedResourceRegistry.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/io/PersistedResourceRegistry.java
index 3adc2435a9..b523999f27 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/io/PersistedResourceRegistry.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/io/PersistedResourceRegistry.java
@@ -71,6 +71,7 @@ import org.apache.asterix.om.types.AUnorderedListType;
import org.apache.asterix.om.types.BuiltinType;
import org.apache.asterix.runtime.compression.CompressionManager;
import org.apache.asterix.runtime.utils.RuntimeComponentsProvider;
+import org.apache.asterix.runtime.utils.VectorDistanceFunctionFactory;
import org.apache.asterix.transaction.management.opcallbacks.PrimaryIndexOperationTrackerFactory;
import org.apache.asterix.transaction.management.opcallbacks.SecondaryIndexOperationTrackerFactory;
import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -126,6 +127,7 @@ import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.UTF8NGramToken
import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.UTF8WordTokenFactory;
import org.apache.hyracks.storage.am.lsm.rtree.dataflow.LSMRTreeLocalResource;
import org.apache.hyracks.storage.am.lsm.rtree.dataflow.LSMRTreeWithAntiMatterLocalResource;
+import org.apache.hyracks.storage.am.lsm.vector.dataflow.LSMVTreeLocalResource;
import org.apache.hyracks.storage.am.rtree.frames.RTreePolicyType;
import org.apache.hyracks.storage.am.rtree.impls.DoublePrimitiveValueProviderFactory;
import org.apache.hyracks.storage.am.rtree.impls.FloatPrimitiveValueProviderFactory;
@@ -162,11 +164,15 @@ public class PersistedResourceRegistry implements IPersistedResourceRegistry {
registeredClasses.put("LSMRTreeWithAntiMatterLocalResource", LSMRTreeWithAntiMatterLocalResource.class);
registeredClasses.put("LSMInvertedIndexLocalResource", LSMInvertedIndexLocalResource.class);
registeredClasses.put("LSMColumnBTreeLocalResource", LSMColumnBTreeLocalResource.class);
+ registeredClasses.put("LSMVTreeLocalResource", LSMVTreeLocalResource.class);
// IVTreeBinaryAccessorFactory — registered so LSMVTreeLocalResource can JSON-round-trip the
// factory via IPersistedResourceRegistry.deserialize() instead of Class.forName lookup.
registeredClasses.put("AOrderedListVectorBinaryAccessorFactory", AOrderedListVectorBinaryAccessorFactory.class);
+ // IVTreeDistanceFunctionFactory — registered so LSMVTreeLocalResource can JSON-round-trip it.
+ registeredClasses.put("VectorDistanceFunctionFactory", VectorDistanceFunctionFactory.class);
+
// ILSMMergePolicyFactory
registeredClasses.put("NoMergePolicyFactory", NoMergePolicyFactory.class);
registeredClasses.put("PrefixMergePolicyFactory", PrefixMergePolicyFactory.class);
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index 5fd404d6d1..3a004a6ca9 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -240,12 +240,15 @@ import org.apache.asterix.metadata.utils.SampleOperationsHelper;
import org.apache.asterix.metadata.utils.TypeUtil;
import org.apache.asterix.om.base.ANull;
import org.apache.asterix.om.base.IAObject;
+import org.apache.asterix.om.types.AOrderedListType;
import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.AUnionType;
import org.apache.asterix.om.types.BuiltinType;
import org.apache.asterix.om.types.BuiltinTypeMap;
import org.apache.asterix.om.types.IAType;
import org.apache.asterix.om.types.TypeSignature;
+import org.apache.asterix.om.types.hierachy.ATypeHierarchy;
import org.apache.asterix.om.utils.RecordUtil;
import org.apache.asterix.optimizer.rules.visitor.FunctionCardinalityInferenceVisitor;
import org.apache.asterix.runtime.fulltext.AbstractFullTextFilterDescriptor;
@@ -312,6 +315,7 @@ import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory;
import org.apache.hyracks.storage.am.lsm.common.dataflow.LSMTreeIndexInsertUpdateDeleteOperatorDescriptor;
import org.apache.hyracks.storage.am.lsm.invertedindex.fulltext.TokenizerCategory;
import org.apache.hyracks.util.LogRedactionUtil;
+import org.apache.hyracks.util.OptionalBoolean;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -1507,6 +1511,9 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
aRecordType = (ARecordType) metadataProvider.findTypeForDatasetWithoutType(aRecordType, ds);
}
+ indexedElements = indexType == IndexType.VTREE ? stmtCreateIndex.getIncludeElements() : indexedElements;
+ indexedElementsCount = indexedElements.size();
+
List<List<IAType>> indexFieldTypes = new ArrayList<>(indexedElementsCount);
boolean hadUnnest = false;
boolean overridesFieldTypes = false;
@@ -1665,7 +1672,7 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
}
if (fieldTypePrime == null) {
- if (indexType != IndexType.BTREE) {
+ if (indexType != IndexType.BTREE && indexType != IndexType.VTREE) {
if (projectPath != null) {
String fieldName =
LogRedactionUtil.userData(RecordUtil.toFullyQualifiedName(projectPath));
@@ -1731,6 +1738,71 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
}
}
Index.IIndexDetails indexDetails;
+ if (indexType == IndexType.VTREE) {
+ List<CreateIndexStatement.IndexedElement> includeElements = stmtCreateIndex.getIncludeElements();
+ int includeElementsCount = includeElements.size();
+
+ List<List<String>> includeFieldNames = new ArrayList<>(includeElementsCount);
+ List<IAType> includeFieldTypes = new ArrayList<>(includeElementsCount);
+ List<Integer> includeFieldSourceIndicators = new ArrayList<>(includeElementsCount);
+
+ for (int i = 0; i < includeElementsCount; i++) {
+ CreateIndexStatement.IndexedElement includeElement = includeElements.get(i);
+ List<String> fieldName = includeElement.getProjectList().get(0).first;
+ IAType fieldType = indexFieldTypes.get(i).get(0);
+ int sourceIndicator = includeElement.getSourceIndicator();
+
+ includeFieldNames.add(fieldName);
+ includeFieldTypes.add(fieldType);
+ includeFieldSourceIndicators.add(sourceIndicator);
+ }
+
+ CreateIndexStatement.IndexedElement indexedElement = stmtCreateIndex.getIndexedElements().get(0);
+ List<String> keyFieldNames = indexedElement.getProjectList().get(0).first;
+
+ // Validate that the vector field exists in the dataset schema
+ Triple<IAType, Boolean, Boolean> vectorFieldType =
+ KeyFieldTypeUtil.getKeyProjectType(aRecordType, keyFieldNames, sourceLoc);
+ if (vectorFieldType == null && !aRecordType.isOpen()) {
+ throw new CompilationException(ErrorCode.COMPILATION_FIELD_NOT_FOUND, sourceLoc,
+ LogRedactionUtil.userData(RecordUtil.toFullyQualifiedName(keyFieldNames)));
+ }
+ // A VTREE index requires the field to be an ordered list of a numeric type (e.g. [double]).
+ // An open/undeclared field (vectorFieldType == null above) has no declared type here and is
+ // validated at runtime; but a declared scalar, record, or non-numeric list must be rejected
+ // now rather than accepted — the storage layer would otherwise read it as if it were [double].
+ if (vectorFieldType != null) {
+ IAType vecType = vectorFieldType.first;
+ boolean numericList = vecType.getTypeTag() == ATypeTag.ANY;
+ if (!numericList && vecType.getTypeTag() == ATypeTag.ARRAY) {
+ IAType itemType = ((AOrderedListType) vecType).getItemType();
+ ATypeTag itemTag = itemType.getTypeTag() == ATypeTag.UNION
+ ? ((AUnionType) itemType).getActualType().getTypeTag() : itemType.getTypeTag();
+ numericList = itemTag == ATypeTag.ANY
+ || ATypeHierarchy.getTypeDomain(itemTag) == ATypeHierarchy.Domain.NUMERIC;
+ }
+ if (!numericList) {
+ throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_KEY_TYPE, sourceLoc,
+ String.valueOf(vecType), "vector");
+ }
+ }
+
+ // excludeUnknownKey is a BTree/Array-index DDL option that does not apply to a vector index;
+ // the vector CREATE INDEX path never sets it, so it is always empty here.
+ indexDetails =
+ new Index.VectorIndexDetails(keyFieldNames, includeFieldNames, includeFieldSourceIndicators,
+ includeFieldTypes, false, OptionalBoolean.empty(), stmtCreateIndex.getWithObjectNode());
+
+ Index newIndex = new Index(databaseName, dataverseName, datasetName, indexName, indexType, indexDetails,
+ stmtCreateIndex.isEnforced(), false, MetadataUtil.PENDING_ADD_OP, creator);
+
+ bActiveTxn = false; // doCreateIndexImpl() takes over the current transaction
+ EntityDetails entityDetails =
+ EntityDetails.newIndex(databaseName, dataverseName, datasetName, indexName);
+ doCreateIndexImpl(hcc, metadataProvider, ds, newIndex, jobFlags, sourceLoc, creator, entityDetails);
+ return;
+ }
+
if (Index.IndexCategory.of(indexType) == Index.IndexCategory.ARRAY) {
if (!stmtCreateIndex.hasExcludeUnknownKey()
|| !stmtCreateIndex.getExcludeUnknownKey().getOrElse(false)) {
@@ -2015,6 +2087,59 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
}
// #. add a new index with PendingAddOp
MetadataManager.INSTANCE.addIndex(metadataProvider.getMetadataTxnContext(), index);
+
+ // VECTOR INDEX: Multi-job pattern (creation -> static structure -> loading)
+ if (index.getIndexType() == IndexType.VTREE) {
+ // JOB 1: Create empty index files
+ spec = IndexUtil.buildSecondaryIndexCreationJobSpec(ds, index, metadataProvider, sourceLoc);
+ if (spec == null) {
+ throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
+ "Failed to create job spec for creating index '" + ds.getDatasetName() + "."
+ + index.getIndexName() + "'");
+ }
+ beforeTxnCommit(metadataProvider, creator, entityDetails);
+ MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+ bActiveTxn = false;
+ progress = ProgressState.ADDED_PENDINGOP_RECORD_TO_METADATA;
+ runJob(hcc, spec, jobFlags);
+
+ if (ds.getDatasetType() == DatasetType.INTERNAL) {
+ FlushDatasetUtil.flushDataset(hcc, metadataProvider, index.getDatabaseName(),
+ index.getDataverseName(), index.getDatasetName());
+ }
+
+ // JOB 2: Create static structure (required before loading records)
+ mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
+ bActiveTxn = true;
+ metadataProvider.setMetadataTxnContext(mdTxnCtx);
+ spec = IndexUtil.buildSecondaryIndexStaticStructureJobSpec(ds, index, metadataProvider, sourceLoc);
+ MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+ bActiveTxn = false;
+ runJob(hcc, spec, jobFlags);
+
+ // JOB 3: Load data into index
+ mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
+ bActiveTxn = true;
+ metadataProvider.setMetadataTxnContext(mdTxnCtx);
+ spec = IndexUtil.buildSecondaryIndexLoadingJobSpec(ds, index, metadataProvider, sourceLoc);
+ MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+ bActiveTxn = false;
+ runJob(hcc, spec, jobFlags);
+
+ // Final cleanup: PendingAddOp -> PendingNoOp
+ mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
+ bActiveTxn = true;
+ metadataProvider.setMetadataTxnContext(mdTxnCtx);
+ MetadataManager.INSTANCE.dropIndex(metadataProvider.getMetadataTxnContext(), index.getDatabaseName(),
+ index.getDataverseName(), index.getDatasetName(), index.getIndexName());
+ index.setPendingOp(MetadataUtil.PENDING_NO_OP);
+ MetadataManager.INSTANCE.addIndex(metadataProvider.getMetadataTxnContext(), index);
+ MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+ bActiveTxn = false;
+
+ return;
+ }
+
// #. prepare to create the index artifact in NC.
spec = IndexUtil.buildSecondaryIndexCreationJobSpec(ds, index, metadataProvider, sourceLoc);
if (spec == null) {
@@ -5302,9 +5427,6 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
}
DatasetStreamStats stats = new DatasetStreamStats(opStats.get(0));
- LOGGER.info("ANALYZED statement stats: coldReads: {} -- pinnedPages: {} -- cloudPageReads:{}",
- stats.getColdReads(), stats.getPinnedPages(), stats.getCloudPageReads());
-
Index.SampleIndexDetails newIndexDetailsFinal = new Index.SampleIndexDetails(dsDetails.getPrimaryKey(),
dsDetails.getKeySourceIndicator(), dsDetails.getPrimaryKeyType(), sampleCardinalityTarget,
stats.getCardinality(), stats.getAvgTupleSize(), sampleSeed, sampleMethod, stats.getIndexesStats());
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/NullMissingExceptionTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/NullMissingExceptionTest.java
index 3c6796abaa..642c38fbb3 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/NullMissingExceptionTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/NullMissingExceptionTest.java
@@ -75,10 +75,14 @@ public class NullMissingExceptionTest {
// Include only functions annotated with MissingNullInOutFunction
if (functionDescriptor.getClass().isAnnotationPresent(MissingNullInOutFunction.class)) {
- // We test all functions except record and cast functions, which requires type settings (we test them
- // in runtime tests).
+ // We test all functions except record and cast functions (which require type settings) and
+ // ann-distance (which requires its metric argument to be a constant string, so it cannot take a
+ // non-constant null/missing). We test those in runtime tests instead.
// TODO(ali): ASTERIXDB-2982 do it in a proper way so that it does not exclude classes inadvertently
- if (!className.contains("record") && !className.contains("Cast")) {
+ // ann-distance builds its factory via DescriptorFactoryUtil, so its factory class name is
+ // generic; match on the descriptor class name instead.
+ if (!className.contains("record") && !className.contains("Cast")
+ && !functionDescriptor.getClass().getName().contains("ANNDistance")) {
tests.add(new Object[] { getTestName(functionDescriptor.getClass()), functionDescriptor });
} else {
LOGGER.log(Level.INFO, "Excluding " + className);
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/NullMissingTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/NullMissingTest.java
index d3adc0d2c4..7c1d55fa33 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/NullMissingTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/NullMissingTest.java
@@ -89,12 +89,16 @@ public class NullMissingTest {
// Include only functions annotated with MissingNullInOutFunction
if (functionDescriptor.getClass().isAnnotationPresent(MissingNullInOutFunction.class)) {
- // We test all functions except record, cast and full-text contains functions,
- // which requires type settings or argument settings.
+ // We test all functions except record, cast, full-text contains, and ann-distance
+ // functions, which require type settings or a constant argument (ann-distance requires
+ // its metric argument to be a constant string, so it cannot take a non-constant null/missing).
// Instead, we test them in runtime tests.
// TODO(ali): ASTERIXDB-2982 do it in a proper way so that it does not exclude classes inadvertently
+ // ann-distance builds its factory via DescriptorFactoryUtil, so its factory class name is
+ // generic; match on the descriptor class name instead.
if (!className.contains("record") && !className.contains("Cast")
- && !className.contains("FullTextContains")) {
+ && !className.contains("FullTextContains")
+ && !functionDescriptor.getClass().getName().contains("ANNDistance")) {
tests.add(new Object[] { getTestName(functionDescriptor.getClass()), functionDescriptor });
} else {
LOGGER.log(Level.INFO, "Excluding " + className);
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/VectorQueries.xml b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/VectorQueries.xml
index 27d4ddfafb..5cfb9f8b73 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/VectorQueries.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/VectorQueries.xml
@@ -22,4 +22,66 @@
<output-dir compare="Clean-JSON">distance-functions</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="vector">
+ <compilation-unit name="create-index-vtree">
+ <output-dir compare="Clean-JSON">create-index-vtree</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="vector">
+ <compilation-unit name="create-index-vtree-nested">
+ <output-dir compare="Clean-JSON">create-index-vtree-nested</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="vector">
+ <compilation-unit name="create-index-vtree-include-filter">
+ <output-dir compare="Clean-JSON">create-index-vtree-include-filter</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="vector">
+ <compilation-unit name="create-index-vtree-include-nested">
+ <output-dir compare="Clean-JSON">create-index-vtree-include-nested</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="vector">
+ <compilation-unit name="create-index-vtree-similarity-normalize">
+ <output-dir compare="Clean-JSON">create-index-vtree-similarity-normalize</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="vector">
+ <compilation-unit name="ann-distance-invalid-metric">
+ <output-dir compare="Text">ann-distance-invalid-metric</output-dir>
+ <expected-error>unknown ann_distance metric</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="vector">
+ <compilation-unit name="create-index-vtree-open-undeclared">
+ <output-dir compare="Clean-JSON">create-index-vtree-open-undeclared</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="vector">
+ <compilation-unit name="create-index-vtree-invalid-field-type">
+ <output-dir compare="Text">create-index-vtree-invalid-field-type</output-dir>
+ <expected-error>cannot be used as a vector key field</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="vector">
+ <compilation-unit name="create-index-vtree-composite-pk">
+ <output-dir compare="Clean-JSON">create-index-vtree-composite-pk</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="vector">
+ <compilation-unit name="create-index-vtree-movie">
+ <output-dir compare="Clean-JSON">create-index-vtree-movie</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="vector">
+ <compilation-unit name="create-index-vtree-glove">
+ <output-dir compare="Clean-JSON">create-index-vtree-glove</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="vector">
+ <compilation-unit name="soft-keyword-identifiers">
+ <output-dir compare="Clean-JSON">soft-keyword-identifiers</output-dir>
+ </compilation-unit>
+ </test-case>
</test-group>
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/ann-distance-invalid-metric/ann-distance-invalid-metric.1.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/ann-distance-invalid-metric/ann-distance-invalid-metric.1.query.sqlpp
new file mode 100644
index 0000000000..4d3e42d735
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/ann-distance-invalid-metric/ann-distance-invalid-metric.1.query.sqlpp
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// ann_distance validates its metric at compile time (during the SQL++ rewrite). An unknown metric must
+// fail compilation with a clear message -- not reach the runtime. (This is the fix for the old behavior
+// where the runtime evaluator threw a COMPILATION_ERROR at execution time.)
+SELECT VALUE ann_distance([1.0, 2.0, 3.0, 4.0], [0.0, 0.0, 0.0, 0.0], "bogus_metric");
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-composite-pk/create-index-vtree-composite-pk.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-composite-pk/create-index-vtree-composite-pk.1.ddl.sqlpp
new file mode 100644
index 0000000000..674c7daf8d
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-composite-pk/create-index-vtree-composite-pk.1.ddl.sqlpp
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// Regression test for composite-primary-key filter pushdown: the INCLUDE-field physical
+// index in PushFilterIntoVectorSearchRule must skip ALL primary key columns, not just one.
+
+DROP DATAVERSE testcpk IF EXISTS;
+CREATE DATAVERSE testcpk;
+USE testcpk;
+
+CREATE TYPE MovieCpkType AS CLOSED {
+ id1: int,
+ id2: int,
+ year: int,
+ title: string,
+ embedding: [double]
+};
+
+CREATE DATASET MovieCpk(MovieCpkType) PRIMARY KEY id1, id2;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-composite-pk/create-index-vtree-composite-pk.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-composite-pk/create-index-vtree-composite-pk.2.update.sqlpp
new file mode 100644
index 0000000000..e2b640d316
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-composite-pk/create-index-vtree-composite-pk.2.update.sqlpp
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE testcpk;
+
+// Two well-separated groups. Group (id1=1): near the origin, years 1991..2005.
+// Group (id1=2): near [10,10,10,*], years 2001..2015. Distances to the query point
+// [0,0,0,0] are unique by construction, so top-k is fully hand-derivable.
+// The `year` values (>= 1991) never overlap the id2 values (<= 15): a filter that is
+// mistakenly evaluated against a PK column admits nothing, making the bug observable.
+INSERT INTO MovieCpk ([
+ {"id1": 1, "id2": 1, "year": 1991, "title": "t1_1", "embedding": [0.0, 0.0, 0.0, 0.1]},
+ {"id1": 1, "id2": 2, "year": 1992, "title": "t1_2", "embedding": [0.0, 0.0, 0.0, 0.2]},
+ {"id1": 1, "id2": 3, "year": 1993, "title": "t1_3", "embedding": [0.0, 0.0, 0.0, 0.3]},
+ {"id1": 1, "id2": 4, "year": 1994, "title": "t1_4", "embedding": [0.0, 0.0, 0.0, 0.4]},
+ {"id1": 1, "id2": 5, "year": 1995, "title": "t1_5", "embedding": [0.0, 0.0, 0.0, 0.5]},
+ {"id1": 1, "id2": 6, "year": 1996, "title": "t1_6", "embedding": [0.0, 0.0, 0.0, 0.6]},
+ {"id1": 1, "id2": 7, "year": 1997, "title": "t1_7", "embedding": [0.0, 0.0, 0.0, 0.7]},
+ {"id1": 1, "id2": 8, "year": 1998, "title": "t1_8", "embedding": [0.0, 0.0, 0.0, 0.8]},
+ {"id1": 1, "id2": 9, "year": 1999, "title": "t1_9", "embedding": [0.0, 0.0, 0.0, 0.9]},
+ {"id1": 1, "id2": 10, "year": 2000, "title": "t1_10", "embedding": [0.0, 0.0, 0.0, 1.0]},
+ {"id1": 1, "id2": 11, "year": 2001, "title": "t1_11", "embedding": [0.0, 0.0, 0.0, 1.1]},
+ {"id1": 1, "id2": 12, "year": 2002, "title": "t1_12", "embedding": [0.0, 0.0, 0.0, 1.2]},
+ {"id1": 1, "id2": 13, "year": 2003, "title": "t1_13", "embedding": [0.0, 0.0, 0.0, 1.3]},
+ {"id1": 1, "id2": 14, "year": 2004, "title": "t1_14", "embedding": [0.0, 0.0, 0.0, 1.4]},
+ {"id1": 1, "id2": 15, "year": 2005, "title": "t1_15", "embedding": [0.0, 0.0, 0.0, 1.5]},
+ {"id1": 2, "id2": 1, "year": 2001, "title": "t2_1", "embedding": [10.0, 10.0, 10.0, 0.1]},
+ {"id1": 2, "id2": 2, "year": 2002, "title": "t2_2", "embedding": [10.0, 10.0, 10.0, 0.2]},
+ {"id1": 2, "id2": 3, "year": 2003, "title": "t2_3", "embedding": [10.0, 10.0, 10.0, 0.3]},
+ {"id1": 2, "id2": 4, "year": 2004, "title": "t2_4", "embedding": [10.0, 10.0, 10.0, 0.4]},
+ {"id1": 2, "id2": 5, "year": 2005, "title": "t2_5", "embedding": [10.0, 10.0, 10.0, 0.5]},
+ {"id1": 2, "id2": 6, "year": 2006, "title": "t2_6", "embedding": [10.0, 10.0, 10.0, 0.6]},
+ {"id1": 2, "id2": 7, "year": 2007, "title": "t2_7", "embedding": [10.0, 10.0, 10.0, 0.7]},
+ {"id1": 2, "id2": 8, "year": 2008, "title": "t2_8", "embedding": [10.0, 10.0, 10.0, 0.8]},
+ {"id1": 2, "id2": 9, "year": 2009, "title": "t2_9", "embedding": [10.0, 10.0, 10.0, 0.9]},
+ {"id1": 2, "id2": 10, "year": 2010, "title": "t2_10", "embedding": [10.0, 10.0, 10.0, 1.0]},
+ {"id1": 2, "id2": 11, "year": 2011, "title": "t2_11", "embedding": [10.0, 10.0, 10.0, 1.1]},
+ {"id1": 2, "id2": 12, "year": 2012, "title": "t2_12", "embedding": [10.0, 10.0, 10.0, 1.2]},
+ {"id1": 2, "id2": 13, "year": 2013, "title": "t2_13", "embedding": [10.0, 10.0, 10.0, 1.3]},
+ {"id1": 2, "id2": 14, "year": 2014, "title": "t2_14", "embedding": [10.0, 10.0, 10.0, 1.4]},
+ {"id1": 2, "id2": 15, "year": 2015, "title": "t2_15", "embedding": [10.0, 10.0, 10.0, 1.5]}
+]);
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-composite-pk/create-index-vtree-composite-pk.3.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-composite-pk/create-index-vtree-composite-pk.3.ddl.sqlpp
new file mode 100644
index 0000000000..b35866fbda
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-composite-pk/create-index-vtree-composite-pk.3.ddl.sqlpp
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// A sample index is required before creating a vector index (quantization training
+// scans it in Job 1). Fixed sample-seed keeps the sample deterministic.
+ANALYZE DATASET testcpk.MovieCpk WITH {"sample-seed": 1000};
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-composite-pk/create-index-vtree-composite-pk.4.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-composite-pk/create-index-vtree-composite-pk.4.ddl.sqlpp
new file mode 100644
index 0000000000..a20ea44599
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-composite-pk/create-index-vtree-composite-pk.4.ddl.sqlpp
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE testcpk;
+
+// Pin the k-means training RNG so index creation is reproducible run-to-run.
+SET `compiler.vector.trainseed` "42";
+
+CREATE INDEX idx_cpk_emb
+ ON MovieCpk(embedding VECTOR) INCLUDE (year)
+ TYPE VTREE
+ WITH { "dimension": 4, "similarity": "euclidean", "num_clusters": 2, "train_list_fraction": 1.0 }
+ EXCLUDE UNKNOWN KEY;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-composite-pk/create-index-vtree-composite-pk.5.plans.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-composite-pk/create-index-vtree-composite-pk.5.plans.sqlpp
new file mode 100644
index 0000000000..5315232928
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-composite-pk/create-index-vtree-composite-pk.5.plans.sqlpp
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+-- param compile-only:string=true
+-- param optimized-logical-plan:string=true
+
+// The optimized plan must show the year filter PUSHED into the vector index search:
+// the vector unnest-map (index-search on idx_cpk_emb) carries "condition (...)"
+// (PushFilterIntoVectorSearchRule). Projecting the non-indexed `title` forces the
+// lookup-and-rerank shape, which is where the rule fires. Same query executed in .6.
+USE testcpk;
+
+SELECT VALUE [m.id1, m.id2, m.title]
+FROM MovieCpk m
+WHERE m.year > 2000
+ORDER BY ann_distance(m.embedding, [0.0, 0.0, 0.0, 0.0], "euclidean", 1.0)
+LIMIT 3;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-composite-pk/create-index-vtree-composite-pk.6.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-composite-pk/create-index-vtree-composite-pk.6.query.sqlpp
new file mode 100644
index 0000000000..4923bb6f21
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-composite-pk/create-index-vtree-composite-pk.6.query.sqlpp
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE testcpk;
+
+// Exact top-3 with a pushed INCLUDE filter over a composite PK. Projecting the
+// non-indexed `title` forces the lookup-and-rerank shape (where the pushdown rule
+// fires); min_probe_fraction 1.0 scans every cluster, so the result is independent of
+// centroid placement. year > 2000 admits (1,11)..(1,15) at distances 1.1..1.5 plus all
+// of group id1=2 at distance >= sqrt(300); expected: [1,11], [1,12], [1,13]. Before
+// the composite-PK offset fix, the pushed filter read PK column id2 (<= 15) instead of
+// year and admitted nothing.
+SELECT VALUE [m.id1, m.id2, m.title]
+FROM MovieCpk m
+WHERE m.year > 2000
+ORDER BY ann_distance(m.embedding, [0.0, 0.0, 0.0, 0.0], "euclidean", 1.0)
+LIMIT 3;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-composite-pk/create-index-vtree-composite-pk.99.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-composite-pk/create-index-vtree-composite-pk.99.ddl.sqlpp
new file mode 100644
index 0000000000..59794927c8
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-composite-pk/create-index-vtree-composite-pk.99.ddl.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+DROP DATAVERSE testcpk IF EXISTS;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-glove/create-index-vtree-glove.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-glove/create-index-vtree-glove.1.ddl.sqlpp
new file mode 100644
index 0000000000..44486c811b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-glove/create-index-vtree-glove.1.ddl.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+DROP DATAVERSE testg IF EXISTS;
+CREATE DATAVERSE testg;
+USE testg;
+CREATE TYPE GloveRealType AS CLOSED { idx: int, label: string, embedding: [double] };
+CREATE DATASET GloveReal(GloveRealType) PRIMARY KEY idx;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-glove/create-index-vtree-glove.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-glove/create-index-vtree-glove.2.update.sqlpp
new file mode 100644
index 0000000000..d4ffcf4261
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-glove/create-index-vtree-glove.2.update.sqlpp
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+USE testg;
+LOAD DATASET GloveReal USING localfs
+ (("path"="asterix_nc1://data/vector/glove-sub.adm"),("format"="adm"));
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-glove/create-index-vtree-glove.3.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-glove/create-index-vtree-glove.3.ddl.sqlpp
new file mode 100644
index 0000000000..8a4147ea0e
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-glove/create-index-vtree-glove.3.ddl.sqlpp
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+ANALYZE DATASET testg.GloveReal WITH {"sample-seed": 1000};
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-glove/create-index-vtree-glove.4.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-glove/create-index-vtree-glove.4.ddl.sqlpp
new file mode 100644
index 0000000000..069c5576bf
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-glove/create-index-vtree-glove.4.ddl.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+USE testg;
+SET `compiler.vector.trainseed` "42";
+CREATE INDEX idx_emb ON GloveReal(embedding VECTOR) TYPE VTREE
+ WITH { "dimension": 100, "similarity": "cosine", "num_clusters": 4, "train_list_fraction": 1.0 }
+ EXCLUDE UNKNOWN KEY;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-glove/create-index-vtree-glove.5.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-glove/create-index-vtree-glove.5.query.sqlpp
new file mode 100644
index 0000000000..3327177eae
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-glove/create-index-vtree-glove.5.query.sqlpp
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+USE testg;
+// Selecting a non-PK field forces the lookup-and-rerank plan: final order is exact
+// ann_distance over real vectors. kmultiplier 30 + min_probe_fraction 1.0 makes the
+// candidate set cover the whole subset, so the golden equals brute force by construction.
+SET `compiler.vector.kmultiplier` "30";
+SELECT VALUE m.label
+FROM GloveReal m
+ORDER BY ann_distance(m.embedding, [-0.31189998984336853, -0.017993999645113945, -0.955560028553009, -0.2546199858188629, 0.3377699851989746, 0.63714998960495, -0.03886299952864647, 0.11513999849557877, -0.049150001257658005, -0.60139000415802, -0.05385199934244156, 0.49226999282836914, -2.186800003051758, 0.5229499936103821, -0.20976999402046204, 0.05969399958848953, -0.3080799877643585, 1.2232999801635742, 0.1712999939918518, -0.025250999256968498, -0.36294999718666077, -0.467610001564 [...]
+LIMIT 5;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-glove/create-index-vtree-glove.99.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-glove/create-index-vtree-glove.99.ddl.sqlpp
new file mode 100644
index 0000000000..24af5fe50e
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-glove/create-index-vtree-glove.99.ddl.sqlpp
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+DROP DATAVERSE testg IF EXISTS;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-include-filter/create-index-vtree-include-filter.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-include-filter/create-index-vtree-include-filter.1.ddl.sqlpp
new file mode 100644
index 0000000000..2148196bd9
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-include-filter/create-index-vtree-include-filter.1.ddl.sqlpp
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+// Regression for the index-only + WHERE-on-INCLUDE empty-result bug: a PK-only projection
+// with a WHERE on an INCLUDE field must return correct rows (guard falls back to lookup-and-rerank),
+// not an empty result from a neutralized select(missing).
+DROP DATAVERSE test IF EXISTS;
+CREATE DATAVERSE test;
+USE test;
+CREATE TYPE MovieType AS {
+ id: int,
+ year: int,
+ embedding: [double]
+};
+CREATE DATASET MovieSmall(MovieType) PRIMARY KEY id;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-include-filter/create-index-vtree-include-filter.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-include-filter/create-index-vtree-include-filter.2.update.sqlpp
new file mode 100644
index 0000000000..4700117922
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-include-filter/create-index-vtree-include-filter.2.update.sqlpp
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+USE test;
+// year = 2000 + id, so WHERE year > 2005 keeps ids 6..30.
+INSERT INTO MovieSmall ([
+ {"id": 1, "year": 2001, "embedding": [0.0, 0.0, 0.0, 0.1]},
+ {"id": 2, "year": 2002, "embedding": [0.0, 0.0, 0.0, 0.2]},
+ {"id": 3, "year": 2003, "embedding": [0.0, 0.0, 0.0, 0.3]},
+ {"id": 4, "year": 2004, "embedding": [0.0, 0.0, 0.0, 0.4]},
+ {"id": 5, "year": 2005, "embedding": [0.0, 0.0, 0.0, 0.5]},
+ {"id": 6, "year": 2006, "embedding": [0.0, 0.0, 0.0, 0.6]},
+ {"id": 7, "year": 2007, "embedding": [0.0, 0.0, 0.0, 0.7]},
+ {"id": 8, "year": 2008, "embedding": [0.0, 0.0, 0.0, 0.8]},
+ {"id": 9, "year": 2009, "embedding": [0.0, 0.0, 0.0, 0.9]},
+ {"id": 10, "year": 2010, "embedding": [0.0, 0.0, 0.0, 1.0]},
+ {"id": 11, "year": 2011, "embedding": [0.0, 0.0, 0.0, 1.1]},
+ {"id": 12, "year": 2012, "embedding": [0.0, 0.0, 0.0, 1.2]},
+ {"id": 13, "year": 2013, "embedding": [0.0, 0.0, 0.0, 1.3]},
+ {"id": 14, "year": 2014, "embedding": [0.0, 0.0, 0.0, 1.4]},
+ {"id": 15, "year": 2015, "embedding": [0.0, 0.0, 0.0, 1.5]},
+ {"id": 16, "year": 2016, "embedding": [10.0, 10.0, 10.0, 0.1]},
+ {"id": 17, "year": 2017, "embedding": [10.0, 10.0, 10.0, 0.2]},
+ {"id": 18, "year": 2018, "embedding": [10.0, 10.0, 10.0, 0.3]},
+ {"id": 19, "year": 2019, "embedding": [10.0, 10.0, 10.0, 0.4]},
+ {"id": 20, "year": 2020, "embedding": [10.0, 10.0, 10.0, 0.5]},
+ {"id": 21, "year": 2021, "embedding": [10.0, 10.0, 10.0, 0.6]},
+ {"id": 22, "year": 2022, "embedding": [10.0, 10.0, 10.0, 0.7]},
+ {"id": 23, "year": 2023, "embedding": [10.0, 10.0, 10.0, 0.8]},
+ {"id": 24, "year": 2024, "embedding": [10.0, 10.0, 10.0, 0.9]},
+ {"id": 25, "year": 2025, "embedding": [10.0, 10.0, 10.0, 1.0]},
+ {"id": 26, "year": 2026, "embedding": [10.0, 10.0, 10.0, 1.1]},
+ {"id": 27, "year": 2027, "embedding": [10.0, 10.0, 10.0, 1.2]},
+ {"id": 28, "year": 2028, "embedding": [10.0, 10.0, 10.0, 1.3]},
+ {"id": 29, "year": 2029, "embedding": [10.0, 10.0, 10.0, 1.4]},
+ {"id": 30, "year": 2030, "embedding": [10.0, 10.0, 10.0, 1.5]}
+]);
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-include-filter/create-index-vtree-include-filter.3.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-include-filter/create-index-vtree-include-filter.3.ddl.sqlpp
new file mode 100644
index 0000000000..b63fc7af98
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-include-filter/create-index-vtree-include-filter.3.ddl.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+// A sample index is required before creating a vector index (quantization training scans it).
+ANALYZE DATASET test.MovieSmall WITH {"sample-seed": 1000};
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-include-filter/create-index-vtree-include-filter.4.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-include-filter/create-index-vtree-include-filter.4.ddl.sqlpp
new file mode 100644
index 0000000000..e00e640643
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-include-filter/create-index-vtree-include-filter.4.ddl.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+USE test;
+SET `compiler.vector.trainseed` "42";
+CREATE INDEX idx_emb
+ ON MovieSmall(embedding VECTOR)
+ INCLUDE (year)
+ TYPE VTREE
+ WITH { "dimension": 4, "similarity": "euclidean", "num_clusters": 2, "train_list_fraction": 1.0 }
+ EXCLUDE UNKNOWN KEY;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-include-filter/create-index-vtree-include-filter.5.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-include-filter/create-index-vtree-include-filter.5.query.sqlpp
new file mode 100644
index 0000000000..039e1fc184
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-include-filter/create-index-vtree-include-filter.5.query.sqlpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+USE test;
+// PK-only projection (SELECT VALUE m.id) + WHERE on the INCLUDE field `year`. min_probe_fraction 1.0
+// scans every cluster, so the expected result is clustering-independent. Filtered set is ids 6..30;
+// nearest 3 to [0,0,0,0] are the near-origin ids 6, 7, 8. Before the guard this returned zero rows.
+SELECT VALUE m.id
+FROM MovieSmall m
+WHERE m.year > 2005
+ORDER BY ann_distance(m.embedding, [0.0, 0.0, 0.0, 0.0], "euclidean", 1.0)
+LIMIT 3;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-include-filter/create-index-vtree-include-filter.99.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-include-filter/create-index-vtree-include-filter.99.ddl.sqlpp
new file mode 100644
index 0000000000..d240ef212d
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-include-filter/create-index-vtree-include-filter.99.ddl.sqlpp
@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+DROP DATAVERSE test IF EXISTS;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-include-nested/create-index-vtree-include-nested.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-include-nested/create-index-vtree-include-nested.1.ddl.sqlpp
new file mode 100644
index 0000000000..09a8cb6c14
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-include-nested/create-index-vtree-include-nested.1.ddl.sqlpp
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// Round-trip test for a NESTED INCLUDE path (IndexTupleTranslator writeIncludeFields/read). The include
+// field info.year is a two-component path; before the fix only its first component was persisted and it did
+// not read back correctly. 'info' is a declared nested record so the covering INCLUDE resolves.
+DROP DATAVERSE test IF EXISTS;
+CREATE DATAVERSE test;
+USE test;
+CREATE TYPE MovieType AS {
+ id: int,
+ info: { year: int },
+ embedding: [double]
+};
+CREATE DATASET MovieSmall(MovieType) PRIMARY KEY id;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-include-nested/create-index-vtree-include-nested.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-include-nested/create-index-vtree-include-nested.2.update.sqlpp
new file mode 100644
index 0000000000..d9d84d9f48
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-include-nested/create-index-vtree-include-nested.2.update.sqlpp
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+// year = 2000 + id, nested under info; WHERE info.year > 2005 keeps ids 6..30.
+INSERT INTO MovieSmall ([
+ {"id": 1, "info": {"year": 2001}, "embedding": [0.0, 0.0, 0.0, 0.1]},
+ {"id": 2, "info": {"year": 2002}, "embedding": [0.0, 0.0, 0.0, 0.2]},
+ {"id": 3, "info": {"year": 2003}, "embedding": [0.0, 0.0, 0.0, 0.3]},
+ {"id": 4, "info": {"year": 2004}, "embedding": [0.0, 0.0, 0.0, 0.4]},
+ {"id": 5, "info": {"year": 2005}, "embedding": [0.0, 0.0, 0.0, 0.5]},
+ {"id": 6, "info": {"year": 2006}, "embedding": [0.0, 0.0, 0.0, 0.6]},
+ {"id": 7, "info": {"year": 2007}, "embedding": [0.0, 0.0, 0.0, 0.7]},
+ {"id": 8, "info": {"year": 2008}, "embedding": [0.0, 0.0, 0.0, 0.8]},
+ {"id": 9, "info": {"year": 2009}, "embedding": [0.0, 0.0, 0.0, 0.9]},
+ {"id": 10, "info": {"year": 2010}, "embedding": [0.0, 0.0, 0.0, 1.0]},
+ {"id": 11, "info": {"year": 2011}, "embedding": [0.0, 0.0, 0.0, 1.1]},
+ {"id": 12, "info": {"year": 2012}, "embedding": [0.0, 0.0, 0.0, 1.2]},
+ {"id": 13, "info": {"year": 2013}, "embedding": [0.0, 0.0, 0.0, 1.3]},
+ {"id": 14, "info": {"year": 2014}, "embedding": [0.0, 0.0, 0.0, 1.4]},
+ {"id": 15, "info": {"year": 2015}, "embedding": [0.0, 0.0, 0.0, 1.5]},
+ {"id": 16, "info": {"year": 2016}, "embedding": [10.0, 10.0, 10.0, 0.1]},
+ {"id": 17, "info": {"year": 2017}, "embedding": [10.0, 10.0, 10.0, 0.2]},
+ {"id": 18, "info": {"year": 2018}, "embedding": [10.0, 10.0, 10.0, 0.3]},
+ {"id": 19, "info": {"year": 2019}, "embedding": [10.0, 10.0, 10.0, 0.4]},
+ {"id": 20, "info": {"year": 2020}, "embedding": [10.0, 10.0, 10.0, 0.5]},
+ {"id": 21, "info": {"year": 2021}, "embedding": [10.0, 10.0, 10.0, 0.6]},
+ {"id": 22, "info": {"year": 2022}, "embedding": [10.0, 10.0, 10.0, 0.7]},
+ {"id": 23, "info": {"year": 2023}, "embedding": [10.0, 10.0, 10.0, 0.8]},
+ {"id": 24, "info": {"year": 2024}, "embedding": [10.0, 10.0, 10.0, 0.9]},
+ {"id": 25, "info": {"year": 2025}, "embedding": [10.0, 10.0, 10.0, 1.0]},
+ {"id": 26, "info": {"year": 2026}, "embedding": [10.0, 10.0, 10.0, 1.1]},
+ {"id": 27, "info": {"year": 2027}, "embedding": [10.0, 10.0, 10.0, 1.2]},
+ {"id": 28, "info": {"year": 2028}, "embedding": [10.0, 10.0, 10.0, 1.3]},
+ {"id": 29, "info": {"year": 2029}, "embedding": [10.0, 10.0, 10.0, 1.4]},
+ {"id": 30, "info": {"year": 2030}, "embedding": [10.0, 10.0, 10.0, 1.5]}
+]);
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-include-nested/create-index-vtree-include-nested.3.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-include-nested/create-index-vtree-include-nested.3.ddl.sqlpp
new file mode 100644
index 0000000000..8deeec9a1f
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-include-nested/create-index-vtree-include-nested.3.ddl.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+ANALYZE DATASET test.MovieSmall WITH {"sample-seed": 1000};
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-include-nested/create-index-vtree-include-nested.4.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-include-nested/create-index-vtree-include-nested.4.ddl.sqlpp
new file mode 100644
index 0000000000..ca3b9cb48b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-include-nested/create-index-vtree-include-nested.4.ddl.sqlpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+SET `compiler.vector.trainseed` "42";
+CREATE INDEX idx_emb
+ ON MovieSmall(embedding VECTOR)
+ INCLUDE (info.year)
+ TYPE VTREE
+ WITH { "dimension": 4, "similarity": "euclidean", "num_clusters": 2, "train_list_fraction": 1.0 }
+ EXCLUDE UNKNOWN KEY;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-include-nested/create-index-vtree-include-nested.5.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-include-nested/create-index-vtree-include-nested.5.query.sqlpp
new file mode 100644
index 0000000000..98f04e029b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-include-nested/create-index-vtree-include-nested.5.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// The persisted INCLUDE path must round-trip with all its components as [["info","year"]]. Before the fix
+// the write side kept only the first component and used a typed (non-self-describing) string list, so this
+// query did not return the nested path (it produced a truncated / misread value here).
+SELECT VALUE i.IncludeFields
+FROM Metadata.`Index` i
+WHERE i.DataverseName = "test" AND i.DatasetName = "MovieSmall" AND i.IndexName = "idx_emb";
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-include-nested/create-index-vtree-include-nested.6.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-include-nested/create-index-vtree-include-nested.6.query.sqlpp
new file mode 100644
index 0000000000..e4bacc192d
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-include-nested/create-index-vtree-include-nested.6.query.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+// Functional check: WHERE on the nested INCLUDE field. Filtered set ids 6..30; nearest 3 to origin are 6,7,8.
+SELECT VALUE m.id
+FROM MovieSmall m
+WHERE m.info.year > 2005
+ORDER BY ann_distance(m.embedding, [0.0, 0.0, 0.0, 0.0], "euclidean", 1.0)
+LIMIT 3;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-include-nested/create-index-vtree-include-nested.99.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-include-nested/create-index-vtree-include-nested.99.ddl.sqlpp
new file mode 100644
index 0000000000..20dc6fde5e
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-include-nested/create-index-vtree-include-nested.99.ddl.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+DROP DATAVERSE test IF EXISTS;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-invalid-field-type/create-index-vtree-invalid-field-type.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-invalid-field-type/create-index-vtree-invalid-field-type.1.ddl.sqlpp
new file mode 100644
index 0000000000..d08a19f6bc
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-invalid-field-type/create-index-vtree-invalid-field-type.1.ddl.sqlpp
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// Negative test (ASTERIXDB-3771 review): a VTREE index whose declared vector field is not an
+// ordered list of a numeric type must be rejected at CREATE INDEX, not silently accepted.
+// Here `embedding` is declared `string`, so CREATE INDEX must fail with ASX1022.
+
+DROP DATAVERSE test IF EXISTS;
+CREATE DATAVERSE test;
+USE test;
+
+CREATE TYPE MovieType AS {
+ id: int,
+ embedding: string
+};
+
+CREATE DATASET MovieSmall(MovieType) PRIMARY KEY id;
+
+CREATE INDEX idx_emb
+ ON MovieSmall(embedding VECTOR)
+ TYPE VTREE
+ WITH { "dimension": 4, "similarity": "euclidean" }
+ EXCLUDE UNKNOWN KEY;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-movie/create-index-vtree-movie.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-movie/create-index-vtree-movie.1.ddl.sqlpp
new file mode 100644
index 0000000000..308bc536f4
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-movie/create-index-vtree-movie.1.ddl.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+DROP DATAVERSE testm IF EXISTS;
+CREATE DATAVERSE testm;
+USE testm;
+CREATE TYPE MovieRealType AS CLOSED { id: int, title: string, embedding: [double] };
+CREATE DATASET MovieReal(MovieRealType) PRIMARY KEY id;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-movie/create-index-vtree-movie.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-movie/create-index-vtree-movie.2.update.sqlpp
new file mode 100644
index 0000000000..1a3e15e8a8
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-movie/create-index-vtree-movie.2.update.sqlpp
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+USE testm;
+LOAD DATASET MovieReal USING localfs
+ (("path"="asterix_nc1://data/vector/movie-sub.adm"),("format"="adm"));
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-movie/create-index-vtree-movie.3.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-movie/create-index-vtree-movie.3.ddl.sqlpp
new file mode 100644
index 0000000000..f2c3f71dae
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-movie/create-index-vtree-movie.3.ddl.sqlpp
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+ANALYZE DATASET testm.MovieReal WITH {"sample-seed": 1000};
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-movie/create-index-vtree-movie.4.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-movie/create-index-vtree-movie.4.ddl.sqlpp
new file mode 100644
index 0000000000..c978239ecb
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-movie/create-index-vtree-movie.4.ddl.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+USE testm;
+SET `compiler.vector.trainseed` "42";
+CREATE INDEX idx_emb ON MovieReal(embedding VECTOR) TYPE VTREE
+ WITH { "dimension": 384, "similarity": "l2_squared", "num_clusters": 4, "train_list_fraction": 1.0 }
+ EXCLUDE UNKNOWN KEY;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-movie/create-index-vtree-movie.5.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-movie/create-index-vtree-movie.5.query.sqlpp
new file mode 100644
index 0000000000..ef0e1892c1
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-movie/create-index-vtree-movie.5.query.sqlpp
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+USE testm;
+// Selecting a non-PK field forces the lookup-and-rerank plan: final order is exact
+// ann_distance over real vectors. kmultiplier 30 + min_probe_fraction 1.0 makes the
+// candidate set cover the whole subset, so the golden equals brute force by construction.
+SET `compiler.vector.kmultiplier` "30";
+SELECT VALUE m.title
+FROM MovieReal m
+ORDER BY ann_distance(m.embedding, [-0.01857922412455082, -0.0614464208483696, -0.07124938815832138, 0.003173143370077014, -0.026651009917259216, 0.07428602129220963, 0.014454759657382965, 0.0009055715636350214, 0.09926825761795044, -0.05026109889149666, 0.050097666680812836, 0.041439659893512726, 0.009396902285516262, 0.04711673781275749, 0.004123533610254526, -0.03571753948926926, 0.03176308423280716, 0.0560125932097435, 0.04764378443360329, 0.009818535298109055, -0.005663667339831591, [...]
+LIMIT 5;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-movie/create-index-vtree-movie.99.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-movie/create-index-vtree-movie.99.ddl.sqlpp
new file mode 100644
index 0000000000..839a80a0e3
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-movie/create-index-vtree-movie.99.ddl.sqlpp
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+DROP DATAVERSE testm IF EXISTS;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-nested/create-index-vtree-nested.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-nested/create-index-vtree-nested.1.ddl.sqlpp
new file mode 100644
index 0000000000..1fef2fe114
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-nested/create-index-vtree-nested.1.ddl.sqlpp
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// Nested-path variant of create-index-vtree: the vector field lives one level deep (meta.embedding),
+// exercising CreateIndexStatement path extraction (getProjectList().get(0).first) for a dotted key AND
+// a dotted INCLUDE field, not just a flat reference.
+DROP DATAVERSE test IF EXISTS;
+CREATE DATAVERSE test;
+USE test;
+
+CREATE TYPE InnerType AS {
+ embedding: [double],
+ year: int
+};
+
+CREATE TYPE MovieType AS {
+ id: int,
+ meta: InnerType
+};
+
+CREATE DATASET MovieNested(MovieType) PRIMARY KEY id;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-nested/create-index-vtree-nested.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-nested/create-index-vtree-nested.2.update.sqlpp
new file mode 100644
index 0000000000..4a23fb2771
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-nested/create-index-vtree-nested.2.update.sqlpp
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+// Two well-separated groups; distances to query point [0,0,0,0] are unique by construction.
+INSERT INTO MovieNested ([
+ {"id": 1, "meta": {"embedding": [0.0, 0.0, 0.0, 0.1], "year": 2001}},
+ {"id": 2, "meta": {"embedding": [0.0, 0.0, 0.0, 0.2], "year": 2002}},
+ {"id": 3, "meta": {"embedding": [0.0, 0.0, 0.0, 0.3], "year": 2003}},
+ {"id": 4, "meta": {"embedding": [0.0, 0.0, 0.0, 0.4], "year": 2004}},
+ {"id": 5, "meta": {"embedding": [0.0, 0.0, 0.0, 0.5], "year": 2005}},
+ {"id": 6, "meta": {"embedding": [0.0, 0.0, 0.0, 0.6], "year": 2006}},
+ {"id": 7, "meta": {"embedding": [0.0, 0.0, 0.0, 0.7], "year": 2007}},
+ {"id": 8, "meta": {"embedding": [0.0, 0.0, 0.0, 0.8], "year": 2008}},
+ {"id": 9, "meta": {"embedding": [0.0, 0.0, 0.0, 0.9], "year": 2009}},
+ {"id": 10, "meta": {"embedding": [0.0, 0.0, 0.0, 0.10], "year": 2010}},
+ {"id": 11, "meta": {"embedding": [0.0, 0.0, 0.0, 0.11], "year": 2011}},
+ {"id": 12, "meta": {"embedding": [0.0, 0.0, 0.0, 0.12], "year": 2012}},
+ {"id": 13, "meta": {"embedding": [0.0, 0.0, 0.0, 0.13], "year": 2013}},
+ {"id": 14, "meta": {"embedding": [0.0, 0.0, 0.0, 0.14], "year": 2014}},
+ {"id": 15, "meta": {"embedding": [0.0, 0.0, 0.0, 0.15], "year": 2015}},
+ {"id": 16, "meta": {"embedding": [10.0, 10.0, 10.0, 0.1], "year": 2016}},
+ {"id": 17, "meta": {"embedding": [10.0, 10.0, 10.0, 0.2], "year": 2017}},
+ {"id": 18, "meta": {"embedding": [10.0, 10.0, 10.0, 0.3], "year": 2018}},
+ {"id": 19, "meta": {"embedding": [10.0, 10.0, 10.0, 0.4], "year": 2019}},
+ {"id": 20, "meta": {"embedding": [10.0, 10.0, 10.0, 0.5], "year": 2020}},
+ {"id": 21, "meta": {"embedding": [10.0, 10.0, 10.0, 0.6], "year": 2021}},
+ {"id": 22, "meta": {"embedding": [10.0, 10.0, 10.0, 0.7], "year": 2022}},
+ {"id": 23, "meta": {"embedding": [10.0, 10.0, 10.0, 0.8], "year": 2023}},
+ {"id": 24, "meta": {"embedding": [10.0, 10.0, 10.0, 0.9], "year": 2024}},
+ {"id": 25, "meta": {"embedding": [10.0, 10.0, 10.0, 0.10], "year": 2025}},
+ {"id": 26, "meta": {"embedding": [10.0, 10.0, 10.0, 0.11], "year": 2026}},
+ {"id": 27, "meta": {"embedding": [10.0, 10.0, 10.0, 0.12], "year": 2027}},
+ {"id": 28, "meta": {"embedding": [10.0, 10.0, 10.0, 0.13], "year": 2028}},
+ {"id": 29, "meta": {"embedding": [10.0, 10.0, 10.0, 0.14], "year": 2029}},
+ {"id": 30, "meta": {"embedding": [10.0, 10.0, 10.0, 0.15], "year": 2030}}
+]);
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-nested/create-index-vtree-nested.3.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-nested/create-index-vtree-nested.3.ddl.sqlpp
new file mode 100644
index 0000000000..e72b71c60f
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-nested/create-index-vtree-nested.3.ddl.sqlpp
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// A sample index is required before creating a vector index (quantization training scans it).
+ANALYZE DATASET test.MovieNested WITH {"sample-seed": 1000};
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-nested/create-index-vtree-nested.4.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-nested/create-index-vtree-nested.4.ddl.sqlpp
new file mode 100644
index 0000000000..29219d4c20
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-nested/create-index-vtree-nested.4.ddl.sqlpp
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+// Nested key path (meta.embedding) and nested INCLUDE path (meta.year).
+SET `compiler.vector.trainseed` "42";
+CREATE INDEX idx_emb
+ ON MovieNested(meta.embedding VECTOR)
+ INCLUDE (meta.year)
+ TYPE VTREE
+ WITH { "dimension": 4, "similarity": "euclidean", "num_clusters": 2, "train_list_fraction": 1.0 }
+ EXCLUDE UNKNOWN KEY;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-nested/create-index-vtree-nested.5.plans.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-nested/create-index-vtree-nested.5.plans.sqlpp
new file mode 100644
index 0000000000..54dd64a60d
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-nested/create-index-vtree-nested.5.plans.sqlpp
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+-- param compile-only:string=true
+-- param optimized-logical-plan:string=true
+
+// The optimized plan must reference idx_emb: proves IntroduceTopKAccessMethodRule rewrote the scan
+// into a vector index search on the NESTED key path meta.embedding (a fallback scan never names it).
+SELECT VALUE m.id
+FROM test.MovieNested m
+ORDER BY ann_distance(m.meta.embedding, [0.0, 0.0, 0.0, 0.0], "euclidean", 1.0)
+LIMIT 3;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-nested/create-index-vtree-nested.99.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-nested/create-index-vtree-nested.99.ddl.sqlpp
new file mode 100644
index 0000000000..20dc6fde5e
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-nested/create-index-vtree-nested.99.ddl.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+DROP DATAVERSE test IF EXISTS;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-open-undeclared/create-index-vtree-open-undeclared.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-open-undeclared/create-index-vtree-open-undeclared.1.ddl.sqlpp
new file mode 100644
index 0000000000..afbda1e824
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-open-undeclared/create-index-vtree-open-undeclared.1.ddl.sqlpp
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// An UNDECLARED vector field on an OPEN type is a supported, indexable case. The vector DDL requires
+// EXCLUDE UNKNOWN KEY, which skips records whose key is missing and indexes those that have it, so an
+// open/undeclared embedding present in the records builds and queries normally (it does not fail with
+// "no training vectors"). This test guards that behavior end to end.
+DROP DATAVERSE test IF EXISTS;
+CREATE DATAVERSE test;
+USE test;
+
+// OPEN type: 'embedding' is NOT declared; it exists only in the record data.
+CREATE TYPE MovieType AS OPEN {
+ id: int
+};
+CREATE DATASET MovieOpen(MovieType) PRIMARY KEY id;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-open-undeclared/create-index-vtree-open-undeclared.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-open-undeclared/create-index-vtree-open-undeclared.2.update.sqlpp
new file mode 100644
index 0000000000..406ce7deef
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-open-undeclared/create-index-vtree-open-undeclared.2.update.sqlpp
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+// Same well-separated data as create-index-vtree (ids 1-15 near origin, 16-30 near [10,10,10,*],
+// 4th coord 0.1..1.5 so distances to [0,0,0,0] are unique), EXCEPT the embedding field is undeclared
+// (the type is OPEN). id 31 has NO embedding -> exercises EXCLUDE UNKNOWN KEY (must be skipped, not fail).
+INSERT INTO MovieOpen ([
+ {"id": 1, "embedding": [0.0, 0.0, 0.0, 0.1]},
+ {"id": 2, "embedding": [0.0, 0.0, 0.0, 0.2]},
+ {"id": 3, "embedding": [0.0, 0.0, 0.0, 0.3]},
+ {"id": 4, "embedding": [0.0, 0.0, 0.0, 0.4]},
+ {"id": 5, "embedding": [0.0, 0.0, 0.0, 0.5]},
+ {"id": 6, "embedding": [0.0, 0.0, 0.0, 0.6]},
+ {"id": 7, "embedding": [0.0, 0.0, 0.0, 0.7]},
+ {"id": 8, "embedding": [0.0, 0.0, 0.0, 0.8]},
+ {"id": 9, "embedding": [0.0, 0.0, 0.0, 0.9]},
+ {"id": 10, "embedding": [0.0, 0.0, 0.0, 1.0]},
+ {"id": 11, "embedding": [0.0, 0.0, 0.0, 1.1]},
+ {"id": 12, "embedding": [0.0, 0.0, 0.0, 1.2]},
+ {"id": 13, "embedding": [0.0, 0.0, 0.0, 1.3]},
+ {"id": 14, "embedding": [0.0, 0.0, 0.0, 1.4]},
+ {"id": 15, "embedding": [0.0, 0.0, 0.0, 1.5]},
+ {"id": 16, "embedding": [10.0, 10.0, 10.0, 0.1]},
+ {"id": 17, "embedding": [10.0, 10.0, 10.0, 0.2]},
+ {"id": 18, "embedding": [10.0, 10.0, 10.0, 0.3]},
+ {"id": 19, "embedding": [10.0, 10.0, 10.0, 0.4]},
+ {"id": 20, "embedding": [10.0, 10.0, 10.0, 0.5]},
+ {"id": 21, "embedding": [10.0, 10.0, 10.0, 0.6]},
+ {"id": 22, "embedding": [10.0, 10.0, 10.0, 0.7]},
+ {"id": 23, "embedding": [10.0, 10.0, 10.0, 0.8]},
+ {"id": 24, "embedding": [10.0, 10.0, 10.0, 0.9]},
+ {"id": 25, "embedding": [10.0, 10.0, 10.0, 1.0]},
+ {"id": 26, "embedding": [10.0, 10.0, 10.0, 1.1]},
+ {"id": 27, "embedding": [10.0, 10.0, 10.0, 1.2]},
+ {"id": 28, "embedding": [10.0, 10.0, 10.0, 1.3]},
+ {"id": 29, "embedding": [10.0, 10.0, 10.0, 1.4]},
+ {"id": 30, "embedding": [10.0, 10.0, 10.0, 1.5]},
+ {"id": 31}
+]);
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-open-undeclared/create-index-vtree-open-undeclared.3.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-open-undeclared/create-index-vtree-open-undeclared.3.ddl.sqlpp
new file mode 100644
index 0000000000..0f7733d3e0
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-open-undeclared/create-index-vtree-open-undeclared.3.ddl.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+ANALYZE DATASET test.MovieOpen WITH {"sample-seed": 1000};
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-open-undeclared/create-index-vtree-open-undeclared.4.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-open-undeclared/create-index-vtree-open-undeclared.4.ddl.sqlpp
new file mode 100644
index 0000000000..bb681822ff
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-open-undeclared/create-index-vtree-open-undeclared.4.ddl.sqlpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+SET `compiler.vector.trainseed` "42";
+// Index on the UNDECLARED 'embedding' field of the OPEN type; this builds like any declared vector field.
+CREATE INDEX idx_emb
+ ON MovieOpen(embedding VECTOR)
+ TYPE VTREE
+ WITH { "dimension": 4, "similarity": "euclidean", "num_clusters": 2, "train_list_fraction": 1.0 }
+ EXCLUDE UNKNOWN KEY;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-open-undeclared/create-index-vtree-open-undeclared.5.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-open-undeclared/create-index-vtree-open-undeclared.5.query.sqlpp
new file mode 100644
index 0000000000..63632d9533
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-open-undeclared/create-index-vtree-open-undeclared.5.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+// The 3 nearest to [0,0,0,0] are ids 1,2,3; id 31 (no embedding) is excluded by EXCLUDE UNKNOWN KEY.
+SELECT VALUE m.id
+FROM MovieOpen m
+ORDER BY ann_distance(m.embedding, [0.0, 0.0, 0.0, 0.0], "euclidean", 1.0)
+LIMIT 3;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-open-undeclared/create-index-vtree-open-undeclared.6.plans.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-open-undeclared/create-index-vtree-open-undeclared.6.plans.sqlpp
new file mode 100644
index 0000000000..a87ace44e2
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-open-undeclared/create-index-vtree-open-undeclared.6.plans.sqlpp
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+-- param compile-only:string=true
+-- param optimized-logical-plan:string=true
+// Decisive check: the optimized plan must name idx_emb. This proves the index over the UNDECLARED (open)
+// 'embedding' field was actually populated and is served by IntroduceTopKAccessMethodRule -- i.e. the open
+// field is genuinely trained/indexed, NOT an empty index that silently falls back to a full scan (which
+// would also return 1,2,3 and mask the difference). Confirms open vector fields are a supported case.
+SELECT VALUE m.id
+FROM test.MovieOpen m
+ORDER BY ann_distance(m.embedding, [0.0, 0.0, 0.0, 0.0], "euclidean", 1.0)
+LIMIT 3;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-open-undeclared/create-index-vtree-open-undeclared.99.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-open-undeclared/create-index-vtree-open-undeclared.99.ddl.sqlpp
new file mode 100644
index 0000000000..20dc6fde5e
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-open-undeclared/create-index-vtree-open-undeclared.99.ddl.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+DROP DATAVERSE test IF EXISTS;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-similarity-normalize/create-index-vtree-similarity-normalize.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-similarity-normalize/create-index-vtree-similarity-normalize.1.ddl.sqlpp
new file mode 100644
index 0000000000..314934adef
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-similarity-normalize/create-index-vtree-similarity-normalize.1.ddl.sqlpp
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// Regression for `similarity` validation + normalization in VectorIndexDeclUtil: the WITH clause writes an
+// alias/casing ("L2") and the persisted metadata must store the canonical spelling ("euclidean").
+
+DROP DATAVERSE test IF EXISTS;
+CREATE DATAVERSE test;
+USE test;
+
+CREATE TYPE MovieType AS {
+ id: int,
+ embedding: [double]
+};
+
+CREATE DATASET MovieSmall(MovieType) PRIMARY KEY id;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-similarity-normalize/create-index-vtree-similarity-normalize.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-similarity-normalize/create-index-vtree-similarity-normalize.2.update.sqlpp
new file mode 100644
index 0000000000..4afab9ff58
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-similarity-normalize/create-index-vtree-similarity-normalize.2.update.sqlpp
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+// Two well-separated groups: ids 1-15 near the origin, ids 16-30 near [10,10,10,*].
+// Distances to the query point [0,0,0,0] are unique by construction.
+INSERT INTO MovieSmall ([
+ {"id": 1, "embedding": [0.0, 0.0, 0.0, 0.1]},
+ {"id": 2, "embedding": [0.0, 0.0, 0.0, 0.2]},
+ {"id": 3, "embedding": [0.0, 0.0, 0.0, 0.3]},
+ {"id": 4, "embedding": [0.0, 0.0, 0.0, 0.4]},
+ {"id": 5, "embedding": [0.0, 0.0, 0.0, 0.5]},
+ {"id": 6, "embedding": [0.0, 0.0, 0.0, 0.6]},
+ {"id": 7, "embedding": [0.0, 0.0, 0.0, 0.7]},
+ {"id": 8, "embedding": [0.0, 0.0, 0.0, 0.8]},
+ {"id": 9, "embedding": [0.0, 0.0, 0.0, 0.9]},
+ {"id": 10, "embedding": [0.0, 0.0, 0.0, 1.0]},
+ {"id": 11, "embedding": [0.0, 0.0, 0.0, 1.1]},
+ {"id": 12, "embedding": [0.0, 0.0, 0.0, 1.2]},
+ {"id": 13, "embedding": [0.0, 0.0, 0.0, 1.3]},
+ {"id": 14, "embedding": [0.0, 0.0, 0.0, 1.4]},
+ {"id": 15, "embedding": [0.0, 0.0, 0.0, 1.5]},
+ {"id": 16, "embedding": [10.0, 10.0, 10.0, 0.1]},
+ {"id": 17, "embedding": [10.0, 10.0, 10.0, 0.2]},
+ {"id": 18, "embedding": [10.0, 10.0, 10.0, 0.3]},
+ {"id": 19, "embedding": [10.0, 10.0, 10.0, 0.4]},
+ {"id": 20, "embedding": [10.0, 10.0, 10.0, 0.5]},
+ {"id": 21, "embedding": [10.0, 10.0, 10.0, 0.6]},
+ {"id": 22, "embedding": [10.0, 10.0, 10.0, 0.7]},
+ {"id": 23, "embedding": [10.0, 10.0, 10.0, 0.8]},
+ {"id": 24, "embedding": [10.0, 10.0, 10.0, 0.9]},
+ {"id": 25, "embedding": [10.0, 10.0, 10.0, 1.0]},
+ {"id": 26, "embedding": [10.0, 10.0, 10.0, 1.1]},
+ {"id": 27, "embedding": [10.0, 10.0, 10.0, 1.2]},
+ {"id": 28, "embedding": [10.0, 10.0, 10.0, 1.3]},
+ {"id": 29, "embedding": [10.0, 10.0, 10.0, 1.4]},
+ {"id": 30, "embedding": [10.0, 10.0, 10.0, 1.5]}
+]);
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-similarity-normalize/create-index-vtree-similarity-normalize.3.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-similarity-normalize/create-index-vtree-similarity-normalize.3.ddl.sqlpp
new file mode 100644
index 0000000000..9f3b1a4884
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-similarity-normalize/create-index-vtree-similarity-normalize.3.ddl.sqlpp
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// A sample index is required before creating a vector index (quantization training
+// scans it in Job 1). Fixed sample-seed keeps the sample deterministic.
+ANALYZE DATASET test.MovieSmall WITH {"sample-seed": 1000};
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-similarity-normalize/create-index-vtree-similarity-normalize.4.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-similarity-normalize/create-index-vtree-similarity-normalize.4.ddl.sqlpp
new file mode 100644
index 0000000000..0d34c950b2
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-similarity-normalize/create-index-vtree-similarity-normalize.4.ddl.sqlpp
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+SET `compiler.vector.trainseed` "42";
+// similarity given as the alias "L2" (not the canonical "euclidean"); validateSimilarity must accept it
+// via VectorSimilarityMetric.fromAlias and normalize the stored value to the canonical spelling.
+CREATE INDEX idx_emb
+ ON MovieSmall(embedding VECTOR)
+ TYPE VTREE
+ WITH { "dimension": 4, "similarity": "L2", "num_clusters": 2, "train_list_fraction": 1.0 }
+ EXCLUDE UNKNOWN KEY;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-similarity-normalize/create-index-vtree-similarity-normalize.5.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-similarity-normalize/create-index-vtree-similarity-normalize.5.query.sqlpp
new file mode 100644
index 0000000000..7faf54f6f4
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-similarity-normalize/create-index-vtree-similarity-normalize.5.query.sqlpp
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// The stored similarity must be the canonical "euclidean", not the alias/casing "L2" that was written.
+SELECT VALUE i.similarity
+FROM Metadata.`Index` i
+WHERE i.DataverseName = "test" AND i.DatasetName = "MovieSmall" AND i.IndexName = "idx_emb";
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-similarity-normalize/create-index-vtree-similarity-normalize.99.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-similarity-normalize/create-index-vtree-similarity-normalize.99.ddl.sqlpp
new file mode 100644
index 0000000000..20dc6fde5e
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-similarity-normalize/create-index-vtree-similarity-normalize.99.ddl.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+DROP DATAVERSE test IF EXISTS;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree/create-index-vtree.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree/create-index-vtree.1.ddl.sqlpp
new file mode 100644
index 0000000000..df09de151b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree/create-index-vtree.1.ddl.sqlpp
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// Deterministic end-to-end regression for VTREE index creation (three-job pipeline):
+// quantization training, hierarchical k-means (seeded), static structure build, bulk load.
+
+DROP DATAVERSE test IF EXISTS;
+CREATE DATAVERSE test;
+USE test;
+
+CREATE TYPE MovieType AS {
+ id: int,
+ embedding: [double]
+};
+
+CREATE DATASET MovieSmall(MovieType) PRIMARY KEY id;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree/create-index-vtree.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree/create-index-vtree.2.update.sqlpp
new file mode 100644
index 0000000000..4afab9ff58
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree/create-index-vtree.2.update.sqlpp
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+// Two well-separated groups: ids 1-15 near the origin, ids 16-30 near [10,10,10,*].
+// Distances to the query point [0,0,0,0] are unique by construction.
+INSERT INTO MovieSmall ([
+ {"id": 1, "embedding": [0.0, 0.0, 0.0, 0.1]},
+ {"id": 2, "embedding": [0.0, 0.0, 0.0, 0.2]},
+ {"id": 3, "embedding": [0.0, 0.0, 0.0, 0.3]},
+ {"id": 4, "embedding": [0.0, 0.0, 0.0, 0.4]},
+ {"id": 5, "embedding": [0.0, 0.0, 0.0, 0.5]},
+ {"id": 6, "embedding": [0.0, 0.0, 0.0, 0.6]},
+ {"id": 7, "embedding": [0.0, 0.0, 0.0, 0.7]},
+ {"id": 8, "embedding": [0.0, 0.0, 0.0, 0.8]},
+ {"id": 9, "embedding": [0.0, 0.0, 0.0, 0.9]},
+ {"id": 10, "embedding": [0.0, 0.0, 0.0, 1.0]},
+ {"id": 11, "embedding": [0.0, 0.0, 0.0, 1.1]},
+ {"id": 12, "embedding": [0.0, 0.0, 0.0, 1.2]},
+ {"id": 13, "embedding": [0.0, 0.0, 0.0, 1.3]},
+ {"id": 14, "embedding": [0.0, 0.0, 0.0, 1.4]},
+ {"id": 15, "embedding": [0.0, 0.0, 0.0, 1.5]},
+ {"id": 16, "embedding": [10.0, 10.0, 10.0, 0.1]},
+ {"id": 17, "embedding": [10.0, 10.0, 10.0, 0.2]},
+ {"id": 18, "embedding": [10.0, 10.0, 10.0, 0.3]},
+ {"id": 19, "embedding": [10.0, 10.0, 10.0, 0.4]},
+ {"id": 20, "embedding": [10.0, 10.0, 10.0, 0.5]},
+ {"id": 21, "embedding": [10.0, 10.0, 10.0, 0.6]},
+ {"id": 22, "embedding": [10.0, 10.0, 10.0, 0.7]},
+ {"id": 23, "embedding": [10.0, 10.0, 10.0, 0.8]},
+ {"id": 24, "embedding": [10.0, 10.0, 10.0, 0.9]},
+ {"id": 25, "embedding": [10.0, 10.0, 10.0, 1.0]},
+ {"id": 26, "embedding": [10.0, 10.0, 10.0, 1.1]},
+ {"id": 27, "embedding": [10.0, 10.0, 10.0, 1.2]},
+ {"id": 28, "embedding": [10.0, 10.0, 10.0, 1.3]},
+ {"id": 29, "embedding": [10.0, 10.0, 10.0, 1.4]},
+ {"id": 30, "embedding": [10.0, 10.0, 10.0, 1.5]}
+]);
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree/create-index-vtree.3.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree/create-index-vtree.3.ddl.sqlpp
new file mode 100644
index 0000000000..9f3b1a4884
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree/create-index-vtree.3.ddl.sqlpp
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// A sample index is required before creating a vector index (quantization training
+// scans it in Job 1). Fixed sample-seed keeps the sample deterministic.
+ANALYZE DATASET test.MovieSmall WITH {"sample-seed": 1000};
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree/create-index-vtree.4.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree/create-index-vtree.4.ddl.sqlpp
new file mode 100644
index 0000000000..17ea81c6a9
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree/create-index-vtree.4.ddl.sqlpp
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+// Pin the k-means training RNG so index creation is reproducible run-to-run.
+SET `compiler.vector.trainseed` "42";
+
+CREATE INDEX idx_emb
+ ON MovieSmall(embedding VECTOR)
+ TYPE VTREE
+ WITH { "dimension": 4, "similarity": "euclidean", "num_clusters": 2, "train_list_fraction": 1.0 }
+ EXCLUDE UNKNOWN KEY;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree/create-index-vtree.5.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree/create-index-vtree.5.query.sqlpp
new file mode 100644
index 0000000000..d4efd5b4ee
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree/create-index-vtree.5.query.sqlpp
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// Index registered in metadata and out of pending state.
+SELECT VALUE i.IndexStructure
+FROM Metadata.`Index` i
+WHERE i.DataverseName = "test" AND i.DatasetName = "MovieSmall" AND i.IndexName = "idx_emb";
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree/create-index-vtree.6.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree/create-index-vtree.6.query.sqlpp
new file mode 100644
index 0000000000..1e653c1ce8
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree/create-index-vtree.6.query.sqlpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+// Exact top-3: min_probe_fraction 1.0 scans every cluster, so the expected result is
+// independent of how k-means happened to place the centroids.
+SELECT VALUE m.id
+FROM MovieSmall m
+ORDER BY ann_distance(m.embedding, [0.0, 0.0, 0.0, 0.0], "euclidean", 1.0)
+LIMIT 3;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree/create-index-vtree.7.plans.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree/create-index-vtree.7.plans.sqlpp
new file mode 100644
index 0000000000..ddce8deda0
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree/create-index-vtree.7.plans.sqlpp
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+-- param compile-only:string=true
+-- param optimized-logical-plan:string=true
+
+// The optimized plan must reference idx_emb: proves IntroduceTopKAccessMethodRule
+// rewrote the scan into a vector index search (a fallback scan plan never names the index).
+SELECT VALUE m.id
+FROM test.MovieSmall m
+ORDER BY ann_distance(m.embedding, [0.0, 0.0, 0.0, 0.0], "euclidean", 1.0)
+LIMIT 3;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree/create-index-vtree.8.plans.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree/create-index-vtree.8.plans.sqlpp
new file mode 100644
index 0000000000..3360585a6f
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree/create-index-vtree.8.plans.sqlpp
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+-- param compile-only:string=true
+-- param optimized-logical-plan:string=true
+
+// A DESC order-by asks for the farthest K, but the top-K cursor returns the nearest K. So
+// IntroduceTopKAccessMethodRule must NOT rewrite this into a vector index search: the optimized
+// plan falls back to a full scan and therefore does not name idx_emb. (Contrast with step 7, ASC.)
+SELECT VALUE m.id
+FROM test.MovieSmall m
+ORDER BY ann_distance(m.embedding, [0.0, 0.0, 0.0, 0.0], "euclidean", 1.0) DESC
+LIMIT 3;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree/create-index-vtree.9.plans.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree/create-index-vtree.9.plans.sqlpp
new file mode 100644
index 0000000000..aafc7206c3
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree/create-index-vtree.9.plans.sqlpp
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+-- param compile-only:string=true
+-- param optimized-logical-plan:string=true
+
+// Discriminator check: vector_distance is EXACT KNN. It desugars to the same distance builtin as
+// ann_distance but carries NO ANN hint, so IntroduceTopKAccessMethodRule must NOT rewrite it into a
+// vector index search. The optimized plan therefore does not name idx_emb (contrast step 7, ann_distance).
+SELECT VALUE m.id
+FROM test.MovieSmall m
+ORDER BY vector_distance(m.embedding, [0.0, 0.0, 0.0, 0.0], "euclidean")
+LIMIT 3;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree/create-index-vtree.99.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree/create-index-vtree.99.ddl.sqlpp
new file mode 100644
index 0000000000..20dc6fde5e
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree/create-index-vtree.99.ddl.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+DROP DATAVERSE test IF EXISTS;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/soft-keyword-identifiers/soft-keyword-identifiers.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/soft-keyword-identifiers/soft-keyword-identifiers.1.ddl.sqlpp
new file mode 100644
index 0000000000..eaddbab223
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/soft-keyword-identifiers/soft-keyword-identifiers.1.ddl.sqlpp
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// ASTERIXDB-3771 review (Ian Maxon): 'vector' and 'vtree' are soft keywords, so they must be usable as
+// ordinary identifiers. This declares fields literally named 'vector' and 'vtree'; before the soft-keyword
+// change these were reserved tokens and this DDL failed to parse.
+DROP DATAVERSE test IF EXISTS;
+CREATE DATAVERSE test;
+USE test;
+
+CREATE TYPE T AS {
+ id: int,
+ vector: bigint,
+ vtree: string
+};
+
+CREATE DATASET ds(T) PRIMARY KEY id;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/soft-keyword-identifiers/soft-keyword-identifiers.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/soft-keyword-identifiers/soft-keyword-identifiers.2.update.sqlpp
new file mode 100644
index 0000000000..f09e45d10e
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/soft-keyword-identifiers/soft-keyword-identifiers.2.update.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+INSERT INTO ds ([
+ {"id": 1, "vector": 10, "vtree": "keep"},
+ {"id": 2, "vector": 20, "vtree": "skip"},
+ {"id": 3, "vector": 30, "vtree": "keep"}
+]);
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/soft-keyword-identifiers/soft-keyword-identifiers.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/soft-keyword-identifiers/soft-keyword-identifiers.3.query.sqlpp
new file mode 100644
index 0000000000..992dd05239
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/soft-keyword-identifiers/soft-keyword-identifiers.3.query.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+// 'vector' as a range variable, 'vector'/'vtree' as field names - all legal now that they are soft keywords.
+SELECT VALUE vector.vector
+FROM ds AS vector
+WHERE vector.vtree = "keep"
+ORDER BY vector.id;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/soft-keyword-identifiers/soft-keyword-identifiers.99.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/soft-keyword-identifiers/soft-keyword-identifiers.99.ddl.sqlpp
new file mode 100644
index 0000000000..20dc6fde5e
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/soft-keyword-identifiers/soft-keyword-identifiers.99.ddl.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+DROP DATAVERSE test IF EXISTS;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/create-index-vtree-composite-pk/create-index-vtree-composite-pk.5.regex b/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/create-index-vtree-composite-pk/create-index-vtree-composite-pk.5.regex
new file mode 100644
index 0000000000..ee8b6e5e82
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/create-index-vtree-composite-pk/create-index-vtree-composite-pk.5.regex
@@ -0,0 +1 @@
+/(?s).*index-search\("idx_cpk_emb"[^\n]*condition \(gt\(\$\$\d+, 2000\)\).*/
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/create-index-vtree-composite-pk/create-index-vtree-composite-pk.6.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/create-index-vtree-composite-pk/create-index-vtree-composite-pk.6.adm
new file mode 100644
index 0000000000..ae58c0baed
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/create-index-vtree-composite-pk/create-index-vtree-composite-pk.6.adm
@@ -0,0 +1,3 @@
+[ 1, 11, "t1_11" ]
+[ 1, 12, "t1_12" ]
+[ 1, 13, "t1_13" ]
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/create-index-vtree-glove/create-index-vtree-glove.5.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/create-index-vtree-glove/create-index-vtree-glove.5.adm
new file mode 100644
index 0000000000..c0694de8f4
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/create-index-vtree-glove/create-index-vtree-glove.5.adm
@@ -0,0 +1,5 @@
+"g9"
+"g145"
+"g53"
+"g141"
+"g29"
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/create-index-vtree-include-filter/create-index-vtree-include-filter.5.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/create-index-vtree-include-filter/create-index-vtree-include-filter.5.adm
new file mode 100644
index 0000000000..21245e8a9d
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/create-index-vtree-include-filter/create-index-vtree-include-filter.5.adm
@@ -0,0 +1,3 @@
+6
+7
+8
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/create-index-vtree-include-nested/create-index-vtree-include-nested.5.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/create-index-vtree-include-nested/create-index-vtree-include-nested.5.adm
new file mode 100644
index 0000000000..1b854b7c8e
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/create-index-vtree-include-nested/create-index-vtree-include-nested.5.adm
@@ -0,0 +1 @@
+[ [ "info", "year" ] ]
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/create-index-vtree-include-nested/create-index-vtree-include-nested.6.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/create-index-vtree-include-nested/create-index-vtree-include-nested.6.adm
new file mode 100644
index 0000000000..11ce57c69c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/create-index-vtree-include-nested/create-index-vtree-include-nested.6.adm
@@ -0,0 +1,3 @@
+6
+7
+8
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/create-index-vtree-movie/create-index-vtree-movie.5.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/create-index-vtree-movie/create-index-vtree-movie.5.adm
new file mode 100644
index 0000000000..beafec714a
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/create-index-vtree-movie/create-index-vtree-movie.5.adm
@@ -0,0 +1,5 @@
+"Pulp Fiction"
+"The Wolf of Wall Street"
+"The Dark Knight"
+"Joker"
+"Batman Begins"
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/create-index-vtree-nested/create-index-vtree-nested.5.regex b/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/create-index-vtree-nested/create-index-vtree-nested.5.regex
new file mode 100644
index 0000000000..2641d57116
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/create-index-vtree-nested/create-index-vtree-nested.5.regex
@@ -0,0 +1 @@
+/(?s).*idx_emb.*/
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/create-index-vtree-open-undeclared/create-index-vtree-open-undeclared.5.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/create-index-vtree-open-undeclared/create-index-vtree-open-undeclared.5.adm
new file mode 100644
index 0000000000..01e79c32a8
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/create-index-vtree-open-undeclared/create-index-vtree-open-undeclared.5.adm
@@ -0,0 +1,3 @@
+1
+2
+3
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/create-index-vtree-open-undeclared/create-index-vtree-open-undeclared.6.regex b/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/create-index-vtree-open-undeclared/create-index-vtree-open-undeclared.6.regex
new file mode 100644
index 0000000000..2641d57116
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/create-index-vtree-open-undeclared/create-index-vtree-open-undeclared.6.regex
@@ -0,0 +1 @@
+/(?s).*idx_emb.*/
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/create-index-vtree-similarity-normalize/create-index-vtree-similarity-normalize.5.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/create-index-vtree-similarity-normalize/create-index-vtree-similarity-normalize.5.adm
new file mode 100644
index 0000000000..6d0533dcc7
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/create-index-vtree-similarity-normalize/create-index-vtree-similarity-normalize.5.adm
@@ -0,0 +1 @@
+"euclidean"
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/create-index-vtree/create-index-vtree.5.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/create-index-vtree/create-index-vtree.5.adm
new file mode 100644
index 0000000000..da73414b0e
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/create-index-vtree/create-index-vtree.5.adm
@@ -0,0 +1 @@
+"VTREE"
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/create-index-vtree/create-index-vtree.6.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/create-index-vtree/create-index-vtree.6.adm
new file mode 100644
index 0000000000..01e79c32a8
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/create-index-vtree/create-index-vtree.6.adm
@@ -0,0 +1,3 @@
+1
+2
+3
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/create-index-vtree/create-index-vtree.7.regex b/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/create-index-vtree/create-index-vtree.7.regex
new file mode 100644
index 0000000000..2641d57116
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/create-index-vtree/create-index-vtree.7.regex
@@ -0,0 +1 @@
+/(?s).*idx_emb.*/
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/create-index-vtree/create-index-vtree.8.regex b/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/create-index-vtree/create-index-vtree.8.regex
new file mode 100644
index 0000000000..538f5078bb
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/create-index-vtree/create-index-vtree.8.regex
@@ -0,0 +1 @@
+/(?s)^(?!.*idx_emb).*$/
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/create-index-vtree/create-index-vtree.9.regex b/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/create-index-vtree/create-index-vtree.9.regex
new file mode 100644
index 0000000000..538f5078bb
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/create-index-vtree/create-index-vtree.9.regex
@@ -0,0 +1 @@
+/(?s)^(?!.*idx_emb).*$/
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/soft-keyword-identifiers/soft-keyword-identifiers.3.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/soft-keyword-identifiers/soft-keyword-identifiers.3.adm
new file mode 100644
index 0000000000..3d1851cceb
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/soft-keyword-identifiers/soft-keyword-identifiers.3.adm
@@ -0,0 +1,2 @@
+10
+30
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/annotations/AnnSearchPreferenceAnnotation.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/annotations/AnnSearchPreferenceAnnotation.java
new file mode 100644
index 0000000000..149f32b009
--- /dev/null
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/annotations/AnnSearchPreferenceAnnotation.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.common.annotations;
+
+import org.apache.hyracks.algebricks.core.algebra.expressions.IExpressionAnnotation;
+
+/**
+ * Marks a vector-distance function call as an approximate-nearest-neighbor (ANN) request, i.e. the desugared
+ * form of {@code ann_distance(...)}. Its presence tells {@code IntroduceTopKAccessMethodRule} that the call is
+ * eligible to be served by a vector index (a plain {@code vector_distance} call carries no such annotation and
+ * stays exact). The annotation also carries the ANN-only search parameters that have no place on the 2-arg
+ * distance builtin the call is rewritten into.
+ */
+public final class AnnSearchPreferenceAnnotation implements IExpressionAnnotation {
+
+ private final String metric;
+ private final double minProbeFraction;
+ private final int kMultiplier;
+
+ public AnnSearchPreferenceAnnotation(String metric, double minProbeFraction, int kMultiplier) {
+ this.metric = metric;
+ this.minProbeFraction = minProbeFraction;
+ this.kMultiplier = kMultiplier;
+ }
+
+ /** The distance metric as written in the query (used at compile time for index selection). */
+ public String getMetric() {
+ return metric;
+ }
+
+ /** Fraction of clusters to probe during the ANN search; default 0.1. */
+ public double getMinProbeFraction() {
+ return minProbeFraction;
+ }
+
+ /** Over-fetch factor applied to k during the ANN search; default 1. */
+ public int getKMultiplier() {
+ return kMultiplier;
+ }
+
+ @Override
+ public String toString() {
+ return String.format("ann-search:%s,%f,%d", metric, minProbeFraction, kMultiplier);
+ }
+}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
index 16e2e1f01d..c458fe1c84 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
@@ -302,6 +302,7 @@ public class CompilerProperties extends AbstractProperties {
public static final String COMPILER_DELTALAKE_FILESPLITS_KEY = Option.COMPILER_DELTALAKE_FILESPLITS.ini();
public static final String COMPILER_PARQUET_FILESPLITS_KEY = Option.COMPILER_PARQUET_FILESPLITS.ini();
public static final String COMPILER_HDFS_SPLIT_PARALLELISM_KEY = Option.COMPILER_HDFS_SPLIT_PARALLELISM.ini();
+ public static final String COMPILER_VECTOR_K_MULTIPLIER_KEY = "compiler.vector.kmultiplier";
// Seeds the k-means training RNG of vector index creation; set per request for deterministic tests.
public static final String COMPILER_VECTOR_TRAINSEED_KEY = "compiler.vector.trainseed";
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/DatasetLocalResource.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/DatasetLocalResource.java
index 58d5f9b557..3bf52a2a59 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/DatasetLocalResource.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/DatasetLocalResource.java
@@ -24,6 +24,7 @@ import org.apache.hyracks.api.io.IJsonSerializable;
import org.apache.hyracks.api.io.IPersistedResourceRegistry;
import org.apache.hyracks.storage.common.IIndex;
import org.apache.hyracks.storage.common.IResource;
+import org.apache.hyracks.storage.common.IResourceWrapper;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
@@ -31,7 +32,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode;
/**
* A local resource with a dataset id and an assigned partition
*/
-public class DatasetLocalResource implements IResource {
+public class DatasetLocalResource implements IResource, IResourceWrapper {
private static final long serialVersionUID = 1L;
/**
@@ -72,6 +73,7 @@ public class DatasetLocalResource implements IResource {
this.partition = partition;
}
+ @Override
public IResource getResource() {
return resource;
}
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CreateIndexStatement.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CreateIndexStatement.java
index 1d9e5ab7d9..72bbeded0b 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CreateIndexStatement.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CreateIndexStatement.java
@@ -31,8 +31,11 @@ import org.apache.asterix.common.metadata.Namespace;
import org.apache.asterix.lang.common.base.AbstractStatement;
import org.apache.asterix.lang.common.base.Statement;
import org.apache.asterix.lang.common.expression.IndexedTypeExpression;
+import org.apache.asterix.lang.common.expression.RecordConstructor;
import org.apache.asterix.lang.common.struct.Identifier;
+import org.apache.asterix.lang.common.util.VectorIndexDeclUtil;
import org.apache.asterix.lang.common.visitor.base.ILangVisitor;
+import org.apache.asterix.object.base.AdmObjectNode;
import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.algebricks.common.utils.Triple;
import org.apache.hyracks.api.exceptions.SourceLocation;
@@ -55,9 +58,14 @@ public class CreateIndexStatement extends AbstractStatement {
private final OptionalBoolean castDefaultNull;
private final Map<String, String> castConfig;
+ private final List<CreateIndexStatement.IndexedElement> includeElements;
+ private final AdmObjectNode withObjectNode;
+
public CreateIndexStatement(Namespace namespace, Identifier datasetName, Identifier indexName, IndexType indexType,
List<IndexedElement> indexedElements, boolean enforced, int gramLength, String fullTextConfigName,
- boolean ifNotExists, Boolean excludeUnknownKey, Boolean castDefaultNull, Map<String, String> castConfig) {
+ boolean ifNotExists, Boolean excludeUnknownKey, Boolean castDefaultNull, Map<String, String> castConfig,
+ List<CreateIndexStatement.IndexedElement> includeElements, RecordConstructor withObjectRecord)
+ throws CompilationException {
this.namespace = namespace;
this.datasetName = Objects.requireNonNull(datasetName);
this.indexName = Objects.requireNonNull(indexName);
@@ -70,6 +78,8 @@ public class CreateIndexStatement extends AbstractStatement {
this.excludeUnknownKey = OptionalBoolean.ofNullable(excludeUnknownKey);
this.castDefaultNull = OptionalBoolean.ofNullable(castDefaultNull);
this.castConfig = castConfig == null ? Collections.emptyMap() : castConfig;
+ this.includeElements = includeElements;
+ this.withObjectNode = VectorIndexDeclUtil.validateAndGetWithObjectNode(withObjectRecord);
}
public String getFullTextConfigName() {
@@ -100,6 +110,10 @@ public class CreateIndexStatement extends AbstractStatement {
return indexedElements;
}
+ public List<IndexedElement> getIncludeElements() {
+ return includeElements;
+ }
+
public boolean isEnforced() {
return enforced;
}
@@ -132,6 +146,10 @@ public class CreateIndexStatement extends AbstractStatement {
return castConfig;
}
+ public AdmObjectNode getWithObjectNode() {
+ return withObjectNode;
+ }
+
@Override
public Kind getKind() {
return Statement.Kind.CREATE_INDEX;
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/CommonFunctionMapUtil.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/CommonFunctionMapUtil.java
index 8730cb8b61..08776c5bb7 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/CommonFunctionMapUtil.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/CommonFunctionMapUtil.java
@@ -168,9 +168,13 @@ public class CommonFunctionMapUtil {
// Vector functions — canonical SQL++ names: euclidean_distance, euclidean_squared_distance,
// cosine_similarity, dot_product, vector_distance (compile-time rewrite to concrete metric)
+ addFunctionMapping("approx_vector_distance", "ann-distance");
// Compatibility aliases
addFunctionMapping("l2_distance", "euclidean-distance");
addFunctionMapping("l2_squared_distance", "euclidean-squared-distance");
+ // cosine-distance is a private builtin (use vector_distance / ann_distance for cosine), so there
+ // is deliberately no public cosine_distance alias — mapping one resolves to a private function.
+ addFunctionMapping("dot_product", "dot-product");
// OGC SFA geospatial function aliases.
// Many GIS dialects use CamelCase (e.g. ST_IsEmpty) which lowercases to
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java
index 8218be8442..96e7882f71 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java
@@ -273,8 +273,12 @@ public class FunctionUtil {
DatasetFullyQualifiedName datasetOrViewName = parseDatasetFunctionArguments(argList, 0,
datasetFn.getSourceLocation(), ExpressionUtils::getStringLiteral);
boolean isView = argList.size() > 3 && Boolean.TRUE.equals(ExpressionUtils.getBooleanLiteral(argList.get(3)));
- DatasetFullyQualifiedName synonymName = argList.size() > 4 ? parseDatasetFunctionArguments(argList, 4,
- datasetFn.getSourceLocation(), ExpressionUtils::getStringLiteral) : null;
+ Boolean isSample = argList.size() > 4 ? ExpressionUtils.getBooleanLiteral(argList.get(4)) : null;
+ DatasetFullyQualifiedName synonymName = null;
+ if (isSample == null || !isSample) {
+ synonymName = argList.size() > 4 ? parseDatasetFunctionArguments(argList, 4, datasetFn.getSourceLocation(),
+ ExpressionUtils::getStringLiteral) : null;
+ }
return new Triple<>(datasetOrViewName, isView, synonymName);
}
@@ -308,7 +312,7 @@ public class FunctionUtil {
return new DatasetFullyQualifiedName(databaseName, dataverseName, datasetName);
}
- private static String getStringConstant(Mutable<ILogicalExpression> arg) {
+ public static String getStringConstant(Mutable<ILogicalExpression> arg) {
return ConstantExpressionUtil.getStringConstant(arg.getValue());
}
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/VectorIndexDeclUtil.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/VectorIndexDeclUtil.java
new file mode 100644
index 0000000000..86659fa543
--- /dev/null
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/VectorIndexDeclUtil.java
@@ -0,0 +1,318 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.lang.common.util;
+
+import java.util.Arrays;
+import java.util.Locale;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.apache.asterix.common.exceptions.CompilationException;
+import org.apache.asterix.common.exceptions.ErrorCode;
+import org.apache.asterix.common.vector.VectorSimilarityMetric;
+import org.apache.asterix.lang.common.expression.RecordConstructor;
+import org.apache.asterix.object.base.AdmBigIntNode;
+import org.apache.asterix.object.base.AdmDoubleNode;
+import org.apache.asterix.object.base.AdmObjectNode;
+import org.apache.asterix.object.base.AdmStringNode;
+import org.apache.asterix.object.base.IAdmNode;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.hyracks.api.exceptions.SourceLocation;
+
+/**
+ * Validates the {@code WITH} clause of a {@code CREATE INDEX ... TYPE VTREE} statement and
+ * produces the canonical {@link AdmObjectNode} stored on the {@code CreateIndexStatement}.
+ */
+public class VectorIndexDeclUtil {
+
+ public static final String VECTOR_INDEX_PARAMETER_DIMENSION = "dimension";
+ public static final String VECTOR_INDEX_PARAMETER_QUANTIZATION = "quantization";
+ public static final String VECTOR_INDEX_PARAMETER_TRAIN_LIST_FRACTION = "train_list_fraction";
+ public static final String VECTOR_INDEX_PARAMETER_SIMILARITY = "similarity";
+ public static final String VECTOR_INDEX_PARAMETER_NUM_K = "num_clusters";
+ public static final String VECTOR_INDEX_PARAMETER_EPSILON = "epsilon";
+ /**
+ * Cross-pollination factor: each record is written into the M closest leaf centroids at bulk-load,
+ * not just the closest one. M=1 reproduces the legacy (no cross-pollination) behavior.
+ */
+ public static final String VECTOR_INDEX_PARAMETER_CROSS_POLLINATION_M = "cross_pollination_m";
+ /**
+ * RNG (relative neighborhood graph) factor for SPTAG-style cross-pollination diversity. A candidate
+ * centroid {@code c_i} is rejected iff some already-accepted replica {@code r} satisfies
+ * {@code rng_factor * dist(c_i, r) < dist(x, c_i)}. The canonical SPTAG rule is {@code 1.0}; larger
+ * values loosen the rule (more replicas survive), smaller values tighten. A non-finite value
+ * effectively disables RNG, degrading to the pure top-{@code cross_pollination_m} slice.
+ */
+ public static final String VECTOR_INDEX_PARAMETER_RNG_FACTOR = "rng_factor";
+ public static final String VECTOR_INDEX_DEFAULT_QUANTIZATION = "SQ8";
+ /** Default for level-wise centroid search and ANN search predicate (matches VectorSearchPredicate). */
+ public static final double VECTOR_INDEX_DEFAULT_TRAIN_LIST = 0.1;
+ public static final double VECTOR_INDEX_DEFAULT_EPSILON = 0.25;
+ public static final long VECTOR_INDEX_DEFAULT_CROSS_POLLINATION_M = 1L;
+ /** Sanity cap on cross_pollination_m: anything larger almost certainly indicates a user error. */
+ public static final long VECTOR_INDEX_MAX_CROSS_POLLINATION_M = 1024L;
+ /** Default RNG factor: canonical SPTAG rule. */
+ public static final double VECTOR_INDEX_DEFAULT_RNG_FACTOR = 1.0;
+
+ /**
+ * Human-readable list of the accepted {@code similarity} values, derived from
+ * {@link VectorSimilarityMetric} so it never drifts from the actual set of recognized metrics.
+ */
+ private static final String ALLOWED_SIMILARITY_VALUES = Arrays.stream(VectorSimilarityMetric.values())
+ .map(m -> m.canonical().toUpperCase(Locale.ROOT)).collect(Collectors.joining(", "));
+
+ private static final Set<String> ALLOWED_VECTOR_INDEX_QUANTIZATION = Set.of("SQ4", "SQ8");
+
+ /**
+ * Only these keys may appear in the vector index {@code WITH} clause (unknown keys are a compile error).
+ */
+ private static final Set<String> ALLOWED_VECTOR_INDEX_WITH_FIELDS = Set.of(VECTOR_INDEX_PARAMETER_DIMENSION,
+ VECTOR_INDEX_PARAMETER_SIMILARITY, VECTOR_INDEX_PARAMETER_TRAIN_LIST_FRACTION,
+ VECTOR_INDEX_PARAMETER_QUANTIZATION, VECTOR_INDEX_PARAMETER_EPSILON, VECTOR_INDEX_PARAMETER_NUM_K,
+ VECTOR_INDEX_PARAMETER_CROSS_POLLINATION_M, VECTOR_INDEX_PARAMETER_RNG_FACTOR);
+
+ private VectorIndexDeclUtil() {
+ }
+
+ public static AdmObjectNode validateAndGetWithObjectNode(RecordConstructor withRecord) throws CompilationException {
+ if (withRecord == null) {
+ return null;
+ }
+ return validateAndGetWithObjectNode(withRecord, withRecord.getSourceLocation());
+ }
+
+ public static AdmObjectNode validateAndGetWithObjectNode(RecordConstructor withRecord, SourceLocation sourceLoc)
+ throws CompilationException {
+ if (withRecord == null) {
+ return null;
+ }
+ final AdmObjectNode node = ExpressionUtils.toNode(withRecord);
+
+ validateWithClauseFieldNames(node);
+ validateDimension(node);
+ validateTrainList(node);
+ validateSimilarity(node);
+ validateQuantization(node);
+ validateEpsilon(node);
+ validateNumClusters(node);
+ validateCrossPollinationM(node);
+ validateRngFactor(node);
+
+ return node;
+ }
+
+ private static void validateWithClauseFieldNames(AdmObjectNode node) throws CompilationException {
+ for (String name : node.getFieldNames()) {
+ if (!ALLOWED_VECTOR_INDEX_WITH_FIELDS.contains(name)) {
+ throw new CompilationException("Failed to create vector index. Unknown field `" + name
+ + "` in WITH clause. Allowed fields: dimension, similarity, train_list_fraction, quantization, "
+ + "epsilon, num_clusters, cross_pollination_m, rng_factor");
+ }
+ }
+ }
+
+ private static void validateDimension(AdmObjectNode node) throws CompilationException {
+ IAdmNode dimNode = node.get(VECTOR_INDEX_PARAMETER_DIMENSION);
+ if (dimNode == null) {
+ throw new CompilationException(ErrorCode.COMPILATION_VECTOR_INDEX_CREATION_FAILED,
+ "Missing required parameter `dimension` in WITH clause.");
+ }
+ long dimValue;
+ switch (dimNode.getType()) {
+ case BIGINT:
+ long lv = ((AdmBigIntNode) dimNode).get();
+ if (lv <= 0 || lv > Integer.MAX_VALUE) {
+ throw new CompilationException(ErrorCode.COMPILATION_VECTOR_INDEX_CREATION_FAILED,
+ "Invalid `dimension` parameter value. It must be an integer greater than 0");
+ }
+ dimValue = lv;
+ break;
+ default:
+ throw new CompilationException(ErrorCode.COMPILATION_VECTOR_INDEX_CREATION_FAILED,
+ "Invalid `dimension` parameter value. It must be an integer greater than 0");
+ }
+ node.remove(VECTOR_INDEX_PARAMETER_DIMENSION);
+ node.set(VECTOR_INDEX_PARAMETER_DIMENSION, new AdmBigIntNode(dimValue));
+ }
+
+ private static void validateSimilarity(AdmObjectNode node) throws CompilationException {
+ IAdmNode simNode = node.get(VECTOR_INDEX_PARAMETER_SIMILARITY);
+ if (simNode == null) {
+ throw new CompilationException(ErrorCode.COMPILATION_VECTOR_INDEX_CREATION_FAILED,
+ "Missing required parameter `similarity` in WITH clause.");
+ }
+ if (simNode.getType() != ATypeTag.STRING) {
+ throw new CompilationException(ErrorCode.COMPILATION_VECTOR_INDEX_CREATION_FAILED,
+ "Invalid `similarity` parameter value. " + "Allowed values: " + ALLOWED_SIMILARITY_VALUES);
+ }
+ // Validate against the single source of truth for metric aliases and normalize the stored value to
+ // the canonical spelling, so the persisted `similarity` does not depend on the alias/casing written.
+ String similarity = ((AdmStringNode) simNode).get();
+ VectorSimilarityMetric metric = VectorSimilarityMetric.fromAlias(similarity);
+ if (metric == null) {
+ throw new CompilationException(ErrorCode.COMPILATION_VECTOR_INDEX_CREATION_FAILED,
+ "Invalid `similarity` parameter value. " + "Allowed values: " + ALLOWED_SIMILARITY_VALUES);
+ }
+ node.update(VECTOR_INDEX_PARAMETER_SIMILARITY, new AdmStringNode(metric.canonical()));
+ }
+
+ private static void validateQuantization(AdmObjectNode node) throws CompilationException {
+ IAdmNode qNode = node.get(VECTOR_INDEX_PARAMETER_QUANTIZATION);
+ if (qNode == null) {
+ node.set(VECTOR_INDEX_PARAMETER_QUANTIZATION, new AdmStringNode(VECTOR_INDEX_DEFAULT_QUANTIZATION));
+ return;
+ }
+ switch (qNode.getType()) {
+ case STRING:
+ String quantization = ((AdmStringNode) qNode).get();
+ String qNorm = quantization.trim().toUpperCase(Locale.ROOT);
+ if (!ALLOWED_VECTOR_INDEX_QUANTIZATION.contains(qNorm)) {
+ throw new CompilationException(ErrorCode.COMPILATION_VECTOR_INDEX_CREATION_FAILED,
+ "Invalid `quantization` parameter value. Allowed values: SQ4 and SQ8");
+ } else {
+ node.set(VECTOR_INDEX_PARAMETER_QUANTIZATION, new AdmStringNode(qNorm));
+ }
+ break;
+ default:
+ throw new CompilationException(ErrorCode.COMPILATION_VECTOR_INDEX_CREATION_FAILED,
+ "Invalid `quantization` parameter value. Allowed values: SQ4 and SQ8");
+ }
+ }
+
+ /**
+ * Training list size is specified only via {@code train_list_fraction} (with ANALYZE/cardinality at build time).
+ */
+ private static void validateTrainList(AdmObjectNode node) throws CompilationException {
+ IAdmNode fn = node.get(VECTOR_INDEX_PARAMETER_TRAIN_LIST_FRACTION);
+ if (fn == null) {
+ node.set(VECTOR_INDEX_PARAMETER_TRAIN_LIST_FRACTION, new AdmDoubleNode(VECTOR_INDEX_DEFAULT_TRAIN_LIST));
+ return;
+ }
+ double trainListFractionValue = parseDoubleOrBigInt(fn,
+ "Invalid `train_list_fraction` parameter value. It must be in the range of (0,1]");
+ if (trainListFractionValue <= 0 || trainListFractionValue > 1) {
+ throw new CompilationException(ErrorCode.COMPILATION_VECTOR_INDEX_CREATION_FAILED,
+ "Invalid `train_list_fraction` parameter value. It must be in the range of (0,1]");
+ }
+ if (fn.getType() == ATypeTag.BIGINT) {
+ node.remove(VECTOR_INDEX_PARAMETER_TRAIN_LIST_FRACTION);
+ node.set(VECTOR_INDEX_PARAMETER_TRAIN_LIST_FRACTION, new AdmDoubleNode(trainListFractionValue));
+ }
+ }
+
+ private static void validateEpsilon(AdmObjectNode node) throws CompilationException {
+ IAdmNode epsNode = node.get(VECTOR_INDEX_PARAMETER_EPSILON);
+ if (epsNode == null) {
+ node.set(VECTOR_INDEX_PARAMETER_EPSILON, new AdmDoubleNode(VECTOR_INDEX_DEFAULT_EPSILON));
+ return;
+ }
+ double v = parseDoubleOrBigInt(epsNode, "Invalid `epsilon` parameter value. It must be in the range of [0,1]");
+ if (v < 0 || v > 1) {
+ throw new CompilationException(ErrorCode.COMPILATION_VECTOR_INDEX_CREATION_FAILED,
+ "Invalid `epsilon` parameter value. It must be in the range of [0,1]");
+ }
+ if (epsNode.getType() == ATypeTag.BIGINT) {
+ node.remove(VECTOR_INDEX_PARAMETER_EPSILON);
+ node.set(VECTOR_INDEX_PARAMETER_EPSILON, new AdmDoubleNode(v));
+ }
+ }
+
+ /**
+ * Validates {@code cross_pollination_m}: each record is written into the M closest leaf centroids at
+ * bulk-load (M=1 means no cross-pollination, matching legacy behavior). Must be a positive integer;
+ * capped at {@link #VECTOR_INDEX_MAX_CROSS_POLLINATION_M} as a sanity check.
+ */
+ private static void validateCrossPollinationM(AdmObjectNode node) throws CompilationException {
+ IAdmNode mNode = node.get(VECTOR_INDEX_PARAMETER_CROSS_POLLINATION_M);
+ if (mNode == null) {
+ node.set(VECTOR_INDEX_PARAMETER_CROSS_POLLINATION_M,
+ new AdmBigIntNode(VECTOR_INDEX_DEFAULT_CROSS_POLLINATION_M));
+ return;
+ }
+ if (mNode.getType() != ATypeTag.BIGINT) {
+ throw new CompilationException(ErrorCode.COMPILATION_VECTOR_INDEX_CREATION_FAILED,
+ "Invalid `cross_pollination_m` parameter " + "value. It must be an integer in [1, "
+ + VECTOR_INDEX_MAX_CROSS_POLLINATION_M + "]");
+ }
+ long v = ((AdmBigIntNode) mNode).get();
+ if (v < 1 || v > VECTOR_INDEX_MAX_CROSS_POLLINATION_M) {
+ throw new CompilationException(ErrorCode.COMPILATION_VECTOR_INDEX_CREATION_FAILED,
+ "Invalid `cross_pollination_m` parameter " + "value. It must be an integer in [1, "
+ + VECTOR_INDEX_MAX_CROSS_POLLINATION_M + "]");
+ }
+ }
+
+ /**
+ * Validates {@code num_clusters}: the target number of leaf clusters per storage partition. Optional —
+ * when absent the builder derives it as sqrt(cardinality / numPartitions), so no default is injected
+ * here. When present it must be a positive BIGINT that fits in an int (it is later read via
+ * {@code getOptionalInt}); a wrong-typed or non-positive value previously slipped past DDL and either
+ * threw a ClassCastException during metadata serialization or was silently dropped.
+ */
+ private static void validateNumClusters(AdmObjectNode node) throws CompilationException {
+ IAdmNode kNode = node.get(VECTOR_INDEX_PARAMETER_NUM_K);
+ if (kNode == null) {
+ return;
+ }
+ if (kNode.getType() != ATypeTag.BIGINT) {
+ throw new CompilationException(ErrorCode.COMPILATION_VECTOR_INDEX_CREATION_FAILED,
+ "Invalid `num_clusters` parameter value. " + "It must be an integer greater than 0.");
+ }
+ long v = ((AdmBigIntNode) kNode).get();
+ if (v < 1 || v > Integer.MAX_VALUE) {
+ throw new CompilationException(ErrorCode.COMPILATION_VECTOR_INDEX_CREATION_FAILED,
+ "Invalid `num_clusters` parameter value. " + "It must be an integer greater than 0.");
+ }
+ }
+
+ /**
+ * Validates {@code rng_factor}: SPTAG-style RNG acceptance multiplier applied at bulk-load after the
+ * eps-filtered candidate list (see {@code RngAcceptanceFilter}). Must be a positive finite double. A
+ * BIGINT literal is coerced to DOUBLE. To effectively disable RNG, use a value much larger than the
+ * maximum expected centroid-to-centroid distance.
+ */
+ private static void validateRngFactor(AdmObjectNode node) throws CompilationException {
+ IAdmNode rngNode = node.get(VECTOR_INDEX_PARAMETER_RNG_FACTOR);
+ if (rngNode == null) {
+ node.set(VECTOR_INDEX_PARAMETER_RNG_FACTOR, new AdmDoubleNode(VECTOR_INDEX_DEFAULT_RNG_FACTOR));
+ return;
+ }
+ double v = parseDoubleOrBigInt(rngNode,
+ "Invalid `rng_factor` parameter " + "value. It must be a positive finite number.");
+ if (!(v > 0.0) || !Double.isFinite(v)) {
+ throw new CompilationException(ErrorCode.COMPILATION_VECTOR_INDEX_CREATION_FAILED,
+ "Invalid `rng_factor` parameter value. " + "It must be a positive finite number.");
+ }
+ if (rngNode.getType() == ATypeTag.BIGINT) {
+ node.remove(VECTOR_INDEX_PARAMETER_RNG_FACTOR);
+ node.set(VECTOR_INDEX_PARAMETER_RNG_FACTOR, new AdmDoubleNode(v));
+ }
+ }
+
+ private static double parseDoubleOrBigInt(IAdmNode n, String errorMsg) throws CompilationException {
+ switch (n.getType()) {
+ case DOUBLE:
+ return ((AdmDoubleNode) n).get();
+ case BIGINT:
+ return ((AdmBigIntNode) n).get();
+ default:
+ throw new CompilationException(ErrorCode.COMPILATION_VECTOR_INDEX_CREATION_FAILED, errorMsg);
+ }
+ }
+}
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppVectorDistanceRewriteVisitor.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppVectorDistanceRewriteVisitor.java
index 4a62e95603..fe9f3c8b5f 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppVectorDistanceRewriteVisitor.java
+++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppVectorDistanceRewriteVisitor.java
@@ -23,12 +23,15 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
+import org.apache.asterix.common.annotations.AnnSearchPreferenceAnnotation;
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.common.functions.FunctionSignature;
import org.apache.asterix.lang.common.base.Expression;
import org.apache.asterix.lang.common.base.ILangExpression;
+import org.apache.asterix.lang.common.base.Literal;
import org.apache.asterix.lang.common.expression.CallExpr;
+import org.apache.asterix.lang.common.expression.LiteralExpr;
import org.apache.asterix.lang.common.util.ExpressionUtils;
import org.apache.asterix.lang.common.util.VectorDistanceMetric;
import org.apache.asterix.lang.sqlpp.visitor.base.AbstractSqlppSimpleExpressionVisitor;
@@ -52,6 +55,9 @@ public final class SqlppVectorDistanceRewriteVisitor extends AbstractSqlppSimple
if (isVectorDistanceCall(callExpr)) {
return rewriteVectorDistance(callExpr);
}
+ if (isAnnDistanceCall(callExpr)) {
+ return rewriteAnnDistance(callExpr);
+ }
return callExpr;
}
@@ -86,4 +92,79 @@ public final class SqlppVectorDistanceRewriteVisitor extends AbstractSqlppSimple
rewritten.setSourceLocation(callExpr.getSourceLocation());
return rewritten;
}
+
+ // ---- ann_distance ----------------------------------------------------------------------------
+ // ann_distance(vec1, vec2, metric [, min_probe_fraction] [, k_multiplier]) is desugared into the same
+ // 2-arg distance builtin as vector_distance, plus an AnnSearchPreferenceAnnotation that (a) marks the
+ // call as index-eligible for IntroduceTopKAccessMethodRule and (b) carries the ANN-only search
+ // parameters. Metric/parameter validation happens here at compile time.
+
+ private static final double DEFAULT_MIN_PROBE_FRACTION = 0.1;
+ private static final int DEFAULT_K_MULTIPLIER = 1;
+
+ private static boolean isAnnDistanceCall(CallExpr callExpr) {
+ FunctionSignature fs = callExpr.getFunctionSignature();
+ return fs != null && BuiltinFunctions.ANN_DISTANCE.getName().equals(fs.getName());
+ }
+
+ private static CallExpr rewriteAnnDistance(CallExpr callExpr) throws CompilationException {
+ List<Expression> args = callExpr.getExprList();
+ if (args.size() < 3 || args.size() > 5) {
+ throw new CompilationException(ErrorCode.COMPILATION_ERROR, callExpr.getSourceLocation(),
+ "ann_distance expects 3 to 5 arguments");
+ }
+ String metric = ExpressionUtils.getStringLiteral(args.get(2));
+ if (metric == null) {
+ throw new CompilationException(ErrorCode.COMPILATION_ERROR, args.get(2).getSourceLocation(),
+ "ann_distance metric must be a compile-time string literal");
+ }
+ Optional<String> builtinName = VectorDistanceMetric.resolve(metric);
+ if (builtinName.isEmpty()) {
+ throw new CompilationException(ErrorCode.COMPILATION_ERROR, args.get(2).getSourceLocation(),
+ "unknown ann_distance metric '" + metric + "'; supported metrics: "
+ + VectorDistanceMetric.supportedMetricsMessage());
+ }
+ double minProbeFraction = DEFAULT_MIN_PROBE_FRACTION;
+ if (args.size() > 3) {
+ minProbeFraction = numericLiteral(args.get(3), "min_probe_fraction");
+ if (minProbeFraction < 0.0 || minProbeFraction > 1.0) {
+ throw new CompilationException(ErrorCode.COMPILATION_ERROR, args.get(3).getSourceLocation(),
+ "ann_distance min_probe_fraction must be in [0, 1]");
+ }
+ }
+ int kMultiplier = DEFAULT_K_MULTIPLIER;
+ if (args.size() > 4) {
+ double km = numericLiteral(args.get(4), "k_multiplier");
+ if (km != Math.floor(km) || km < 1.0) {
+ throw new CompilationException(ErrorCode.COMPILATION_ERROR, args.get(4).getSourceLocation(),
+ "ann_distance k_multiplier must be a positive integer");
+ }
+ kMultiplier = (int) km;
+ }
+ List<Expression> targetArgs = new ArrayList<>(2);
+ targetArgs.add(args.get(0));
+ targetArgs.add(args.get(1));
+ CallExpr rewritten = new CallExpr(FunctionSignature.newAsterix(builtinName.get(), 2), targetArgs,
+ callExpr.getAggregateFilterExpr());
+ rewritten.setSourceLocation(callExpr.getSourceLocation());
+ rewritten.addHint(new AnnSearchPreferenceAnnotation(metric, minProbeFraction, kMultiplier));
+ return rewritten;
+ }
+
+ private static double numericLiteral(Expression arg, String paramName) throws CompilationException {
+ if (arg.getKind() == Expression.Kind.LITERAL_EXPRESSION) {
+ Literal lit = ((LiteralExpr) arg).getValue();
+ switch (lit.getLiteralType()) {
+ case DOUBLE:
+ case FLOAT:
+ case LONG:
+ case INTEGER:
+ return ((Number) lit.getValue()).doubleValue();
+ default:
+ break;
+ }
+ }
+ throw new CompilationException(ErrorCode.COMPILATION_ERROR, arg.getSourceLocation(),
+ "ann_distance " + paramName + " must be a compile-time numeric literal");
+ }
}
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
index 968cd0f015..6ad1db3e56 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
+++ b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
@@ -298,6 +298,8 @@ class SQLPPParser extends ScopeChecker implements IParser {
private static final String NAME = "NAME";
private static final String SNAPSHOT = "SNAPSHOT";
private static final String TIMESTAMP = "TIMESTAMP";
+ private static final String VECTOR = "VECTOR";
+ private static final String VTREE = "VTREE";
private static final String INT_TYPE_NAME = "int";
@@ -1579,6 +1581,8 @@ CreateIndexStatement IndexSpecification(Token startStmtToken) throws ParseExcept
Pair<Map<String, String>, Boolean> castConfigDefaultNull;
Boolean castDefaultNull = null;
Map<String, String> castConfig = null;
+ RecordConstructor withRecord = null;
+ List<CreateIndexStatement.IndexedElement> nonVectorFieldsList = new ArrayList<CreateIndexStatement.IndexedElement>();
}
{
(
@@ -1602,8 +1606,35 @@ CreateIndexStatement IndexSpecification(Token startStmtToken) throws ParseExcept
}
)*
<RIGHTPAREN>
+
+ // Parse INCLUDE clause for vector index additional fields (e.g., INCLUDE(title, popularity))
+ // Lookahead checks that INCLUDE is followed by LEFTPAREN to disambiguate from INCLUDE UNKNOWN KEY
+ ( LOOKAHEAD({laIdentifier(INCLUDE) && laToken(2, LEFTPAREN)}) <IDENTIFIER>
+ {
+ if (!isToken(INCLUDE)) {
+ throw createUnexpectedTokenError();
+ }
+ }
+ <LEFTPAREN> { startElementToken = token; }
+ indexedElement = IndexedElement(startElementToken) {
+ nonVectorFieldsList.add(indexedElement);
+ }
+ (<COMMA> { startElementToken = token; }
+ indexedElement = IndexedElement(startElementToken) {
+ nonVectorFieldsList.add(indexedElement);
+ }
+ )*
+ <RIGHTPAREN>
+ )?
+
( <TYPE> indexParams = IndexType() )? ( <ENFORCED> { enforced = true; } )?
- ( LOOKAHEAD({laIdentifier(EXCLUDE) || laIdentifier(INCLUDE)}) <IDENTIFIER>
+
+ // Parse WITH clause for vector index configuration
+ ( <WITH> withRecord = RecordConstructor() )?
+
+ // Parse INCLUDE/EXCLUDE UNKNOWN KEY
+ // Lookahead checks that INCLUDE/EXCLUDE is followed by UNKNOWN to disambiguate from INCLUDE(fields)
+ ( LOOKAHEAD({(laIdentifier(EXCLUDE) || laIdentifier(INCLUDE)) && laToken(2, UNKNOWN)}) <IDENTIFIER>
{
if (isToken(EXCLUDE)) {
excludeUnknown = true;
@@ -1623,22 +1654,102 @@ CreateIndexStatement IndexSpecification(Token startStmtToken) throws ParseExcept
)?
)
{
- IndexType indexType;
- int gramLength;
- if (indexParams != null) {
- indexType = indexParams.type;
- gramLength = indexParams.gramLength;
- fullTextConfigName = indexParams.fullTextConfig;
- } else {
- indexType = hasUnnest ? IndexType.ARRAY : IndexType.BTREE;
- gramLength = -1;
- fullTextConfigName = null;
+ // Check if VECTOR field type annotation is present
+ boolean hasVectorField = false;
+ if (indexedElementList.size() == 1) {
+ CreateIndexStatement.IndexedElement field = indexedElementList.get(0);
+ List<Pair<List<String>, IndexedTypeExpression>> projectList = field.getProjectList();
+ if (projectList != null && !projectList.isEmpty()) {
+ IndexedTypeExpression typeExpr = projectList.get(0).second;
+ if (typeExpr != null) {
+ TypeExpression innerType = typeExpr.getType();
+ if (innerType != null && innerType instanceof TypeReferenceExpression) {
+ TypeReferenceExpression typeRef = (TypeReferenceExpression) innerType;
+ Pair<Namespace, Identifier> ident = typeRef.getIdent();
+ if (ident != null && ident.second != null &&
+ "$$VECTOR_MARKER$$".equals(ident.second.getValue())) {
+ hasVectorField = true;
+ }
+ }
+ }
+ }
+ }
+
+ boolean isVTree = indexParams != null && indexParams.type == IndexType.VTREE;
+
+ // Validate: VECTOR annotation requires TYPE VTREE
+ if (hasVectorField && !isVTree) {
+ throw new SqlppParseException(getSourceLocation(startStmtToken),
+ "VECTOR field annotation requires TYPE VTREE. Expected syntax: CREATE INDEX ... (field VECTOR) TYPE VTREE WITH {...} EXCLUDE UNKNOWN KEY");
+ }
+
+ // Validate: TYPE VTREE requires VECTOR annotation
+ if (isVTree && !hasVectorField) {
+ throw new SqlppParseException(getSourceLocation(startStmtToken),
+ "TYPE VTREE requires VECTOR field annotation. Expected syntax: CREATE INDEX ... (field VECTOR) TYPE VTREE WITH {...} EXCLUDE UNKNOWN KEY");
+ }
+
+ if (isVTree) {
+ // Validate exactly one field for vector index
+ if (indexedElementList.size() != 1) {
+ throw new SqlppParseException(getSourceLocation(startStmtToken),
+ "Vector index requires exactly one field");
+ }
+
+ // Validate that WITH clause is provided
+ if (withRecord == null) {
+ throw new SqlppParseException(getSourceLocation(startStmtToken),
+ "Vector index requires WITH clause specifying index configuration");
+ }
+
+ // Validate that EXCLUDE UNKNOWN KEY is specified
+ if (excludeUnknown == null || !excludeUnknown) {
+ throw new SqlppParseException(getSourceLocation(startStmtToken),
+ "Vector index requires EXCLUDE UNKNOWN KEY");
+ }
+
+ // Create vector index statement
+ try {
+ CreateIndexStatement stmt = new CreateIndexStatement(
+ nameComponents.first, nameComponents.second,
+ new Identifier(indexName), IndexType.VTREE,
+ Collections.singletonList(indexedElementList.get(0)), // vectorField
+ false, 0, null, ifNotExists, null, null, null,
+ nonVectorFieldsList, // INCLUDE fields
+ withRecord);
+ return addSourceLocation(stmt, startStmtToken);
+ } catch (CompilationException e) {
+ throw new SqlppParseException(getSourceLocation(startStmtToken), e.getMessage());
+ }
+ }
+
+ // Validate: INCLUDE clause is only for vector indexes
+ if (!nonVectorFieldsList.isEmpty()) {
+ throw new SqlppParseException(getSourceLocation(startStmtToken),
+ "INCLUDE clause is only supported for vector indexes (TYPE VTREE)");
+ }
+
+ // Handle regular (non-vector) index creation
+ IndexType indexType;
+ int gramLength;
+ if (indexParams != null) {
+ indexType = indexParams.type;
+ gramLength = indexParams.gramLength;
+ fullTextConfigName = indexParams.fullTextConfig;
+ } else {
+ indexType = hasUnnest ? IndexType.ARRAY : IndexType.BTREE;
+ gramLength = -1;
+ fullTextConfigName = null;
+ }
+ try{
+ CreateIndexStatement stmt = new CreateIndexStatement(nameComponents.first, nameComponents.second,
+ new Identifier(indexName), indexType, indexedElementList, enforced, gramLength, fullTextConfigName, ifNotExists,
+ excludeUnknown, castDefaultNull, castConfig, Collections.emptyList(), null);
+ return addSourceLocation(stmt, startStmtToken);
+ } catch (CompilationException e) {
+ throw new SqlppParseException(getSourceLocation(startStmtToken), e.getMessage());
+ }
}
- CreateIndexStatement stmt = new CreateIndexStatement(nameComponents.first, nameComponents.second,
- new Identifier(indexName), indexType, indexedElementList, enforced, gramLength, fullTextConfigName, ifNotExists,
- excludeUnknown, castDefaultNull, castConfig);
- return addSourceLocation(stmt, startStmtToken);
- }
}
CreateIndexStatement.IndexedElement IndexedElement(Token startElementToken) throws ParseException:
@@ -1743,10 +1854,24 @@ Pair<List<String>, IndexedTypeExpression> IndexedField() throws ParseException:
{
Triple<List<String>, Token, Token> path = null;
IndexedTypeExpression type = null;
+ boolean isVectorType = false;
}
{
- path = MultipartIdentifier() ( <COLON> type = IndexedTypeExpr(true) )?
+ path = MultipartIdentifier()
+ (
+ <COLON> type = IndexedTypeExpr(true)
+ | LOOKAHEAD({laIdentifier(VECTOR)}) <IDENTIFIER> { isVectorType = true; }
+ )?
{
+ // For VECTOR type annotation, create a special marker TypeExpression
+ // This allows IndexSpecification to distinguish between:
+ // - "embedding" (no annotation) → type = null
+ // - "embedding VECTOR" → type = TypeExpression with typeRefKind = ATOMIC and typeName = "$$VECTOR_MARKER$$"
+ if (isVectorType) {
+ TypeExpression vectorMarker = new TypeReferenceExpression(
+ new Pair<Namespace, Identifier>(null, new Identifier("$$VECTOR_MARKER$$")));
+ type = new IndexedTypeExpression(vectorMarker, false);
+ }
return new Pair<List<String>, IndexedTypeExpression>(path.first, type);
}
}
@@ -1769,10 +1894,14 @@ CreateIndexStatement PrimaryIndexSpecification(Token startStmtToken) throws Pars
if (indexName == null) {
indexName = MetadataConstants.PRIMARY_INDEX_PREFIX + nameComponents.second;
}
- CreateIndexStatement stmt = new CreateIndexStatement(nameComponents.first, nameComponents.second,
+ try {
+ CreateIndexStatement stmt = new CreateIndexStatement(nameComponents.first, nameComponents.second,
new Identifier(indexName), IndexType.BTREE, Collections.emptyList(), false, -1, null, ifNotExists, null, null,
- Collections.emptyMap());
- return addSourceLocation(stmt, startStmtToken);
+ Collections.emptyMap(), Collections.emptyList(), null);
+ return addSourceLocation(stmt, startStmtToken);
+ } catch (CompilationException e) {
+ throw new SqlppParseException(getSourceLocation(startStmtToken), e.getMessage());
+ }
}
}
@@ -1823,7 +1952,12 @@ IndexParams IndexType() throws ParseException:
type = IndexType.LENGTH_PARTITIONED_NGRAM_INVIX;
gramLength = Integer.valueOf(token.image);
}
- <RIGHTPAREN>)
+ <RIGHTPAREN>
+ | LOOKAHEAD({laIdentifier(VTREE)}) <IDENTIFIER>
+ {
+ type = IndexType.VTREE;
+ }
+ )
{
return new IndexParams(type, gramLength, fullTextConfig);
}
@@ -3708,6 +3842,7 @@ Statement AnalyzeStatement() throws ParseException:
}
}
+
Statement AnalyzeDatasetStatement(Token startToken, Namespace namespace, Identifier identifier) throws ParseException:
{
RecordConstructor withRecord = null;
@@ -3724,6 +3859,7 @@ Statement AnalyzeDatasetStatement(Token startToken, Namespace namespace, Identif
}
}
+
Statement AnalyzeDatasetDropStatement(Token startToken, Namespace namespace, Identifier identifier) throws ParseException:
{
}
diff --git a/asterixdb/asterix-metadata/pom.xml b/asterixdb/asterix-metadata/pom.xml
index c060009081..160da0e7bf 100644
--- a/asterixdb/asterix-metadata/pom.xml
+++ b/asterixdb/asterix-metadata/pom.xml
@@ -75,6 +75,10 @@
<groupId>org.apache.hyracks</groupId>
<artifactId>hyracks-storage-am-lsm-btree</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-storage-am-lsm-vtree</artifactId>
+ </dependency>
<dependency>
<groupId>org.apache.hyracks</groupId>
<artifactId>hyracks-storage-am-lsm-rtree</artifactId>
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/BTreeResourceFactoryProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/BTreeResourceFactoryProvider.java
index e3c15db89e..c27e70770d 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/BTreeResourceFactoryProvider.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/BTreeResourceFactoryProvider.java
@@ -146,21 +146,29 @@ public class BTreeResourceFactoryProvider implements IResourceFactoryProvider {
&& index.getIndexName().equals(IndexingConstants.getFilesIndexName(dataset.getDatasetName()))) {
return new ITypeTraits[0];
}
- Index.ValueIndexDetails indexDetails = (Index.ValueIndexDetails) index.getIndexDetails();
+
int numPrimaryKeys = dataset.getPrimaryKeys().size();
- int numSecondaryKeys = indexDetails.getKeyFieldNames().size();
+ int numSecondaryKeys;
+ List<List<String>> keyFieldNames;
+ List<IAType> keyFieldTypes;
+ List<Integer> keySourceIndicators;
+
+ Index.ValueIndexDetails indexDetails = (Index.ValueIndexDetails) index.getIndexDetails();
+ numSecondaryKeys = indexDetails.getKeyFieldNames().size();
+ keyFieldNames = indexDetails.getKeyFieldNames();
+ keyFieldTypes = indexDetails.getKeyFieldTypes();
+ keySourceIndicators = indexDetails.getKeyFieldSourceIndicators();
ITypeTraitProvider typeTraitProvider = metadataProvider.getStorageComponentProvider().getTypeTraitProvider();
ITypeTraits[] secondaryTypeTraits = new ITypeTraits[numSecondaryKeys + numPrimaryKeys];
for (int i = 0; i < numSecondaryKeys; i++) {
ARecordType sourceType;
- List<Integer> keySourceIndicators = indexDetails.getKeyFieldSourceIndicators();
if (keySourceIndicators == null || keySourceIndicators.get(i) == 0) {
sourceType = recordType;
} else {
sourceType = metaType;
}
- Pair<IAType, Boolean> keyTypePair = Index.getNonNullableOpenFieldType(index,
- indexDetails.getKeyFieldTypes().get(i), indexDetails.getKeyFieldNames().get(i), sourceType);
+ Pair<IAType, Boolean> keyTypePair =
+ Index.getNonNullableOpenFieldType(index, keyFieldTypes.get(i), keyFieldNames.get(i), sourceType);
IAType keyType = keyTypePair.first;
secondaryTypeTraits[i] = typeTraitProvider.getTypeTrait(keyType);
}
@@ -179,23 +187,31 @@ public class BTreeResourceFactoryProvider implements IResourceFactoryProvider {
&& index.getIndexName().equals(IndexingConstants.getFilesIndexName(dataset.getDatasetName()))) {
return new IBinaryComparatorFactory[0];
}
- Index.ValueIndexDetails indexDetails = (Index.ValueIndexDetails) index.getIndexDetails();
+
int numPrimaryKeys = dataset.getPrimaryKeys().size();
- int numSecondaryKeys = indexDetails.getKeyFieldNames().size();
+ int numSecondaryKeys;
+ List<List<String>> keyFieldNames;
+ List<IAType> keyFieldTypes;
+ List<Integer> keySourceIndicators;
+
+ Index.ValueIndexDetails indexDetails = (Index.ValueIndexDetails) index.getIndexDetails();
+ numSecondaryKeys = indexDetails.getKeyFieldNames().size();
+ keyFieldNames = indexDetails.getKeyFieldNames();
+ keyFieldTypes = indexDetails.getKeyFieldTypes();
+ keySourceIndicators = indexDetails.getKeyFieldSourceIndicators();
IBinaryComparatorFactoryProvider cmpFactoryProvider =
metadataProvider.getStorageComponentProvider().getComparatorFactoryProvider();
IBinaryComparatorFactory[] secondaryCmpFactories =
new IBinaryComparatorFactory[numSecondaryKeys + numPrimaryKeys];
for (int i = 0; i < numSecondaryKeys; i++) {
ARecordType sourceType;
- List<Integer> keySourceIndicators = indexDetails.getKeyFieldSourceIndicators();
if (keySourceIndicators == null || keySourceIndicators.get(i) == 0) {
sourceType = recordType;
} else {
sourceType = metaType;
}
- Pair<IAType, Boolean> keyTypePair = Index.getNonNullableOpenFieldType(index,
- indexDetails.getKeyFieldTypes().get(i), indexDetails.getKeyFieldNames().get(i), sourceType);
+ Pair<IAType, Boolean> keyTypePair =
+ Index.getNonNullableOpenFieldType(index, keyFieldTypes.get(i), keyFieldNames.get(i), sourceType);
IAType keyType = keyTypePair.first;
secondaryCmpFactories[i] = cmpFactoryProvider.getBinaryComparatorFactory(keyType, true);
}
@@ -221,7 +237,7 @@ public class BTreeResourceFactoryProvider implements IResourceFactoryProvider {
switch (index.getIndexType()) {
case BTREE:
case RTREE:
- // secondary btrees and rtrees do not have bloom filters
+ // Secondary btrees and rtrees do not have bloom filters.
return null;
case LENGTH_PARTITIONED_NGRAM_INVIX:
case LENGTH_PARTITIONED_WORD_INVIX:
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataManagerUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataManagerUtil.java
index 143b615ba3..b861ee46fe 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataManagerUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataManagerUtil.java
@@ -30,6 +30,7 @@ import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.common.metadata.DataverseName;
import org.apache.asterix.common.metadata.MetadataConstants;
+import org.apache.asterix.common.metadata.MetadataUtil;
import org.apache.asterix.metadata.MetadataManager;
import org.apache.asterix.metadata.MetadataTransactionContext;
import org.apache.asterix.metadata.entities.Dataset;
@@ -224,6 +225,34 @@ public class MetadataManagerUtil {
return lookupSourceInMetadata(clusterStateManager, mdTxnCtx, id);
}
+ public static SampleDataSource findSampleDataSource(IClusterStateManager clusterStateManager,
+ MetadataTransactionContext mdTxnCtx, String sampleIndexName, DataSourceId id, boolean usingDatabase)
+ throws AlgebricksException {
+ Dataset dataset = findDataset(mdTxnCtx, id.getDatabaseName(), id.getDataverseName(), id.getDatasourceName());
+ if (dataset == null) {
+ throw new AsterixException(ErrorCode.UNKNOWN_DATASET_IN_DATAVERSE, id.getDatasourceName(),
+ MetadataUtil.dataverseName(id.getDatabaseName(), id.getDataverseName(), usingDatabase));
+ }
+ // Only INTERNAL / EXTERNAL datasets can have a sample data source.
+ switch (dataset.getDatasetType()) {
+ case INTERNAL:
+ case EXTERNAL:
+ break;
+ default:
+ throw new AsterixException(ErrorCode.UNKNOWN_DATASET_IN_DATAVERSE, id.getDatasourceName(),
+ MetadataUtil.dataverseName(id.getDatabaseName(), id.getDataverseName(), usingDatabase));
+ }
+
+ IAType itemType = findType(mdTxnCtx, dataset.getItemTypeDatabaseName(), dataset.getItemTypeDataverseName(),
+ dataset.getItemTypeName());
+ IAType metaItemType = findType(mdTxnCtx, dataset.getMetaItemTypeDatabaseName(),
+ dataset.getMetaItemTypeDataverseName(), dataset.getMetaItemTypeName());
+ itemType = findTypeForDatasetWithoutType(itemType, dataset);
+
+ INodeDomain domain = findNodeDomain(clusterStateManager, mdTxnCtx, dataset.getNodeGroupName());
+ return new SampleDataSource(dataset, sampleIndexName, itemType, metaItemType, domain);
+ }
+
public static DataSource lookupSourceInMetadata(IClusterStateManager clusterStateManager,
MetadataTransactionContext mdTxnCtx, DataSourceId id) throws AlgebricksException {
Dataset dataset = findDataset(mdTxnCtx, id.getDatabaseName(), id.getDataverseName(), id.getDatasourceName());
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
index 7a35b538dc..3e58295934 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
@@ -37,6 +37,7 @@ import java.util.stream.Collectors;
import org.apache.asterix.common.api.INamespaceResolver;
import org.apache.asterix.common.cluster.PartitioningProperties;
+import org.apache.asterix.common.config.CompilerProperties;
import org.apache.asterix.common.config.DatasetConfig.DatasetType;
import org.apache.asterix.common.config.DatasetConfig.IndexType;
import org.apache.asterix.common.config.StorageProperties;
@@ -59,6 +60,9 @@ import org.apache.asterix.common.transactions.ITxnIdFactory;
import org.apache.asterix.common.transactions.TxnId;
import org.apache.asterix.common.utils.StorageConstants;
import org.apache.asterix.common.utils.StoragePathUtil;
+import org.apache.asterix.common.vector.OptimizedScalarQuantizerFactory;
+import org.apache.asterix.common.vector.VectorSimilarityMetric;
+import org.apache.asterix.dataflow.data.common.AOrderedListVectorBinaryAccessorFactory;
import org.apache.asterix.dataflow.data.nontagged.MissingWriterFactory;
import org.apache.asterix.dataflow.data.nontagged.serde.SerializerDeserializerUtil;
import org.apache.asterix.external.adapter.factory.ExternalAdapterFactory;
@@ -99,6 +103,7 @@ import org.apache.asterix.metadata.utils.DataPartitioningProvider;
import org.apache.asterix.metadata.utils.DatasetUtil;
import org.apache.asterix.metadata.utils.FullTextUtil;
import org.apache.asterix.metadata.utils.IndexUtil;
+import org.apache.asterix.object.base.AdmObjectNode;
import org.apache.asterix.om.functions.BuiltinFunctions;
import org.apache.asterix.om.functions.IFunctionExtensionManager;
import org.apache.asterix.om.functions.IFunctionManager;
@@ -114,6 +119,7 @@ import org.apache.asterix.runtime.operators.LSMPrimaryInsertOperatorDescriptor;
import org.apache.asterix.runtime.operators.LSMSecondaryInsertDeleteWithNestedPlanOperatorDescriptor;
import org.apache.asterix.runtime.operators.LSMSecondaryUpsertOperatorDescriptor;
import org.apache.asterix.runtime.operators.LSMSecondaryUpsertWithNestedPlanOperatorDescriptor;
+import org.apache.asterix.runtime.utils.VectorDistanceFunctionFactory;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
@@ -178,12 +184,18 @@ import org.apache.hyracks.storage.am.lsm.btree.dataflow.LSMBTreeBatchPointSearch
import org.apache.hyracks.storage.am.lsm.invertedindex.dataflow.BinaryTokenizerOperatorDescriptor;
import org.apache.hyracks.storage.am.lsm.invertedindex.fulltext.IFullTextConfigEvaluatorFactory;
import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizerFactory;
+import org.apache.hyracks.storage.am.lsm.vector.dataflow.VTreeSearchOperatorDescriptor;
import org.apache.hyracks.storage.am.rtree.dataflow.RTreeSearchOperatorDescriptor;
+import org.apache.hyracks.storage.am.vector.utils.VTreeDataTupleAccessor;
import org.apache.hyracks.storage.common.IStorageManager;
import org.apache.hyracks.storage.common.projection.ITupleProjectorFactory;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
public class MetadataProvider implements IMetadataProvider<DataSourceId, String>, IIndexProvider {
+ private static final Logger LOGGER = LogManager.getLogger();
+
private final ICcApplicationContext appCtx;
private final IStorageComponentProvider storageComponentProvider;
private final StorageProperties storageProperties;
@@ -451,6 +463,11 @@ public class MetadataProvider implements IMetadataProvider<DataSourceId, String>
return MetadataManagerUtil.findDataSource(appCtx.getClusterStateManager(), mdTxnCtx, id);
}
+ public SampleDataSource findSampleDataSource(DataSourceId id, String sampleIndex) throws AlgebricksException {
+ return MetadataManagerUtil.findSampleDataSource(appCtx.getClusterStateManager(), mdTxnCtx, sampleIndex, id,
+ isUsingDatabase());
+ }
+
public DataSource lookupSourceInMetadata(DataSourceId aqlId) throws AlgebricksException {
return MetadataManagerUtil.lookupSourceInMetadata(appCtx.getClusterStateManager(), mdTxnCtx, aqlId);
}
@@ -807,6 +824,72 @@ public class MetadataProvider implements IMetadataProvider<DataSourceId, String>
return new Pair<>(rtreeSearchOp, partitioningProperties.getConstraints());
}
+ public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getVectorSearchRuntime(JobSpecification jobSpec,
+ List<LogicalVariable> outputVars, IOperatorSchema opSchema, IVariableTypeEnvironment typeEnv,
+ JobGenContext context, boolean retainInput, Dataset dataset, String indexName, int[] queryFields,
+ ITupleFilterFactory tupleFilterFactory, boolean indexOnly) throws AlgebricksException {
+ Index vectorIndex = MetadataManager.INSTANCE.getIndex(mdTxnCtx, dataset.getDatabaseName(),
+ dataset.getDataverseName(), dataset.getDatasetName(), indexName);
+ if (vectorIndex == null) {
+ throw new AlgebricksException(
+ "Code generation error: no index " + indexName + " for dataset " + dataset.getDatasetName());
+ }
+
+ RecordDescriptor outputRecDesc = JobGenHelper.mkRecordDescriptor(typeEnv, opSchema, context);
+ PartitioningProperties partitioningProperties = getPartitioningProperties(dataset, vectorIndex.getIndexName());
+
+ int numPrimaryKeys = dataset.getPrimaryKeys().size();
+ int[] primaryKeyFields = new int[numPrimaryKeys];
+ for (int i = 0; i < numPrimaryKeys; i++) {
+ primaryKeyFields[i] = i;
+ }
+
+ ISearchOperationCallbackFactory searchCallbackFactory = dataset.getSearchCallbackFactory(
+ storageComponentProvider, vectorIndex, IndexOperation.SEARCH, primaryKeyFields, null, false);
+ IIndexDataflowHelperFactory indexDataflowHelperFactory = new IndexDataflowHelperFactory(
+ storageComponentProvider.getStorageManager(), partitioningProperties.getSplitsProvider());
+
+ // Secondary-key layout in a VTree data tuple:
+ // Non-quantized: [distance, centroidId] -> 2 fields
+ // Quantized: [distance, centroidId, quantized_distance, quantized_embedding] -> 4 fields
+ Index.VectorIndexDetails vectorIndexDetails = (Index.VectorIndexDetails) vectorIndex.getIndexDetails();
+ AdmObjectNode withObjectNode = vectorIndexDetails.getWithObjectNode();
+ double indexEpsilon = (withObjectNode != null) ? withObjectNode.getOptionalDouble("epsilon", 0.3) : 0.3;
+ String quantization = (withObjectNode != null) ? withObjectNode.getOptionalString("quantization", null) : null;
+ boolean isQuantized = (quantization != null);
+ int numSecondaryKeys = isQuantized ? VTreeDataTupleAccessor.Q_NUM_SECONDARY_FIELDS
+ : VTreeDataTupleAccessor.NQ_NUM_SECONDARY_FIELDS;
+
+ // Session-level kMultiplier override. A value > 1 signals to the NodePushable to widen
+ // the per-partition top-k to compensate for cross-partition ranking error.
+ int kMultiplier = 1;
+ String kmultStr = (String) getConfig().get(CompilerProperties.COMPILER_VECTOR_K_MULTIPLIER_KEY);
+ if (kmultStr != null && !kmultStr.trim().isEmpty()) {
+ try {
+ int parsed = Integer.parseInt(kmultStr.trim());
+ kMultiplier = Math.max(1, parsed);
+ } catch (NumberFormatException e) {
+ LOGGER.warn("Invalid compiler.vector.kmultiplier '{}', using default 1", kmultStr);
+ }
+ }
+
+ AOrderedListVectorBinaryAccessorFactory vectorAccessorFactory = new AOrderedListVectorBinaryAccessorFactory();
+ // The distance metric is fixed at index creation and baked into the factories the search operator
+ // ships; the storage layer never sees a metric string.
+ String distanceMetric = (withObjectNode != null) ? withObjectNode.getOptionalString("similarity", null) : null;
+ VectorDistanceFunctionFactory distanceFunctionFactory =
+ new VectorDistanceFunctionFactory(VectorSimilarityMetric.fromAlias(distanceMetric));
+ OptimizedScalarQuantizerFactory quantizerFactory = new OptimizedScalarQuantizerFactory(distanceMetric);
+
+ int[][] partitionsMap = partitioningProperties.getComputeStorageMap();
+ VTreeSearchOperatorDescriptor vectorSearchOp = new VTreeSearchOperatorDescriptor(jobSpec, outputRecDesc,
+ queryFields, indexDataflowHelperFactory, retainInput, searchCallbackFactory, vectorAccessorFactory,
+ distanceFunctionFactory, quantizerFactory, partitionsMap, numPrimaryKeys, numSecondaryKeys,
+ tupleFilterFactory, kMultiplier, indexEpsilon, indexOnly);
+
+ return new Pair<>(vectorSearchOp, partitioningProperties.getConstraints());
+ }
+
@Override
public Pair<IPushRuntimeFactory, AlgebricksPartitionConstraint> getWriteFileRuntime(int sourceColumn,
int[] partitionColumns, IBinaryComparatorFactory[] partitionComparatorFactories,
@@ -1299,6 +1382,11 @@ public class MetadataProvider implements IMetadataProvider<DataSourceId, String>
primaryKeys, secondaryKeys, additionalNonKeyFields, filterFactory, prevFilterFactory,
inputRecordDesc, context, spec, indexOp, bulkload, operationVar, prevSecondaryKeys,
prevAdditionalFilteringKeys);
+ case VTREE:
+ return getVectorIndexModificationRuntime(database, dataverseName, datasetName, indexName,
+ propagatedSchema, primaryKeys, secondaryKeys, additionalNonKeyFields, filterFactory,
+ prevFilterFactory, inputRecordDesc, context, spec, indexOp, bulkload, operationVar,
+ prevSecondaryKeys, prevAdditionalFilteringKeys);
case SINGLE_PARTITION_WORD_INVIX:
case SINGLE_PARTITION_NGRAM_INVIX:
case LENGTH_PARTITIONED_WORD_INVIX:
@@ -1583,6 +1671,108 @@ public class MetadataProvider implements IMetadataProvider<DataSourceId, String>
return new Pair<>(op, partitioningProperties.getConstraints());
}
+ private Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getVectorIndexModificationRuntime(String database,
+ DataverseName dataverseName, String datasetName, String indexName, IOperatorSchema propagatedSchema,
+ List<LogicalVariable> primaryKeys, List<LogicalVariable> secondaryKeys,
+ List<LogicalVariable> additionalNonKeyFields, AsterixTupleFilterFactory filterFactory,
+ AsterixTupleFilterFactory prevFilterFactory, RecordDescriptor inputRecordDesc, JobGenContext context,
+ JobSpecification spec, IndexOperation indexOp, boolean bulkload, LogicalVariable operationVar,
+ List<LogicalVariable> prevSecondaryKeys, List<LogicalVariable> prevAdditionalFilteringKeys)
+ throws AlgebricksException {
+
+ Dataset dataset = MetadataManagerUtil.findExistingDataset(mdTxnCtx, database, dataverseName, datasetName);
+ int numKeys = primaryKeys.size() + secondaryKeys.size();
+ int numFilterFields = DatasetUtil.getFilterField(dataset) == null ? 0 : 1;
+
+ // Vector indexes do not support tuple filters; drop any factories supplied by the caller.
+ filterFactory = null;
+ prevFilterFactory = null;
+
+ // Field permutation layout: <secondary keys (vector + include fields), primary keys, filter field?>
+ int[] fieldPermutation = new int[numKeys + numFilterFields];
+ int[] modificationCallbackPrimaryKeyFields = new int[primaryKeys.size()];
+ int[] pkFields = new int[primaryKeys.size()];
+ int i = 0;
+ int j = 0;
+
+ for (LogicalVariable varKey : secondaryKeys) {
+ int idx = propagatedSchema.findVariable(varKey);
+ fieldPermutation[i] = idx;
+ i++;
+ }
+
+ for (LogicalVariable varKey : primaryKeys) {
+ int idx = propagatedSchema.findVariable(varKey);
+ fieldPermutation[i] = idx;
+ pkFields[j] = idx;
+ modificationCallbackPrimaryKeyFields[j] = i;
+ i++;
+ j++;
+ }
+
+ if (numFilterFields > 0) {
+ int idx = propagatedSchema.findVariable(additionalNonKeyFields.get(0));
+ fieldPermutation[numKeys] = idx;
+ }
+
+ // For UPSERT, also build the permutation for the previous tuple.
+ int[] prevFieldPermutation = null;
+ if (indexOp == IndexOperation.UPSERT) {
+ prevFieldPermutation = new int[numKeys + numFilterFields];
+ int k = 0;
+
+ for (LogicalVariable varKey : prevSecondaryKeys) {
+ int idx = propagatedSchema.findVariable(varKey);
+ prevFieldPermutation[k] = idx;
+ k++;
+ }
+
+ for (LogicalVariable varKey : primaryKeys) {
+ int idx = propagatedSchema.findVariable(varKey);
+ prevFieldPermutation[k] = idx;
+ k++;
+ }
+
+ if (numFilterFields > 0) {
+ int idx = propagatedSchema.findVariable(prevAdditionalFilteringKeys.get(0));
+ prevFieldPermutation[numKeys] = idx;
+ }
+ }
+
+ try {
+ Index secondaryIndex = MetadataManager.INSTANCE.getIndex(mdTxnCtx, dataset.getDatabaseName(),
+ dataset.getDataverseName(), dataset.getDatasetName(), indexName);
+ PartitioningProperties partitioningProperties =
+ getPartitioningProperties(dataset, secondaryIndex.getIndexName());
+
+ IModificationOperationCallbackFactory modificationCallbackFactory = dataset.getModificationCallbackFactory(
+ storageComponentProvider, secondaryIndex, indexOp, modificationCallbackPrimaryKeyFields);
+ IIndexDataflowHelperFactory idfh = new IndexDataflowHelperFactory(
+ storageComponentProvider.getStorageManager(), partitioningProperties.getSplitsProvider());
+
+ IBinaryHashFunctionFactory[] pkHashFunFactories = dataset.getPrimaryHashFunctionFactories(this);
+ ITuplePartitionerFactory partitionerFactory = new FieldHashPartitionerFactory(pkFields, pkHashFunFactories,
+ partitioningProperties.getNumberOfPartitions());
+
+ IOperatorDescriptor op;
+ if (indexOp == IndexOperation.UPSERT) {
+ int operationFieldIndex = propagatedSchema.findVariable(operationVar);
+ op = new LSMSecondaryUpsertOperatorDescriptor(spec, inputRecordDesc, fieldPermutation, idfh,
+ filterFactory, prevFilterFactory, modificationCallbackFactory, operationFieldIndex,
+ BinaryIntegerInspector.FACTORY, prevFieldPermutation, partitionerFactory,
+ partitioningProperties.getComputeStorageMap());
+ } else {
+ op = new LSMTreeInsertDeleteOperatorDescriptor(spec, inputRecordDesc, fieldPermutation, indexOp, idfh,
+ filterFactory, false, modificationCallbackFactory, partitionerFactory,
+ partitioningProperties.getComputeStorageMap());
+ }
+
+ return new Pair<>(op, partitioningProperties.getConstraints());
+ } catch (Exception e) {
+ throw new AlgebricksException(e);
+ }
+ }
+
private Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getInvertedIndexModificationRuntime(
String database, DataverseName dataverseName, String datasetName, String indexName,
IOperatorSchema propagatedSchema, List<LogicalVariable> primaryKeys, List<LogicalVariable> secondaryKeys,
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
index b3fc1e533a..eb0cec8d41 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
@@ -69,6 +69,7 @@ import org.apache.asterix.metadata.utils.IndexUtil;
import org.apache.asterix.metadata.utils.InvertedIndexResourceFactoryProvider;
import org.apache.asterix.metadata.utils.RTreeResourceFactoryProvider;
import org.apache.asterix.metadata.utils.TypeUtil;
+import org.apache.asterix.metadata.utils.VTreeResourceFactoryProvider;
import org.apache.asterix.om.functions.BuiltinFunctions;
import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.IAType;
@@ -143,6 +144,8 @@ public class Dataset implements IMetadataEntity<Dataset>, IDataset {
RTreeResourceFactoryProvider.INSTANCE;
private static final InvertedIndexResourceFactoryProvider invertedIndexResourceFactoryProvider =
InvertedIndexResourceFactoryProvider.INSTANCE;
+ private static final VTreeResourceFactoryProvider vTreeResourceFactoryProvider =
+ VTreeResourceFactoryProvider.INSTANCE;
/*
* Members
*/
@@ -488,6 +491,10 @@ public class Dataset implements IMetadataEntity<Dataset>, IDataset {
resourceFactory = BTreeResourceFactoryProvider.INSTANCE.getResourceFactory(mdProvider, this, index,
recordType, metaType, mergePolicyFactory, mergePolicyProperties, null, null);
break;
+ case VTREE:
+ resourceFactory = vTreeResourceFactoryProvider.getResourceFactory(mdProvider, this, index, recordType,
+ metaType, mergePolicyFactory, mergePolicyProperties, filterTypeTraits, filterCmpFactories);
+ break;
default:
throw new CompilationException(ErrorCode.COMPILATION_UNKNOWN_INDEX_TYPE,
index.getIndexType().toString());
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslator.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslator.java
index 8ecf7ce00d..00f0137810 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslator.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslator.java
@@ -49,8 +49,13 @@ import org.apache.asterix.metadata.entities.Index;
import org.apache.asterix.metadata.utils.Creator;
import org.apache.asterix.metadata.utils.KeyFieldTypeUtil;
import org.apache.asterix.metadata.utils.TupleTranslatorUtils;
+import org.apache.asterix.object.base.AdmBigIntNode;
+import org.apache.asterix.object.base.AdmDoubleNode;
+import org.apache.asterix.object.base.AdmObjectNode;
+import org.apache.asterix.object.base.AdmStringNode;
import org.apache.asterix.om.base.ABoolean;
import org.apache.asterix.om.base.ACollectionCursor;
+import org.apache.asterix.om.base.ADouble;
import org.apache.asterix.om.base.AInt32;
import org.apache.asterix.om.base.AInt64;
import org.apache.asterix.om.base.AInt8;
@@ -89,6 +94,7 @@ public class IndexTupleTranslator extends AbstractTupleTranslator<Index> {
// Field name of open field.
public static final String GRAM_LENGTH_FIELD_NAME = "GramLength";
public static final String FULL_TEXT_CONFIG_FIELD_NAME = "FullTextConfig";
+ public static final String INCLUDE_FIELDS_FIELD_NAME = "IncludeFields";
public static final String INDEX_SEARCHKEY_TYPE_FIELD_NAME = "SearchKeyType";
public static final String INDEX_ISENFORCED_FIELD_NAME = "IsEnforced";
public static final String INDEX_EXCLUDE_UNKNOWN_FIELD_NAME = "ExcludeUnknownKey";
@@ -172,6 +178,7 @@ public class IndexTupleTranslator extends AbstractTupleTranslator<Index> {
switch (Index.IndexCategory.of(indexType)) {
case VALUE:
case TEXT:
+ case VTREE:
// Read the key names from the SearchKeyName field
IACursor fieldNameCursor =
((AOrderedList) indexRecord.getValueByPos(indexEntity.searchKeyIndex())).getCursor();
@@ -385,7 +392,7 @@ public class IndexTupleTranslator extends AbstractTupleTranslator<Index> {
Triple<IAType, Boolean, Boolean> projectTypeResult =
KeyFieldTypeUtil.getKeyProjectType((ARecordType) inputTypePrime, projectPath, null);
if (projectTypeResult == null) {
- if (indexType != IndexType.BTREE) {
+ if (indexType != IndexType.BTREE && indexType != IndexType.VTREE) {
throw new AsterixException(ErrorCode.METADATA_ERROR, projectPath.toString());
}
projectTypePrime = BuiltinType.ANY;
@@ -460,6 +467,45 @@ public class IndexTupleTranslator extends AbstractTupleTranslator<Index> {
isOverridingKeyTypes, excludeUnknownKey, castDefaultNull, datetimeFormat, dateFormat,
timeFormat);
break;
+ case VTREE:
+ keyFieldNames =
+ searchElements.stream().map(Pair::getSecond).map(List::getFirst).collect(Collectors.toList());
+ keyFieldTypes = searchKeyType.stream().map(List::getFirst).collect(Collectors.toList());
+
+ excludeUnknownKey = OptionalBoolean.empty();
+ castDefaultNull = OptionalBoolean.empty();
+ AdmObjectNode withObjectNode = readWithProperties(indexRecord);
+
+ // Read include_fields from metadata
+ // Each item is an inner list holding the field's path components, so a nested INCLUDE path like
+ // (a.b) is read back intact (see writeIncludeFields).
+ List<List<String>> includeFieldNames = new ArrayList<>();
+ int includeFieldsPos = indexRecord.getType().getFieldIndex(INCLUDE_FIELDS_FIELD_NAME);
+ if (includeFieldsPos >= 0) {
+ IACursor cursor = ((AOrderedList) indexRecord.getValueByPos(includeFieldsPos)).getCursor();
+ while (cursor.next()) {
+ IACursor pathCursor = ((AOrderedList) cursor.get()).getCursor();
+ List<String> path = new ArrayList<>();
+ while (pathCursor.next()) {
+ path.add(((AString) pathCursor.get()).getStringValue());
+ }
+ includeFieldNames.add(path);
+ }
+ }
+
+ // Create proper source indicators and types for include fields
+ // Include fields always come from the record (not meta), and types are resolved later
+ List<Integer> includeFieldSourceIndicators = new ArrayList<>();
+ List<IAType> includeFieldTypes = new ArrayList<>();
+ for (int i = 0; i < includeFieldNames.size(); i++) {
+ includeFieldSourceIndicators.add(Index.RECORD_INDICATOR);
+ includeFieldTypes.add(BuiltinType.ANY); // Type will be resolved from record schema
+ }
+
+ indexDetails = new Index.VectorIndexDetails(keyFieldNames.getFirst(), includeFieldNames,
+ includeFieldSourceIndicators, includeFieldTypes, isOverridingKeyTypes, excludeUnknownKey,
+ withObjectNode);
+ break;
case TEXT:
keyFieldNames =
searchElements.stream().map(Pair::getSecond).map(l -> l.get(0)).collect(Collectors.toList());
@@ -633,6 +679,9 @@ public class IndexTupleTranslator extends AbstractTupleTranslator<Index> {
case SAMPLE:
searchKey = ((Index.SampleIndexDetails) index.getIndexDetails()).getKeyFieldNames();
break;
+ case VTREE:
+ searchKey = ((Index.VectorIndexDetails) index.getIndexDetails()).getKeyFieldNames();
+ break;
default:
throw new AsterixException(ErrorCode.METADATA_ERROR, indexType.toString());
}
@@ -696,8 +745,19 @@ public class IndexTupleTranslator extends AbstractTupleTranslator<Index> {
case ARRAY:
writeComplexSearchKeys((Index.ArrayIndexDetails) index.getIndexDetails());
break;
+ case VTREE:
+ Index.VectorIndexDetails vectorIndexDetails = (Index.VectorIndexDetails) index.getIndexDetails();
+ writeWithProperties(vectorIndexDetails);
+ writeIncludeFields(vectorIndexDetails);
+ break;
}
writeSearchKeyType(index);
+
+ if (Index.IndexCategory.of(index.getIndexType()) == Index.IndexCategory.VTREE) {
+ // Vector indexes do not have enforced keys.
+ return;
+ }
+
writeEnforced(index);
writeSearchKeySourceIndicator(index);
writeExcludeUnknownKey(index);
@@ -796,6 +856,294 @@ public class IndexTupleTranslator extends AbstractTupleTranslator<Index> {
}
}
+ private void writeWithProperties(Index.VectorIndexDetails index) throws HyracksDataException, AlgebricksException {
+ AdmObjectNode properties = index.getWithObjectNode();
+
+ // Sentinel defaults for the case where no WITH clause was specified.
+ int dimension = -1;
+ double train_list_fraction = -1.0;
+ int num_clusters = -1;
+ String quantization = "INVALID";
+ String similarity = "INVALID";
+ double epsilon = 0.3;
+ // cross_pollination_m: each record replicated into the M closest leaf centroids at bulk-load.
+ // 1 = legacy behavior (no cross-pollination).
+ int cross_pollination_m = 1;
+ // rng_factor / creation_mode: bulk-load routing + training-algorithm knobs. Both are read by the
+ // build path (VTreeResourceFactoryProvider / SecondaryVectorOperationsHelper), so they must survive
+ // a metadata reload; previously they were validated on the statement node but never persisted.
+ double rng_factor = 1.0;
+ String creation_mode = "INVALID";
+
+ if (properties != null) {
+ dimension = properties.getOptionalInt("dimension", -1);
+ train_list_fraction = properties.getOptionalDouble("train_list_fraction", -1.0);
+ num_clusters = properties.getOptionalInt("num_clusters", -1);
+ quantization = properties.getOptionalString("quantization", "INVALID");
+ similarity = properties.getOptionalString("similarity", "INVALID");
+ epsilon = properties.getOptionalDouble("epsilon", 0.3);
+ cross_pollination_m = properties.getOptionalInt("cross_pollination_m", 1);
+ rng_factor = properties.getOptionalDouble("rng_factor", 1.0);
+ creation_mode = properties.getOptionalString("creation_mode", "INVALID");
+ }
+
+ if (dimension < 0) {
+ throw new AsterixException(ErrorCode.COMPILATION_VECTOR_INDEX_CREATION_FAILED,
+ "Missing `dimension` parameter in the WITH clause");
+ }
+
+ if ("INVALID".equals(similarity)) {
+ throw new AsterixException(ErrorCode.COMPILATION_VECTOR_INDEX_CREATION_FAILED,
+ "Missing `similarity` parameter in the WITH clause");
+ }
+ nameValue.reset();
+ aString.setValue("dimension");
+ stringSerde.serialize(aString, nameValue.getDataOutput());
+ fieldValue.reset();
+ int32Serde.serialize(new AInt32(dimension), fieldValue.getDataOutput());
+ recordBuilder.addField(nameValue, fieldValue);
+
+ nameValue.reset();
+ aString.setValue("train_list_fraction");
+ stringSerde.serialize(aString, nameValue.getDataOutput());
+ fieldValue.reset();
+ doubleSerde.serialize(new ADouble(train_list_fraction), fieldValue.getDataOutput());
+ recordBuilder.addField(nameValue, fieldValue);
+
+ if (num_clusters > 0) {
+ nameValue.reset();
+ aString.setValue("num_clusters");
+ stringSerde.serialize(aString, nameValue.getDataOutput());
+ fieldValue.reset();
+ int32Serde.serialize(new AInt32(num_clusters), fieldValue.getDataOutput());
+ recordBuilder.addField(nameValue, fieldValue);
+ }
+
+ if (!"INVALID".equals(quantization)) {
+ nameValue.reset();
+ aString.setValue("quantization");
+ stringSerde.serialize(aString, nameValue.getDataOutput());
+ fieldValue.reset();
+ aString.setValue(quantization);
+ stringSerde.serialize(aString, fieldValue.getDataOutput());
+ recordBuilder.addField(nameValue, fieldValue);
+ }
+
+ if (!"INVALID".equals(similarity)) {
+ nameValue.reset();
+ aString.setValue("similarity");
+ stringSerde.serialize(aString, nameValue.getDataOutput());
+ fieldValue.reset();
+ aString.setValue(similarity);
+ stringSerde.serialize(aString, fieldValue.getDataOutput());
+ recordBuilder.addField(nameValue, fieldValue);
+ }
+
+ nameValue.reset();
+ aString.setValue("epsilon");
+ stringSerde.serialize(aString, nameValue.getDataOutput());
+ fieldValue.reset();
+ doubleSerde.serialize(new ADouble(epsilon), fieldValue.getDataOutput());
+ recordBuilder.addField(nameValue, fieldValue);
+
+ // Only persist cross_pollination_m when > 1 — older metadata (no field) reads back as 1.
+ if (cross_pollination_m > 1) {
+ nameValue.reset();
+ aString.setValue("cross_pollination_m");
+ stringSerde.serialize(aString, nameValue.getDataOutput());
+ fieldValue.reset();
+ int32Serde.serialize(new AInt32(cross_pollination_m), fieldValue.getDataOutput());
+ recordBuilder.addField(nameValue, fieldValue);
+ }
+
+ // Persist rng_factor only when non-default (1.0); absent reads back as 1.0 (older metadata / M=1).
+ if (rng_factor != 1.0) {
+ nameValue.reset();
+ aString.setValue("rng_factor");
+ stringSerde.serialize(aString, nameValue.getDataOutput());
+ fieldValue.reset();
+ doubleSerde.serialize(new ADouble(rng_factor), fieldValue.getDataOutput());
+ recordBuilder.addField(nameValue, fieldValue);
+ }
+
+ // Persist creation_mode only when a non-default mode was chosen; absent reads back as bottom-up.
+ if (!"INVALID".equals(creation_mode) && !"bottom-up".equals(creation_mode)) {
+ nameValue.reset();
+ aString.setValue("creation_mode");
+ stringSerde.serialize(aString, nameValue.getDataOutput());
+ fieldValue.reset();
+ aString.setValue(creation_mode);
+ stringSerde.serialize(aString, fieldValue.getDataOutput());
+ recordBuilder.addField(nameValue, fieldValue);
+ }
+ }
+
+ private AdmObjectNode readWithProperties(ARecord indexRecord) throws AlgebricksException {
+ // Default values match writeWithProperties() above.
+ int dimension = -1;
+ double train_list_fraction = -1.0;
+ int num_clusters = -1;
+ String quantization = "INVALID";
+ String similarity = "INVALID";
+ double epsilon = 0.3;
+ int epsilonPos = indexRecord.getType().getFieldIndex("epsilon");
+ // cross_pollination_m: 1 = legacy (no cross-pollination), absent in older metadata.
+ int cross_pollination_m = 1;
+ int crossPollinationMPos = indexRecord.getType().getFieldIndex("cross_pollination_m");
+ double rng_factor = 1.0;
+ int rngFactorPos = indexRecord.getType().getFieldIndex("rng_factor");
+ String creation_mode = "INVALID";
+ int creationModePos = indexRecord.getType().getFieldIndex("creation_mode");
+
+ int dimensionPos = indexRecord.getType().getFieldIndex("dimension");
+ if (dimensionPos >= 0) {
+ IAObject dimensionObj = indexRecord.getValueByPos(dimensionPos);
+ if (dimensionObj != null && dimensionObj.getType().getTypeTag() == ATypeTag.INTEGER) {
+ dimension = ((AInt32) dimensionObj).getIntegerValue();
+ }
+ }
+
+ int trainListFractionPos = indexRecord.getType().getFieldIndex("train_list_fraction");
+ if (trainListFractionPos >= 0) {
+ IAObject trainListFractionObj = indexRecord.getValueByPos(trainListFractionPos);
+ if (trainListFractionObj != null && trainListFractionObj.getType().getTypeTag() == ATypeTag.DOUBLE) {
+ train_list_fraction = ((ADouble) trainListFractionObj).getDoubleValue();
+ }
+ }
+
+ int numClustersPos = indexRecord.getType().getFieldIndex("num_clusters");
+ if (numClustersPos >= 0) {
+ IAObject numClustersObj = indexRecord.getValueByPos(numClustersPos);
+ if (numClustersObj != null && numClustersObj.getType().getTypeTag() == ATypeTag.INTEGER) {
+ num_clusters = ((AInt32) numClustersObj).getIntegerValue();
+ }
+ }
+
+ int quantizationPos = indexRecord.getType().getFieldIndex("quantization");
+ if (quantizationPos >= 0) {
+ IAObject quantizationObj = indexRecord.getValueByPos(quantizationPos);
+ if (quantizationObj != null && quantizationObj.getType().getTypeTag() == ATypeTag.STRING) {
+ quantization = ((AString) quantizationObj).getStringValue();
+ }
+ }
+
+ int similarityPos = indexRecord.getType().getFieldIndex("similarity");
+ if (similarityPos >= 0) {
+ IAObject similarityObj = indexRecord.getValueByPos(similarityPos);
+ if (similarityObj != null && similarityObj.getType().getTypeTag() == ATypeTag.STRING) {
+ similarity = ((AString) similarityObj).getStringValue();
+ }
+ }
+
+ // epsilon is optional in older metadata records.
+ if (epsilonPos >= 0) {
+ IAObject epsilonObj = indexRecord.getValueByPos(epsilonPos);
+ if (epsilonObj != null && epsilonObj.getType().getTypeTag() == ATypeTag.DOUBLE) {
+ epsilon = ((ADouble) epsilonObj).getDoubleValue();
+ }
+ }
+
+ // Read cross_pollination_m field (optional for older metadata; absence == 1)
+ if (crossPollinationMPos >= 0) {
+ IAObject mObj = indexRecord.getValueByPos(crossPollinationMPos);
+ if (mObj != null && mObj.getType().getTypeTag() == ATypeTag.INTEGER) {
+ cross_pollination_m = ((AInt32) mObj).getIntegerValue();
+ }
+ }
+
+ // rng_factor / creation_mode: optional; absence reads back as the defaults (1.0 / bottom-up).
+ if (rngFactorPos >= 0) {
+ IAObject rngObj = indexRecord.getValueByPos(rngFactorPos);
+ if (rngObj != null && rngObj.getType().getTypeTag() == ATypeTag.DOUBLE) {
+ rng_factor = ((ADouble) rngObj).getDoubleValue();
+ }
+ }
+ if (creationModePos >= 0) {
+ IAObject cmObj = indexRecord.getValueByPos(creationModePos);
+ if (cmObj != null && cmObj.getType().getTypeTag() == ATypeTag.STRING) {
+ creation_mode = ((AString) cmObj).getStringValue();
+ }
+ }
+
+ // Reconstruct AdmObjectNode only if at least one field differs from default
+ boolean hasNonDefaultValues = (dimension != -1) || (train_list_fraction >= 0) || (num_clusters != -1)
+ || (!"default".equals(quantization) && !"INVALID".equals(quantization))
+ || (!"INVALID".equals(similarity)) || (epsilonPos >= 0 && Math.abs(epsilon - 0.3) > 1e-12)
+ || (cross_pollination_m > 1) || (rng_factor != 1.0)
+ || (!"INVALID".equals(creation_mode) && !"bottom-up".equals(creation_mode));
+
+ if (!hasNonDefaultValues) {
+ return null;
+ }
+
+ AdmObjectNode withObjectNode = new AdmObjectNode();
+
+ if (dimension != -1) {
+ withObjectNode.set("dimension", new AdmBigIntNode(dimension));
+ }
+
+ if (train_list_fraction >= 0) {
+ withObjectNode.set("train_list_fraction", new AdmDoubleNode(train_list_fraction));
+ }
+
+ if (num_clusters != -1) {
+ withObjectNode.set("num_clusters", new AdmBigIntNode(num_clusters));
+ }
+
+ if (!"default".equals(quantization) && !"INVALID".equals(quantization)) {
+ withObjectNode.set("quantization", new AdmStringNode(quantization));
+ }
+
+ if (!"INVALID".equals(similarity)) {
+ withObjectNode.set("similarity", new AdmStringNode(similarity));
+ }
+
+ if (epsilonPos >= 0) {
+ withObjectNode.set("epsilon", new AdmDoubleNode(epsilon));
+ }
+
+ if (cross_pollination_m > 1) {
+ withObjectNode.set("cross_pollination_m", new AdmBigIntNode(cross_pollination_m));
+ }
+
+ if (rng_factor != 1.0) {
+ withObjectNode.set("rng_factor", new AdmDoubleNode(rng_factor));
+ }
+
+ if (!"INVALID".equals(creation_mode) && !"bottom-up".equals(creation_mode)) {
+ withObjectNode.set("creation_mode", new AdmStringNode(creation_mode));
+ }
+
+ return withObjectNode;
+ }
+
+ private void writeIncludeFields(Index.VectorIndexDetails index) throws HyracksDataException {
+ List<List<String>> includeElements = index.getIncludeFieldNames();
+ OrderedListBuilder listBuilder = new OrderedListBuilder();
+ OrderedListBuilder pathBuilder = new OrderedListBuilder();
+ listBuilder.reset(AOrderedListType.FULL_OPEN_ORDEREDLIST_TYPE);
+ // Each INCLUDE field is a (possibly nested) path serialized as its own inner list of components, so a
+ // nested path like (a.b) round-trips intact. Mirrors buildSearchKeyNameList and the search-key read path.
+ for (List<String> field : includeElements) {
+ pathBuilder.reset(AOrderedListType.FULL_OPEN_ORDEREDLIST_TYPE);
+ for (String component : field) {
+ itemValue.reset();
+ aString.setValue(component);
+ stringSerde.serialize(aString, itemValue.getDataOutput());
+ pathBuilder.addItem(itemValue);
+ }
+ itemValue.reset();
+ pathBuilder.write(itemValue.getDataOutput(), true);
+ listBuilder.addItem(itemValue);
+ }
+ fieldValue.reset();
+ listBuilder.write(fieldValue.getDataOutput(), true);
+ nameValue.reset();
+ aString.setValue(INCLUDE_FIELDS_FIELD_NAME);
+ stringSerde.serialize(aString, nameValue.getDataOutput());
+ recordBuilder.addField(nameValue, fieldValue);
+ }
+
private void writeSearchKeyType(Index index) throws HyracksDataException, AlgebricksException {
if (!index.getIndexDetails().isOverridingKeyFieldTypes()) {
return;
@@ -811,6 +1159,12 @@ public class IndexTupleTranslator extends AbstractTupleTranslator<Index> {
switch (Index.IndexCategory.of(index.getIndexType())) {
// For value and text indexes, we persist the type as a single string (backwards compatibility).
+ case VTREE:
+ itemValue.reset();
+ aString.setValue("vector");
+ stringSerde.serialize(aString, itemValue.getDataOutput());
+ typeListBuilder.addItem(itemValue);
+ break;
case VALUE:
for (IAType type : ((Index.ValueIndexDetails) index.getIndexDetails()).getKeyFieldTypes()) {
itemValue.reset();
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
index dd22ecd9e5..2cb452f1aa 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
@@ -630,6 +630,9 @@ public class DatasetUtil {
} else if (index.getIndexType() == DatasetConfig.IndexType.ARRAY) {
Index.ArrayIndexDetails indexDetails = (Index.ArrayIndexDetails) index.getIndexDetails();
indexPaths.add(indexDetails.getIndexExpectedType());
+ } else if (index.getIndexType() == DatasetConfig.IndexType.VTREE) {
+ Index.VectorIndexDetails indexDetails = (Index.VectorIndexDetails) index.getIndexDetails();
+ indexPaths.add(indexDetails.getIndexExpectedType());
}
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/KeyFieldTypeUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/KeyFieldTypeUtil.java
index 5cf3463fe5..1eb094e3b5 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/KeyFieldTypeUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/KeyFieldTypeUtil.java
@@ -18,10 +18,13 @@
*/
package org.apache.asterix.metadata.utils;
+import static org.apache.asterix.om.types.AOrderedListType.FULL_OPEN_ORDEREDLIST_TYPE;
+
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
+import org.apache.asterix.common.config.DatasetConfig;
import org.apache.asterix.common.config.DatasetConfig.DatasetType;
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.exceptions.ErrorCode;
@@ -131,6 +134,18 @@ public class KeyFieldTypeUtil {
*/
public static List<Pair<IAType, Boolean>> getBTreeIndexKeyTypes(Index index, ARecordType recordType,
ARecordType metaRecordType) throws AlgebricksException {
+ if (index.getIndexType() == DatasetConfig.IndexType.SAMPLE) {
+ Index.SampleIndexDetails indexDetails = (Index.SampleIndexDetails) index.getIndexDetails();
+ List<Integer> keySourceIndicators = indexDetails.getKeyFieldSourceIndicators();
+ List<Pair<IAType, Boolean>> indexKeyTypes = new ArrayList<>();
+ for (int i = 0; i < indexDetails.getKeyFieldNames().size(); i++) {
+ Pair<IAType, Boolean> keyPairType = Index.getNonNullableOpenFieldType(index,
+ indexDetails.getKeyFieldTypes().get(i), indexDetails.getKeyFieldNames().get(i),
+ chooseSource(keySourceIndicators, i, recordType, metaRecordType));
+ indexKeyTypes.add(keyPairType);
+ }
+ return indexKeyTypes;
+ }
Index.ValueIndexDetails indexDetails = (Index.ValueIndexDetails) index.getIndexDetails();
List<Integer> keySourceIndicators = indexDetails.getKeyFieldSourceIndicators();
List<Pair<IAType, Boolean>> indexKeyTypes = new ArrayList<>();
@@ -210,6 +225,17 @@ public class KeyFieldTypeUtil {
return indexKeyTypes;
}
+ public static List<IAType> getVectorIndexKeyTypes(Index index, ARecordType recordType, ARecordType metaRecordType)
+ throws AlgebricksException {
+ Index.VectorIndexDetails indexDetails = (Index.VectorIndexDetails) index.getIndexDetails();
+ // Vector field is always sourced from the record (not the meta record).
+ Pair<IAType, Boolean> keyPairType = Index.getNonNullableOpenFieldType(index, FULL_OPEN_ORDEREDLIST_TYPE,
+ indexDetails.getKeyFieldNames().get(0), recordType);
+ List<IAType> indexKeyTypes = new ArrayList<>();
+ indexKeyTypes.add(keyPairType.first);
+ return indexKeyTypes;
+ }
+
/**
* Get the number of secondary index keys.
*
@@ -243,6 +269,8 @@ public class KeyFieldTypeUtil {
chooseSource(keySourceIndicators, 0, recordType, metaRecordType));
IAType keyType = keyPairType.first;
return NonTaggedFormatUtil.getNumDimensions(keyType.getTypeTag()) * 2;
+ case VTREE:
+ return ((Index.VectorIndexDetails) index.getIndexDetails()).getKeyFieldNames().size();
default:
throw new CompilationException(ErrorCode.COMPILATION_UNKNOWN_INDEX_TYPE, index.getIndexType());
}
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java
index d41d6cbe1c..dc96565a24 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java
@@ -1217,6 +1217,10 @@ public class BuiltinFunctions {
public static final FunctionIdentifier DOT_DISTANCE = FunctionConstants.newAsterix("dot-distance", 2);
public static final FunctionIdentifier VECTOR_DISTANCE = FunctionConstants.newAsterix("vector-distance", 3);
+ // Vector search functions
+ public static final FunctionIdentifier ANN_DISTANCE =
+ FunctionConstants.newAsterix("ann-distance", FunctionIdentifier.VARARGS);
+
// Temporal functions
public static final FunctionIdentifier UNIX_TIME_FROM_DATE_IN_DAYS =
FunctionConstants.newAsterix("unix-time-from-date-in-days", 1);
@@ -1952,6 +1956,9 @@ public class BuiltinFunctions {
addPrivateFunction(DOT_DISTANCE, ADoubleTypeComputer.INSTANCE, true);
addFunction(VECTOR_DISTANCE, ADoubleTypeComputer.INSTANCE, true);
+ // Ann functions
+ addFunction(ANN_DISTANCE, ADoubleTypeComputer.INSTANCE, true);
+
// Window functions
addFunction(CUME_DIST, ADoubleTypeComputer.INSTANCE, false);
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/PhysicalOperatorTag.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/PhysicalOperatorTag.java
index c052d58c84..95d3a73f45 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/PhysicalOperatorTag.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/PhysicalOperatorTag.java
@@ -84,5 +84,6 @@ public enum PhysicalOperatorTag {
WRITE_RESULT,
INTERSECT,
WINDOW,
- WINDOW_STREAM
+ WINDOW_STREAM,
+ VECTOR_SEARCH
}