You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@asterixdb.apache.org by AsterixDB Code Review <do...@asterix-gerrit.ics.uci.edu> on 2026/05/27 04:23:15 UTC
[L] Change in asterixdb[master]: Add Create Index Statement
From Savyasach Reddy <es...@gmail.com>:
Savyasach Reddy has uploaded this change for review. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21288?usp=email )
Change subject: Add Create Index Statement
......................................................................
Add Create Index Statement
Change-Id: I5a2665290da14a8da83d794c7ee1f68514186e01
---
M asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/util/ValidateUtil.java
M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/io/PersistedResourceRegistry.java
M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/DatasetConfig.java
M asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Index.java
M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslator.java
M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/InvertedIndexResourceFactoryProvider.java
M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/KeyFieldTypeUtil.java
A asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/common/SparseVectorBinaryTokenizer.java
A asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/common/SparseVectorBinaryTokenizerFactory.java
A asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/common/SparseVectorToken.java
M asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java
A asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/sparsevector/SparseVectorDotProductCheckDescriptor.java
A asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/sparsevector/SparseVectorDotProductCheckEvaluator.java
M asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/functions/FunctionCollection.java
M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexSearchOperatorNodePushable.java
18 files changed, 769 insertions(+), 5 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/88/21288/1
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/util/ValidateUtil.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/util/ValidateUtil.java
index 333b82f..a68a8b7 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/util/ValidateUtil.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/util/ValidateUtil.java
@@ -351,6 +351,19 @@
+ " cannot be indexed using the Keyword index.");
}
break;
+ case SPARSE_VECTOR:
+ switch (fieldType.getTypeTag()) {
+ case OBJECT:
+ case ANY:
+ break;
+ default:
+ throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
+ "The field '"
+ + LogRedactionUtil.userData(RecordUtil.toFullyQualifiedName(displayFieldName))
+ + "' which is of type " + fieldType.getTypeTag()
+ + " cannot be indexed using the Sparse Vector index.");
+ }
+ break;
default:
throw new CompilationException(ErrorCode.COMPILATION_UNKNOWN_INDEX_TYPE, sourceLoc,
String.valueOf(indexType));
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/io/PersistedResourceRegistry.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/io/PersistedResourceRegistry.java
index 093b4c1..fd33f2a 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/io/PersistedResourceRegistry.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/io/PersistedResourceRegistry.java
@@ -40,6 +40,7 @@
import org.apache.asterix.dataflow.data.common.AListElementTokenFactory;
import org.apache.asterix.dataflow.data.common.AOrderedListBinaryTokenizerFactory;
import org.apache.asterix.dataflow.data.common.AUnorderedListBinaryTokenizerFactory;
+import org.apache.asterix.dataflow.data.common.SparseVectorBinaryTokenizerFactory;
import org.apache.asterix.dataflow.data.nontagged.comparators.ACirclePartialBinaryComparatorFactory;
import org.apache.asterix.dataflow.data.nontagged.comparators.ADurationPartialBinaryComparatorFactory;
import org.apache.asterix.dataflow.data.nontagged.comparators.AGenericAscBinaryComparatorFactory;
@@ -282,6 +283,7 @@
registeredClasses.put("NGramUTF8StringBinaryTokenizerFactory", NGramUTF8StringBinaryTokenizerFactory.class);
registeredClasses.put("DelimitedUTF8StringBinaryTokenizerFactory",
DelimitedUTF8StringBinaryTokenizerFactory.class);
+ registeredClasses.put("SparseVectorBinaryTokenizerFactory", SparseVectorBinaryTokenizerFactory.class);
// IFullTextConfigFactory
registeredClasses.put("FullTextConfigEvaluatorFactory", FullTextConfigEvaluatorFactory.class);
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index df52f9c..ea4df40 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -1704,6 +1704,10 @@
keyFieldTypes, overridesFieldTypes, stmtCreateIndex.getGramLength(),
stmtCreateIndex.getFullTextConfigName());
break;
+ case SPARSE_VECTOR:
+ indexDetails = new Index.SparseVectorIndexDetails(keyFieldNames, keyFieldSourceIndicators,
+ keyFieldTypes, overridesFieldTypes);
+ break;
default:
throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_STATE, sourceLoc,
String.valueOf(indexType));
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/DatasetConfig.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/DatasetConfig.java
index dbdddad..e2dd39d 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/DatasetConfig.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/DatasetConfig.java
@@ -48,7 +48,8 @@
LENGTH_PARTITIONED_WORD_INVIX,
LENGTH_PARTITIONED_NGRAM_INVIX,
ARRAY,
- SAMPLE
+ SAMPLE,
+ SPARSE_VECTOR
}
public enum TransactionState {
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
index cbe0e7e..dc93972 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
+++ b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
@@ -1819,7 +1819,12 @@
type = IndexType.LENGTH_PARTITIONED_NGRAM_INVIX;
gramLength = Integer.valueOf(token.image);
}
- <RIGHTPAREN>)
+ <RIGHTPAREN>
+ | <SPARSE>
+ {
+ type = IndexType.SPARSE_VECTOR;
+ }
+ )
{
return new IndexParams(type, gramLength, fullTextConfig);
}
@@ -6664,6 +6669,7 @@
| <SATISFIES : "satisfies">
| <SECONDARY : "secondary">
| <SELECT : "select">
+ | <SPARSE : "sparse">
| <SET : "set">
| <SOME : "some">
| <START : "start">
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
index b3fc1e5..cc82071 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
@@ -480,6 +480,7 @@
case LENGTH_PARTITIONED_WORD_INVIX:
case SINGLE_PARTITION_NGRAM_INVIX:
case SINGLE_PARTITION_WORD_INVIX:
+ case SPARSE_VECTOR:
resourceFactory = invertedIndexResourceFactoryProvider.getResourceFactory(mdProvider, this, index,
recordType, metaType, mergePolicyFactory, mergePolicyProperties, filterTypeTraits,
filterCmpFactories);
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Index.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Index.java
index bb564de..27a7766 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Index.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Index.java
@@ -310,6 +310,7 @@
case LENGTH_PARTITIONED_WORD_INVIX:
case SINGLE_PARTITION_NGRAM_INVIX:
case SINGLE_PARTITION_WORD_INVIX:
+ case SPARSE_VECTOR:
return ResourceType.LSM_INVERTED_INDEX;
default:
throw new CompilationException(ErrorCode.COMPILATION_UNKNOWN_INDEX_TYPE, indexType);
@@ -326,7 +327,8 @@
VALUE,
TEXT,
ARRAY,
- SAMPLE;
+ SAMPLE,
+ SPARSE_VECTOR;
public static IndexCategory of(IndexType indexType) {
switch (indexType) {
@@ -342,6 +344,8 @@
return ARRAY;
case SAMPLE:
return SAMPLE;
+ case SPARSE_VECTOR:
+ return SPARSE_VECTOR;
default:
throw new IllegalArgumentException(String.valueOf(indexType));
}
@@ -504,6 +508,46 @@
}
}
+ public static final class SparseVectorIndexDetails extends AbstractIndexDetails {
+
+ private static final long serialVersionUID = 1L;
+
+ private final List<List<String>> keyFieldNames;
+ private final List<Integer> keyFieldSourceIndicators;
+ private final List<IAType> keyFieldTypes;
+ private final boolean overrideKeyFieldTypes;
+
+ public SparseVectorIndexDetails(List<List<String>> keyFieldNames, List<Integer> keyFieldSourceIndicators,
+ List<IAType> keyFieldTypes, boolean overrideKeyFieldTypes) {
+ this.keyFieldNames = keyFieldNames;
+ this.keyFieldSourceIndicators = keyFieldSourceIndicators;
+ this.keyFieldTypes = keyFieldTypes;
+ this.overrideKeyFieldTypes = overrideKeyFieldTypes;
+ }
+
+ @Override
+ IndexCategory getIndexCategory() {
+ return IndexCategory.SPARSE_VECTOR;
+ }
+
+ public List<List<String>> getKeyFieldNames() {
+ return keyFieldNames;
+ }
+
+ public List<Integer> getKeyFieldSourceIndicators() {
+ return keyFieldSourceIndicators;
+ }
+
+ public List<IAType> getKeyFieldTypes() {
+ return keyFieldTypes;
+ }
+
+ @Override
+ public boolean isOverridingKeyFieldTypes() {
+ return overrideKeyFieldTypes;
+ }
+ }
+
public static class ArrayIndexDetails extends AbstractIndexDetails {
private static final long serialVersionUID = 1L;
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslator.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslator.java
index 3eaf155..0af87101 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslator.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslator.java
@@ -171,6 +171,7 @@
switch (Index.IndexCategory.of(indexType)) {
case VALUE:
case TEXT:
+ case SPARSE_VECTOR:
// Read the key names from the SearchKeyName field
IACursor fieldNameCursor =
((AOrderedList) indexRecord.getValueByPos(indexEntity.searchKeyIndex())).getCursor();
@@ -472,6 +473,13 @@
indexDetails = new Index.TextIndexDetails(keyFieldNames, keyFieldSourceIndicator, keyFieldTypes,
isOverridingKeyTypes, gramLength, fullTextConfig);
break;
+ case SPARSE_VECTOR:
+ keyFieldNames =
+ searchElements.stream().map(Pair::getSecond).map(l -> l.get(0)).collect(Collectors.toList());
+ keyFieldTypes = searchKeyType.stream().map(l -> l.get(0)).collect(Collectors.toList());
+ indexDetails = new Index.SparseVectorIndexDetails(keyFieldNames, keyFieldSourceIndicator, keyFieldTypes,
+ isOverridingKeyTypes);
+ break;
case ARRAY:
List<Index.ArrayIndexElement> elementList = new ArrayList<>(searchElementCount);
for (int i = 0; i < searchElementCount; i++) {
@@ -616,6 +624,9 @@
case TEXT:
searchKey = ((Index.TextIndexDetails) index.getIndexDetails()).getKeyFieldNames();
break;
+ case SPARSE_VECTOR:
+ searchKey = ((Index.SparseVectorIndexDetails) index.getIndexDetails()).getKeyFieldNames();
+ break;
case ARRAY:
// If we have a complex index, we persist all of the names in the complex SK name array instead.
searchKey = Collections.emptyList();
@@ -817,6 +828,14 @@
typeListBuilder.addItem(itemValue);
}
break;
+ case SPARSE_VECTOR:
+ for (IAType type : ((Index.SparseVectorIndexDetails) index.getIndexDetails()).getKeyFieldTypes()) {
+ itemValue.reset();
+ aString.setValue(type.getTypeName());
+ stringSerde.serialize(aString, itemValue.getDataOutput());
+ typeListBuilder.addItem(itemValue);
+ }
+ break;
case ARRAY:
// For array indexes we persist the type as a list of strings.
for (Index.ArrayIndexElement element : ((Index.ArrayIndexDetails) index.getIndexDetails())
@@ -861,6 +880,10 @@
case TEXT:
keySourceIndicator = ((Index.TextIndexDetails) index.getIndexDetails()).getKeyFieldSourceIndicators();
break;
+ case SPARSE_VECTOR:
+ keySourceIndicator =
+ ((Index.SparseVectorIndexDetails) index.getIndexDetails()).getKeyFieldSourceIndicators();
+ break;
case ARRAY:
keySourceIndicator = ((Index.ArrayIndexDetails) index.getIndexDetails()).getElementList().stream()
.map(Index.ArrayIndexElement::getSourceIndicator).collect(Collectors.toList());
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/InvertedIndexResourceFactoryProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/InvertedIndexResourceFactoryProvider.java
index 7a50b40..efa9476 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/InvertedIndexResourceFactoryProvider.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/InvertedIndexResourceFactoryProvider.java
@@ -31,6 +31,7 @@
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.exceptions.ErrorCode;
+import org.apache.asterix.dataflow.data.common.SparseVectorBinaryTokenizerFactory;
import org.apache.asterix.formats.nontagged.NullIntrospector;
import org.apache.asterix.metadata.api.IResourceFactoryProvider;
import org.apache.asterix.metadata.declared.MetadataProvider;
@@ -45,7 +46,10 @@
import org.apache.hyracks.algebricks.data.ITypeTraitProvider;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
+import org.apache.hyracks.data.std.accessors.LongBinaryComparatorFactory;
import org.apache.hyracks.data.std.accessors.ShortBinaryComparatorFactory;
+import org.apache.hyracks.data.std.primitive.FloatPointable;
+import org.apache.hyracks.data.std.primitive.LongPointable;
import org.apache.hyracks.data.std.primitive.ShortPointable;
import org.apache.hyracks.storage.am.common.api.IMetadataPageManagerFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackFactory;
@@ -70,6 +74,10 @@
ARecordType recordType, ARecordType metaType, ILSMMergePolicyFactory mergePolicyFactory,
Map<String, String> mergePolicyProperties, ITypeTraits[] filterTypeTraits,
IBinaryComparatorFactory[] filterCmpFactories) throws AlgebricksException {
+ if (index.getIndexType() == IndexType.SPARSE_VECTOR) {
+ return getSparseVectorResourceFactory(mdProvider, dataset, index, recordType, metaType, mergePolicyFactory,
+ mergePolicyProperties, filterTypeTraits, filterCmpFactories);
+ }
// Get basic info
List<List<String>> primaryKeys = dataset.getPrimaryKeys();
Index.TextIndexDetails indexDetails = (Index.TextIndexDetails) index.getIndexDetails();
@@ -267,4 +275,83 @@
return NonTaggedFormatUtil.getBinaryTokenizerFactory(secondaryKeyType.getTypeTag(), indexType,
indexDetails.getGramLength());
}
+
+ private static IResourceFactory getSparseVectorResourceFactory(MetadataProvider mdProvider, Dataset dataset,
+ Index index, ARecordType recordType, ARecordType metaType, ILSMMergePolicyFactory mergePolicyFactory,
+ Map<String, String> mergePolicyProperties, ITypeTraits[] filterTypeTraits,
+ IBinaryComparatorFactory[] filterCmpFactories) throws AlgebricksException {
+ List<List<String>> primaryKeys = dataset.getPrimaryKeys();
+ Index.SparseVectorIndexDetails indexDetails = (Index.SparseVectorIndexDetails) index.getIndexDetails();
+ List<List<String>> secondaryKeys = indexDetails.getKeyFieldNames();
+ List<String> filterFieldName = DatasetUtil.getFilterField(dataset);
+ int numPrimaryKeys = primaryKeys.size();
+ int numSecondaryKeys = secondaryKeys.size();
+ if (dataset.getDatasetType() != DatasetType.INTERNAL) {
+ throw new CompilationException(ErrorCode.COMPILATION_INDEX_TYPE_NOT_SUPPORTED_FOR_DATASET_TYPE,
+ index.getIndexType().name(), dataset.getDatasetType());
+ }
+ if (numPrimaryKeys > 1) {
+ throw new AsterixException(
+ "Cannot create inverted index on " + dataset(PLURAL) + " with composite primary key.");
+ }
+ if (numSecondaryKeys > 1) {
+ throw new AsterixException("Cannot create composite inverted index on multiple fields.");
+ }
+ int numTokenKeyPairFields = 1 + numPrimaryKeys;
+ int[] invertedIndexFields = null;
+ int[] secondaryFilterFieldsForNonBulkLoadOps = null;
+ int[] invertedIndexFieldsForNonBulkLoadOps = null;
+ int[] secondaryFilterFields = null;
+ if (filterFieldName != null) {
+ invertedIndexFields = new int[numTokenKeyPairFields];
+ for (int i = 0; i < invertedIndexFields.length; i++) {
+ invertedIndexFields[i] = i;
+ }
+ secondaryFilterFieldsForNonBulkLoadOps = new int[filterFieldName.size()];
+ secondaryFilterFieldsForNonBulkLoadOps[0] = numSecondaryKeys + numPrimaryKeys;
+ invertedIndexFieldsForNonBulkLoadOps = new int[numSecondaryKeys + numPrimaryKeys];
+ for (int i = 0; i < invertedIndexFieldsForNonBulkLoadOps.length; i++) {
+ invertedIndexFieldsForNonBulkLoadOps[i] = i;
+ }
+ secondaryFilterFields = new int[filterFieldName.size()];
+ secondaryFilterFields[0] = numTokenKeyPairFields - numPrimaryKeys + numPrimaryKeys;
+ }
+ // Inverted list: primary key fields + float32 weight
+ ITypeTraits[] primaryTypeTraits = dataset.getPrimaryTypeTraits(mdProvider, recordType, metaType);
+ int numPkTraits = primaryTypeTraits.length - 1; // strip null sentinel
+ ITypeTraits[] invListTypeTraits = new ITypeTraits[numPkTraits + 1];
+ for (int i = 0; i < numPkTraits; i++) {
+ invListTypeTraits[i] = primaryTypeTraits[i];
+ }
+ invListTypeTraits[numPkTraits] = FloatPointable.TYPE_TRAITS;
+ IBinaryComparatorFactory[] invListCmpFactories =
+ dataset.getPrimaryComparatorFactories(mdProvider, recordType, metaType);
+ // Token: int64 dimension index
+ ITypeTraits[] tokenTypeTraits = new ITypeTraits[] { LongPointable.TYPE_TRAITS };
+ IBinaryComparatorFactory[] tokenCmpFactories =
+ new IBinaryComparatorFactory[] { LongBinaryComparatorFactory.INSTANCE };
+ IFullTextConfigEvaluatorFactory fullTextConfigEvaluatorFactory =
+ FullTextUtil.fetchFilterAndCreateConfigEvaluator(mdProvider, index.getDatabaseName(),
+ index.getDataverseName(), null);
+ IStorageComponentProvider storageComponentProvider = mdProvider.getStorageComponentProvider();
+ IStorageManager storageManager = storageComponentProvider.getStorageManager();
+ ILSMOperationTrackerFactory opTrackerFactory = dataset.getIndexOperationTrackerFactory(index);
+ ILSMIOOperationCallbackFactory ioOpCallbackFactory = dataset.getIoOperationCallbackFactory(index);
+ ILSMPageWriteCallbackFactory pageWriteCallbackFactory = dataset.getPageWriteCallbackFactory();
+ IMetadataPageManagerFactory metadataPageManagerFactory =
+ storageComponentProvider.getMetadataPageManagerFactory();
+ AsterixVirtualBufferCacheProvider vbcProvider = new AsterixVirtualBufferCacheProvider(dataset.getDatasetId());
+ ILSMIOOperationSchedulerProvider ioSchedulerProvider =
+ storageComponentProvider.getIoOperationSchedulerProvider();
+ double bloomFilterFalsePositiveRate = mdProvider.getStorageProperties().getBloomFilterFalsePositiveRate();
+ ITypeTraitProvider typeTraitProvider = mdProvider.getDataFormat().getTypeTraitProvider();
+ return new LSMInvertedIndexLocalResourceFactory(storageManager, invListTypeTraits, invListCmpFactories,
+ filterTypeTraits, filterCmpFactories, secondaryFilterFields, opTrackerFactory, ioOpCallbackFactory,
+ pageWriteCallbackFactory, metadataPageManagerFactory, vbcProvider, ioSchedulerProvider,
+ mergePolicyFactory, mergePolicyProperties, true, tokenTypeTraits, tokenCmpFactories,
+ SparseVectorBinaryTokenizerFactory.INSTANCE, fullTextConfigEvaluatorFactory, false, invertedIndexFields,
+ secondaryFilterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps,
+ bloomFilterFalsePositiveRate, typeTraitProvider.getTypeTrait(BuiltinType.ANULL),
+ NullIntrospector.INSTANCE);
+ }
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/KeyFieldTypeUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/KeyFieldTypeUtil.java
index fcbe298..a83379b 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/KeyFieldTypeUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/KeyFieldTypeUtil.java
@@ -234,6 +234,8 @@
case LENGTH_PARTITIONED_WORD_INVIX:
case LENGTH_PARTITIONED_NGRAM_INVIX:
return ((Index.TextIndexDetails) index.getIndexDetails()).getKeyFieldNames().size();
+ case SPARSE_VECTOR:
+ return ((Index.SparseVectorIndexDetails) index.getIndexDetails()).getKeyFieldNames().size();
case RTREE:
Index.ValueIndexDetails indexDetails = (Index.ValueIndexDetails) index.getIndexDetails();
List<Integer> keySourceIndicators = indexDetails.getKeyFieldSourceIndicators();
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/common/SparseVectorBinaryTokenizer.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/common/SparseVectorBinaryTokenizer.java
new file mode 100644
index 0000000..e31019c
--- /dev/null
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/common/SparseVectorBinaryTokenizer.java
@@ -0,0 +1,203 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.dataflow.data.common;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.asterix.dataflow.data.nontagged.serde.ADoubleSerializerDeserializer;
+import org.apache.asterix.dataflow.data.nontagged.serde.AFloatSerializerDeserializer;
+import org.apache.asterix.dataflow.data.nontagged.serde.AInt32SerializerDeserializer;
+import org.apache.asterix.dataflow.data.nontagged.serde.AInt64SerializerDeserializer;
+import org.apache.asterix.dataflow.data.nontagged.serde.AOrderedListSerializerDeserializer;
+import org.apache.asterix.formats.nontagged.SerializerDeserializerProvider;
+import org.apache.asterix.om.base.AMutableString;
+import org.apache.asterix.om.base.AString;
+import org.apache.asterix.om.pointables.ARecordVisitablePointable;
+import org.apache.asterix.om.pointables.base.IVisitablePointable;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.BuiltinType;
+import org.apache.asterix.om.utils.RecordUtil;
+import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+import org.apache.hyracks.storage.am.lsm.invertedindex.fulltext.TokenizerCategory;
+import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizer;
+import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.IToken;
+import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.TokenizerInfo.TokenizerType;
+
+/**
+ * Tokenizes a sparse vector open record ({@code { "indices": [...], "values": [...] }}) into a stream
+ * of {@link SparseVectorToken} instances, one per non-zero dimension.
+ */
+public class SparseVectorBinaryTokenizer implements IBinaryTokenizer {
+
+ private static final byte[] INDICES_FIELD_NAME;
+ private static final byte[] VALUES_FIELD_NAME;
+
+ static {
+ try {
+ INDICES_FIELD_NAME = serializeFieldName("indices");
+ VALUES_FIELD_NAME = serializeFieldName("values");
+ } catch (HyracksDataException e) {
+ throw new ExceptionInInitializerError(e);
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ private static byte[] serializeFieldName(String name) throws HyracksDataException {
+ ArrayBackedValueStorage storage = new ArrayBackedValueStorage();
+ ISerializerDeserializer<AString> strSerde =
+ SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.ASTRING);
+ strSerde.serialize(new AMutableString(name), storage.getDataOutput());
+ byte[] bytes = new byte[storage.getLength()];
+ System.arraycopy(storage.getByteArray(), storage.getStartOffset(), bytes, 0, storage.getLength());
+ return bytes;
+ }
+
+ private final ARecordVisitablePointable recordPointable =
+ new ARecordVisitablePointable(RecordUtil.FULLY_OPEN_RECORD_TYPE);
+ private final SparseVectorToken currentToken = new SparseVectorToken();
+
+ private byte[] indicesData;
+ private int indicesStart;
+ private byte[] valuesData;
+ private int valuesStart;
+ private int numTokens;
+ private int currentIdx;
+
+ @Override
+ public void reset(byte[] data, int start, int length) {
+ recordPointable.set(data, start, length);
+
+ IVisitablePointable indicesFv = findField(recordPointable, INDICES_FIELD_NAME);
+ IVisitablePointable valuesFv = findField(recordPointable, VALUES_FIELD_NAME);
+
+ if (!isValidList(indicesFv) || !isValidList(valuesFv)) {
+ numTokens = 0;
+ currentIdx = 0;
+ return;
+ }
+
+ indicesData = indicesFv.getByteArray();
+ indicesStart = indicesFv.getStartOffset();
+ valuesData = valuesFv.getByteArray();
+ valuesStart = valuesFv.getStartOffset();
+ numTokens = AOrderedListSerializerDeserializer.getNumberOfItems(indicesData, indicesStart);
+ currentIdx = 0;
+ }
+
+ @Override
+ public boolean hasNext() {
+ return currentIdx < numTokens;
+ }
+
+ @Override
+ public void next() {
+ try {
+ int idxPos = AOrderedListSerializerDeserializer.getItemOffset(indicesData, indicesStart, currentIdx);
+ long dimIndex = readIndex(indicesData, indicesStart, idxPos);
+ int valPos = AOrderedListSerializerDeserializer.getItemOffset(valuesData, valuesStart, currentIdx);
+ double weight = readValue(valuesData, valuesStart, valPos);
+ currentToken.set(dimIndex, weight);
+ currentIdx++;
+ } catch (HyracksDataException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ @Override
+ public IToken getToken() {
+ return currentToken;
+ }
+
+ @Override
+ public short getTokensCount() {
+ return (short) numTokens;
+ }
+
+ @Override
+ public TokenizerType getTokenizerType() {
+ return TokenizerType.LIST;
+ }
+
+ @Override
+ public TokenizerCategory getTokenizerCategory() {
+ return TokenizerCategory.WORD;
+ }
+
+ private static boolean isValidList(IVisitablePointable fv) {
+ return fv != null && fv.getByteArray()[fv.getStartOffset()] == ATypeTag.SERIALIZED_ORDEREDLIST_TYPE_TAG;
+ }
+
+ private static IVisitablePointable findField(ARecordVisitablePointable rec, byte[] targetFieldName) {
+ List<IVisitablePointable> fieldNames = rec.getFieldNames();
+ List<IVisitablePointable> fieldValues = rec.getFieldValues();
+ for (int i = 0, n = fieldNames.size(); i < n; i++) {
+ IVisitablePointable fn = fieldNames.get(i);
+ if (fn.getLength() == targetFieldName.length && Arrays.equals(fn.getByteArray(), fn.getStartOffset(),
+ fn.getStartOffset() + fn.getLength(), targetFieldName, 0, targetFieldName.length)) {
+ return fieldValues.get(i);
+ }
+ }
+ return null;
+ }
+
+ private static long readIndex(byte[] listData, int listOff, int itemPos) throws HyracksDataException {
+ byte elemTag = listData[listOff + 1];
+ if (elemTag == ATypeTag.SERIALIZED_INT32_TYPE_TAG) {
+ return AInt32SerializerDeserializer.getInt(listData, itemPos);
+ }
+ if (elemTag == ATypeTag.SERIALIZED_INT64_TYPE_TAG) {
+ return AInt64SerializerDeserializer.getLong(listData, itemPos);
+ }
+ if (elemTag == ATypeTag.SERIALIZED_ANY_TYPE_TAG) {
+ byte itemTag = listData[itemPos];
+ if (itemTag == ATypeTag.SERIALIZED_INT32_TYPE_TAG) {
+ return AInt32SerializerDeserializer.getInt(listData, itemPos + 1);
+ }
+ if (itemTag == ATypeTag.SERIALIZED_INT64_TYPE_TAG) {
+ return AInt64SerializerDeserializer.getLong(listData, itemPos + 1);
+ }
+ }
+ throw HyracksDataException.create(
+ new IllegalArgumentException("sparse vector indices: expected int32 or int64, got tag " + elemTag));
+ }
+
+ private static double readValue(byte[] listData, int listOff, int itemPos) throws HyracksDataException {
+ byte elemTag = listData[listOff + 1];
+ if (elemTag == ATypeTag.SERIALIZED_FLOAT_TYPE_TAG) {
+ return AFloatSerializerDeserializer.getFloat(listData, itemPos);
+ }
+ if (elemTag == ATypeTag.SERIALIZED_DOUBLE_TYPE_TAG) {
+ return ADoubleSerializerDeserializer.getDouble(listData, itemPos);
+ }
+ if (elemTag == ATypeTag.SERIALIZED_ANY_TYPE_TAG) {
+ byte itemTag = listData[itemPos];
+ if (itemTag == ATypeTag.SERIALIZED_FLOAT_TYPE_TAG) {
+ return AFloatSerializerDeserializer.getFloat(listData, itemPos + 1);
+ }
+ if (itemTag == ATypeTag.SERIALIZED_DOUBLE_TYPE_TAG) {
+ return ADoubleSerializerDeserializer.getDouble(listData, itemPos + 1);
+ }
+ }
+ throw HyracksDataException.create(
+ new IllegalArgumentException("sparse vector values: expected float or double, got tag " + elemTag));
+ }
+}
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/common/SparseVectorBinaryTokenizerFactory.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/common/SparseVectorBinaryTokenizerFactory.java
new file mode 100644
index 0000000..3a22504
--- /dev/null
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/common/SparseVectorBinaryTokenizerFactory.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.dataflow.data.common;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.IJsonSerializable;
+import org.apache.hyracks.api.io.IPersistedResourceRegistry;
+import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizer;
+import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizerFactory;
+
+import com.fasterxml.jackson.databind.JsonNode;
+
+public class SparseVectorBinaryTokenizerFactory implements IBinaryTokenizerFactory {
+
+ public static final SparseVectorBinaryTokenizerFactory INSTANCE = new SparseVectorBinaryTokenizerFactory();
+ private static final long serialVersionUID = 1L;
+
+ private SparseVectorBinaryTokenizerFactory() {
+ }
+
+ @Override
+ public IBinaryTokenizer createTokenizer() {
+ return new SparseVectorBinaryTokenizer();
+ }
+
+ @Override
+ public JsonNode toJson(IPersistedResourceRegistry registry) throws HyracksDataException {
+ return registry.getClassIdentifier(getClass(), serialVersionUID);
+ }
+
+ public static IJsonSerializable fromJson(IPersistedResourceRegistry registry, JsonNode json) {
+ return INSTANCE;
+ }
+}
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/common/SparseVectorToken.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/common/SparseVectorToken.java
new file mode 100644
index 0000000..56c9a87
--- /dev/null
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/common/SparseVectorToken.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.dataflow.data.common;
+
+import java.io.IOException;
+
+import org.apache.hyracks.data.std.util.GrowableArray;
+import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.IToken;
+import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.IWeightedToken;
+
+/**
+ * A token for sparse-vector indexes. Carries a dimension index (serialized as int64, the BTree key)
+ * and a posting weight (used by WAND; serialized as float32 on disk by the bulk-loader).
+ */
+public class SparseVectorToken implements IToken, IWeightedToken {
+
+ private long dimIndex;
+ private double weight;
+
+ public void set(long dimIndex, double weight) {
+ this.dimIndex = dimIndex;
+ this.weight = weight;
+ }
+
+ @Override
+ public long getIndex() {
+ return dimIndex;
+ }
+
+ @Override
+ public double getWeight() {
+ return weight;
+ }
+
+ @Override
+ public void serializeToken(GrowableArray out) throws IOException {
+ out.getDataOutput().writeLong(dimIndex);
+ }
+
+ @Override
+ public byte[] getData() {
+ return null;
+ }
+
+ @Override
+ public int getStartOffset() {
+ return 0;
+ }
+
+ @Override
+ public int getEndOffset() {
+ return Long.BYTES;
+ }
+
+ @Override
+ public int getTokenLength() {
+ return Long.BYTES;
+ }
+
+ @Override
+ public void reset(byte[] data, int startOffset, int endOffset, int tokenLength, int tokenCount) {
+ // Not used; dimension and weight are set directly via set().
+ }
+
+ @Override
+ public void serializeTokenCount(GrowableArray out) throws IOException {
+ // Not applicable for sparse-vector tokens.
+ }
+}
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java
index f98e20a..ea32d8e 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java
@@ -857,6 +857,8 @@
public static final FunctionIdentifier SPARSE_VECTOR_DOT_PRODUCT =
FunctionConstants.newAsterix("sparse-vector-dot-product", 2);
+ public static final FunctionIdentifier SPARSE_VECTOR_DOT_PRODUCT_CHECK =
+ FunctionConstants.newAsterix("sparse-vector-dot-product-check", 3);
public static final FunctionIdentifier SIMILARITY_JACCARD = FunctionConstants.newAsterix("similarity-jaccard", 2);
public static final FunctionIdentifier SIMILARITY_JACCARD_CHECK =
@@ -1930,6 +1932,7 @@
// Similarity functions
addFunction(EDIT_DISTANCE_CONTAINS, OrderedListOfAnyTypeComputer.INSTANCE, true);
addFunction(SPARSE_VECTOR_DOT_PRODUCT, ADoubleTypeComputer.INSTANCE, true);
+ addPrivateFunction(SPARSE_VECTOR_DOT_PRODUCT_CHECK, OrderedListOfAnyTypeComputer.INSTANCE, true);
addFunction(SIMILARITY_JACCARD, AFloatTypeComputer.INSTANCE, true);
addFunction(SIMILARITY_JACCARD_CHECK, OrderedListOfAnyTypeComputer.INSTANCE, true);
addPrivateFunction(SIMILARITY_JACCARD_SORTED, AFloatTypeComputer.INSTANCE, true);
@@ -3258,6 +3261,7 @@
similarityFunctions.add(EDIT_DISTANCE_CHECK);
similarityFunctions.add(EDIT_DISTANCE_CONTAINS);
similarityFunctions.add(SPARSE_VECTOR_DOT_PRODUCT);
+ similarityFunctions.add(SPARSE_VECTOR_DOT_PRODUCT_CHECK);
}
public static boolean isSimilarityFunction(FunctionIdentifier fi) {
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/sparsevector/SparseVectorDotProductCheckDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/sparsevector/SparseVectorDotProductCheckDescriptor.java
new file mode 100644
index 0000000..c46f62e
--- /dev/null
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/sparsevector/SparseVectorDotProductCheckDescriptor.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.runtime.evaluators.functions.sparsevector;
+
+import java.io.Serial;
+
+import org.apache.asterix.common.annotations.MissingNullInOutFunction;
+import org.apache.asterix.om.functions.BuiltinFunctions;
+import org.apache.asterix.om.functions.IFunctionDescriptor;
+import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
+import org.apache.asterix.om.functions.IFunctionTypeInferer;
+import org.apache.asterix.om.typecomputer.impl.TypeComputeUtils;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.om.utils.RecordUtil;
+import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
+import org.apache.asterix.runtime.functions.FunctionTypeInferers;
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import org.apache.hyracks.api.context.IEvaluatorContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+@MissingNullInOutFunction
+public class SparseVectorDotProductCheckDescriptor extends AbstractScalarFunctionDynamicDescriptor {
+
+ @Serial
+ private static final long serialVersionUID = 1L;
+
+ public static final IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() {
+ @Override
+ public IFunctionDescriptor createFunctionDescriptor() {
+ return new SparseVectorDotProductCheckDescriptor();
+ }
+
+ @Override
+ public IFunctionTypeInferer createFunctionTypeInferer() {
+ return FunctionTypeInferers.SET_ARGUMENTS_TYPE;
+ }
+ };
+
+ private ARecordType arg0RecType = RecordUtil.FULLY_OPEN_RECORD_TYPE;
+ private ARecordType arg1RecType = RecordUtil.FULLY_OPEN_RECORD_TYPE;
+
+ @Override
+ public void setImmutableStates(Object... states) {
+ if (states.length < 2) {
+ return;
+ }
+ ARecordType t0 = TypeComputeUtils.extractRecordType((IAType) states[0]);
+ ARecordType t1 = TypeComputeUtils.extractRecordType((IAType) states[1]);
+ arg0RecType = t0 != null ? t0 : RecordUtil.FULLY_OPEN_RECORD_TYPE;
+ arg1RecType = t1 != null ? t1 : RecordUtil.FULLY_OPEN_RECORD_TYPE;
+ }
+
+ @Override
+ public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args) {
+ final ARecordType capturedArg0RecType = arg0RecType;
+ final ARecordType capturedArg1RecType = arg1RecType;
+ return new IScalarEvaluatorFactory() {
+ @Serial
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public IScalarEvaluator createScalarEvaluator(IEvaluatorContext ctx) throws HyracksDataException {
+ return new SparseVectorDotProductCheckEvaluator(args, ctx, capturedArg0RecType, capturedArg1RecType);
+ }
+ };
+ }
+
+ @Override
+ public FunctionIdentifier getIdentifier() {
+ return BuiltinFunctions.SPARSE_VECTOR_DOT_PRODUCT_CHECK;
+ }
+}
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/sparsevector/SparseVectorDotProductCheckEvaluator.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/sparsevector/SparseVectorDotProductCheckEvaluator.java
new file mode 100644
index 0000000..44ae902
--- /dev/null
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/sparsevector/SparseVectorDotProductCheckEvaluator.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.runtime.evaluators.functions.sparsevector;
+
+import java.io.IOException;
+
+import org.apache.asterix.builders.OrderedListBuilder;
+import org.apache.asterix.dataflow.data.nontagged.serde.ADoubleSerializerDeserializer;
+import org.apache.asterix.dataflow.data.nontagged.serde.AFloatSerializerDeserializer;
+import org.apache.asterix.formats.nontagged.SerializerDeserializerProvider;
+import org.apache.asterix.om.base.ABoolean;
+import org.apache.asterix.om.base.ADouble;
+import org.apache.asterix.om.base.AMutableDouble;
+import org.apache.asterix.om.functions.BuiltinFunctions;
+import org.apache.asterix.om.types.AOrderedListType;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.BuiltinType;
+import org.apache.asterix.runtime.evaluators.functions.PointableHelper;
+import org.apache.asterix.runtime.exceptions.TypeMismatchException;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import org.apache.hyracks.api.context.IEvaluatorContext;
+import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IPointable;
+import org.apache.hyracks.data.std.primitive.VoidPointable;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+import org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
+
+/**
+ * Evaluates {@code sparse-vector-dot-product-check(vec0, vec1, threshold)}, returning
+ * an ordered list {@code [boolean, double]}: whether the dot product meets the threshold,
+ * and the dot product value (0.0 if not).
+ */
+public class SparseVectorDotProductCheckEvaluator implements IScalarEvaluator {
+
+ private final SparseVectorDotProductEvaluator dotProductEval;
+ private final IScalarEvaluator threshEval;
+
+ private final IPointable dotProductResult = new VoidPointable();
+ private final IPointable threshPtr = new VoidPointable();
+
+ private final OrderedListBuilder listBuilder = new OrderedListBuilder();
+ private final ArrayBackedValueStorage inputVal = new ArrayBackedValueStorage();
+ private final ArrayBackedValueStorage resultStorage = new ArrayBackedValueStorage();
+ private final AOrderedListType listType = new AOrderedListType(BuiltinType.ANY, "list");
+
+ @SuppressWarnings("unchecked")
+ private final ISerializerDeserializer<ABoolean> booleanSerde =
+ SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.ABOOLEAN);
+ @SuppressWarnings("unchecked")
+ private final ISerializerDeserializer<ADouble> doubleSerde =
+ SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.ADOUBLE);
+ private final AMutableDouble aDouble = new AMutableDouble(0);
+
+ public SparseVectorDotProductCheckEvaluator(IScalarEvaluatorFactory[] args, IEvaluatorContext ctx,
+ ARecordType arg0RecType, ARecordType arg1RecType) throws HyracksDataException {
+ dotProductEval = new SparseVectorDotProductEvaluator(new IScalarEvaluatorFactory[] { args[0], args[1] }, ctx,
+ arg0RecType, arg1RecType);
+ threshEval = args[2].createScalarEvaluator(ctx);
+ }
+
+ @Override
+ public void evaluate(IFrameTupleReference tuple, IPointable result) throws HyracksDataException {
+ dotProductEval.evaluate(tuple, dotProductResult);
+
+ byte dotTag = dotProductResult.getByteArray()[dotProductResult.getStartOffset()];
+ if (dotTag == ATypeTag.SERIALIZED_NULL_TYPE_TAG || dotTag == ATypeTag.SERIALIZED_MISSING_TYPE_TAG) {
+ result.set(dotProductResult);
+ return;
+ }
+
+ threshEval.evaluate(tuple, threshPtr);
+ if (PointableHelper.checkAndSetMissingOrNull(result, threshPtr)) {
+ return;
+ }
+
+ double dotProduct = ADoubleSerializerDeserializer.getDouble(dotProductResult.getByteArray(),
+ dotProductResult.getStartOffset() + 1);
+
+ byte threshTag = threshPtr.getByteArray()[threshPtr.getStartOffset()];
+ double threshold;
+ if (threshTag == ATypeTag.SERIALIZED_DOUBLE_TYPE_TAG) {
+ threshold =
+ ADoubleSerializerDeserializer.getDouble(threshPtr.getByteArray(), threshPtr.getStartOffset() + 1);
+ } else if (threshTag == ATypeTag.SERIALIZED_FLOAT_TYPE_TAG) {
+ threshold = AFloatSerializerDeserializer.getFloat(threshPtr.getByteArray(), threshPtr.getStartOffset() + 1);
+ } else {
+ throw new TypeMismatchException(BuiltinFunctions.SPARSE_VECTOR_DOT_PRODUCT_CHECK, 2, threshTag,
+ ATypeTag.SERIALIZED_DOUBLE_TYPE_TAG);
+ }
+
+ boolean matches = dotProduct >= threshold;
+ try {
+ resultStorage.reset();
+ listBuilder.reset(listType);
+
+ inputVal.reset();
+ booleanSerde.serialize(matches ? ABoolean.TRUE : ABoolean.FALSE, inputVal.getDataOutput());
+ listBuilder.addItem(inputVal);
+
+ inputVal.reset();
+ aDouble.setValue(matches ? dotProduct : 0.0);
+ doubleSerde.serialize(aDouble, inputVal.getDataOutput());
+ listBuilder.addItem(inputVal);
+
+ listBuilder.write(resultStorage.getDataOutput(), true);
+ } catch (IOException e) {
+ throw HyracksDataException.create(e);
+ }
+ result.set(resultStorage);
+ }
+}
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/functions/FunctionCollection.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/functions/FunctionCollection.java
index 6f20370..660b37d 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/functions/FunctionCollection.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/functions/FunctionCollection.java
@@ -580,6 +580,7 @@
import org.apache.asterix.runtime.evaluators.functions.records.RecordTransformDescriptor;
import org.apache.asterix.runtime.evaluators.functions.records.RecordUnwrapDescriptor;
import org.apache.asterix.runtime.evaluators.functions.records.RecordValuesDescriptor;
+import org.apache.asterix.runtime.evaluators.functions.sparsevector.SparseVectorDotProductCheckDescriptor;
import org.apache.asterix.runtime.evaluators.functions.sparsevector.SparseVectorDotProductDescriptor;
import org.apache.asterix.runtime.evaluators.functions.temporal.AdjustDateTimeForTimeZoneDescriptor;
import org.apache.asterix.runtime.evaluators.functions.temporal.AdjustTimeForTimeZoneDescriptor;
@@ -1074,6 +1075,7 @@
// Similarity Functions
fc.add(SparseVectorDotProductDescriptor.FACTORY);
+ fc.add(SparseVectorDotProductCheckDescriptor.FACTORY);
// Comparisons.
fc.add(EqualsDescriptor.FACTORY);
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexSearchOperatorNodePushable.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexSearchOperatorNodePushable.java
index 2e1f29d..95b66ca 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexSearchOperatorNodePushable.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexSearchOperatorNodePushable.java
@@ -19,12 +19,16 @@
package org.apache.hyracks.storage.am.lsm.invertedindex.dataflow;
+import java.util.ArrayList;
+import java.util.List;
+
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.value.IMissingWriterFactory;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.util.HyracksConstants;
import org.apache.hyracks.dataflow.common.data.accessors.FrameTupleReference;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
import org.apache.hyracks.dataflow.common.utils.TaskUtil;
import org.apache.hyracks.dataflow.std.buffermanager.DeallocatableFramePool;
import org.apache.hyracks.dataflow.std.buffermanager.FramePoolBackedFrameBufferManager;
@@ -42,6 +46,8 @@
import org.apache.hyracks.storage.am.lsm.invertedindex.search.WandSearchPredicate;
import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizer;
import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizerFactory;
+import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.IToken;
+import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.IWeightedToken;
import org.apache.hyracks.storage.common.IIndex;
import org.apache.hyracks.storage.common.IIndexAccessParameters;
import org.apache.hyracks.storage.common.ISearchPredicate;
@@ -124,8 +130,18 @@
}
private double[] extractQueryWeights(InvertedIndexSearchPredicate searchPred) {
- // TODO(Patch 2): iterate tokenized query and extract IWeightedToken.getWeight()
- return null;
+ ITupleReference queryTuple = searchPred.getQueryTuple();
+ int fieldIdx = searchPred.getQueryFieldIndex();
+ IFullTextConfigEvaluator evaluator = searchPred.getFullTextConfigEvaluator();
+ evaluator.reset(queryTuple.getFieldData(fieldIdx), queryTuple.getFieldStart(fieldIdx),
+ queryTuple.getFieldLength(fieldIdx));
+ List<Double> weights = new ArrayList<>();
+ while (evaluator.hasNext()) {
+ evaluator.next();
+ IToken token = evaluator.getToken();
+ weights.add(token instanceof IWeightedToken ? ((IWeightedToken) token).getWeight() : 1.0);
+ }
+ return weights.stream().mapToDouble(Double::doubleValue).toArray();
}
@Override
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21288?usp=email
To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings?usp=email
Gerrit-MessageType: newchange
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Change-Id: I5a2665290da14a8da83d794c7ee1f68514186e01
Gerrit-Change-Number: 21288
Gerrit-PatchSet: 1
Gerrit-Owner: Savyasach Reddy <es...@gmail.com>