You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@solr.apache.org by ab...@apache.org on 2023/04/17 13:58:10 UTC
[solr] branch branch_9x updated: SOLR-16674: Introduce dense vector byte encoding (#1435)
This is an automated email from the ASF dual-hosted git repository.
abenedetti pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/branch_9x by this push:
new ad4c93e34e3 SOLR-16674: Introduce dense vector byte encoding (#1435)
ad4c93e34e3 is described below
commit ad4c93e34e31a73b547c0a4007b926905cbac2de
Author: Elia Porciani <e....@sease.io>
AuthorDate: Mon Apr 17 15:53:11 2023 +0200
SOLR-16674: Introduce dense vector byte encoding (#1435)
---
solr/CHANGES.txt | 2 +
.../org/apache/solr/schema/DenseVectorField.java | 209 +++++++-----
.../org/apache/solr/search/neural/KnnQParser.java | 43 +--
.../solr/util/vector/ByteDenseVectorParser.java | 53 +++
.../apache/solr/util/vector/DenseVectorParser.java | 119 +++++++
.../solr/util/vector/FloatDenseVectorParser.java | 53 +++
.../org/apache/solr/util/vector/package-info.java | 19 ++
.../solr/collection1/conf/schema-densevector.xml | 13 +-
.../apache/solr/schema/DenseVectorFieldTest.java | 362 +++++++++++++++++++--
.../apache/solr/search/neural/KnnQParserTest.java | 61 +++-
.../apache/solr/update/DocumentBuilderTest.java | 7 +-
.../query-guide/pages/dense-vector-search.adoc | 12 +
12 files changed, 782 insertions(+), 171 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index fba7bf3a1f9..dfd78bcb0ce 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -15,6 +15,8 @@ New Features
* SOLR-16742: Produce a slim Solr binary and docker image, without the Prometheus Exporter and Solr Modules. (Houston Putman)
+* SOLR-16674: Introduced support for byte vector encoding in DenseVectorField and KnnQParser (Elia Porciani via Alessandro Benedetti).
+
Improvements
---------------------
diff --git a/solr/core/src/java/org/apache/solr/schema/DenseVectorField.java b/solr/core/src/java/org/apache/solr/schema/DenseVectorField.java
index c59b0ead0bb..b57dc39d430 100644
--- a/solr/core/src/java/org/apache/solr/schema/DenseVectorField.java
+++ b/solr/core/src/java/org/apache/solr/schema/DenseVectorField.java
@@ -24,17 +24,25 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import java.util.Map;
-import org.apache.lucene.document.KnnVectorField;
+import org.apache.lucene.document.KnnByteVectorField;
+import org.apache.lucene.document.KnnFloatVectorField;
+import org.apache.lucene.document.StoredField;
import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.VectorEncoding;
import org.apache.lucene.index.VectorSimilarityFunction;
import org.apache.lucene.queries.function.ValueSource;
-import org.apache.lucene.search.KnnVectorQuery;
+import org.apache.lucene.search.KnnByteVectorQuery;
+import org.apache.lucene.search.KnnFloatVectorQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.SortField;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.hnsw.HnswGraph;
import org.apache.solr.common.SolrException;
import org.apache.solr.search.QParser;
import org.apache.solr.uninverting.UninvertingReader;
+import org.apache.solr.util.vector.ByteDenseVectorParser;
+import org.apache.solr.util.vector.DenseVectorParser;
+import org.apache.solr.util.vector.FloatDenseVectorParser;
/**
* Provides a field type to support Lucene's {@link org.apache.lucene.document.KnnVectorField}. See
@@ -46,19 +54,17 @@ import org.apache.solr.uninverting.UninvertingReader;
*/
public class DenseVectorField extends FloatPointField {
public static final String HNSW_ALGORITHM = "hnsw";
-
public static final String DEFAULT_KNN_ALGORITHM = HNSW_ALGORITHM;
static final String KNN_VECTOR_DIMENSION = "vectorDimension";
- static final String KNN_SIMILARITY_FUNCTION = "similarityFunction";
-
static final String KNN_ALGORITHM = "knnAlgorithm";
static final String HNSW_MAX_CONNECTIONS = "hnswMaxConnections";
static final String HNSW_BEAM_WIDTH = "hnswBeamWidth";
-
+ static final String VECTOR_ENCODING = "vectorEncoding";
+ static final VectorEncoding DEFAULT_VECTOR_ENCODING = VectorEncoding.FLOAT32;
+ static final String KNN_SIMILARITY_FUNCTION = "similarityFunction";
+ static final VectorSimilarityFunction DEFAULT_SIMILARITY = VectorSimilarityFunction.EUCLIDEAN;
private int dimension;
private VectorSimilarityFunction similarityFunction;
- private VectorSimilarityFunction DEFAULT_SIMILARITY = VectorSimilarityFunction.EUCLIDEAN;
-
private String knnAlgorithm;
/**
* This parameter is coupled with the hnsw algorithm. Controls how many of the nearest neighbor
@@ -71,27 +77,37 @@ public class DenseVectorField extends FloatPointField {
*/
private int hnswBeamWidth;
+ /**
+ * Encoding for vector value representation. The possible values are FLOAT32 or BYTE. The default
+ * encoding is FLOAT32
+ */
+ private VectorEncoding vectorEncoding;
+
public DenseVectorField() {
super();
}
public DenseVectorField(int dimension) {
- super();
- this.dimension = dimension;
- this.similarityFunction = DEFAULT_SIMILARITY;
+ this(dimension, DEFAULT_SIMILARITY, DEFAULT_VECTOR_ENCODING);
+ }
+
+ public DenseVectorField(int dimension, VectorEncoding vectorEncoding) {
+ this(dimension, DEFAULT_SIMILARITY, vectorEncoding);
}
- public DenseVectorField(int dimension, VectorSimilarityFunction similarityFunction) {
+ public DenseVectorField(
+ int dimension, VectorSimilarityFunction similarityFunction, VectorEncoding vectorEncoding) {
super();
this.dimension = dimension;
this.similarityFunction = similarityFunction;
+ this.vectorEncoding = vectorEncoding;
}
@Override
public void init(IndexSchema schema, Map<String, String> args) {
this.dimension =
ofNullable(args.get(KNN_VECTOR_DIMENSION))
- .map(value -> Integer.parseInt(value))
+ .map(Integer::parseInt)
.orElseThrow(
() ->
new SolrException(
@@ -106,19 +122,20 @@ public class DenseVectorField extends FloatPointField {
args.remove(KNN_SIMILARITY_FUNCTION);
this.knnAlgorithm = args.getOrDefault(KNN_ALGORITHM, DEFAULT_KNN_ALGORITHM);
-
args.remove(KNN_ALGORITHM);
+ this.vectorEncoding =
+ ofNullable(args.get(VECTOR_ENCODING))
+ .map(value -> VectorEncoding.valueOf(value.toUpperCase(Locale.ROOT)))
+ .orElse(DEFAULT_VECTOR_ENCODING);
+ args.remove(VECTOR_ENCODING);
+
this.hnswMaxConn =
- ofNullable(args.get(HNSW_MAX_CONNECTIONS))
- .map(value -> Integer.parseInt(value))
- .orElse(DEFAULT_MAX_CONN);
+ ofNullable(args.get(HNSW_MAX_CONNECTIONS)).map(Integer::parseInt).orElse(DEFAULT_MAX_CONN);
args.remove(HNSW_MAX_CONNECTIONS);
this.hnswBeamWidth =
- ofNullable(args.get(HNSW_BEAM_WIDTH))
- .map(value -> Integer.parseInt(value))
- .orElse(DEFAULT_BEAM_WIDTH);
+ ofNullable(args.get(HNSW_BEAM_WIDTH)).map(Integer::parseInt).orElse(DEFAULT_BEAM_WIDTH);
args.remove(HNSW_BEAM_WIDTH);
this.properties &= ~MULTIVALUED;
@@ -147,6 +164,10 @@ public class DenseVectorField extends FloatPointField {
return hnswBeamWidth;
}
+ public VectorEncoding getVectorEncoding() {
+ return vectorEncoding;
+ }
+
@Override
public void checkSchemaField(final SchemaField field) throws SolrException {
super.checkSchemaField(field);
@@ -165,38 +186,70 @@ public class DenseVectorField extends FloatPointField {
@Override
public List<IndexableField> createFields(SchemaField field, Object value) {
- ArrayList<IndexableField> fields = new ArrayList<>();
- float[] parsedVector;
try {
- parsedVector = parseVector(value);
+ ArrayList<IndexableField> fields = new ArrayList<>();
+ DenseVectorParser vectorBuilder =
+ getVectorBuilder(value, DenseVectorParser.BuilderPhase.INDEX);
+
+ if (field.indexed()) {
+ fields.add(createField(field, vectorBuilder));
+ }
+ if (field.stored()) {
+ switch (vectorEncoding) {
+ case FLOAT32:
+ fields.ensureCapacity(vectorBuilder.getFloatVector().length + 1);
+ for (float vectorElement : vectorBuilder.getFloatVector()) {
+ fields.add(getStoredField(field, vectorElement));
+ }
+ break;
+ case BYTE:
+ fields.add(new StoredField(field.getName(), vectorBuilder.getByteVector()));
+ break;
+ }
+ }
+ return fields;
} catch (RuntimeException e) {
throw new SolrException(
SolrException.ErrorCode.SERVER_ERROR,
- "Error while creating field '"
- + field
- + "' from value '"
- + value
- + "', expected format:'[f1, f2, f3...fn]' e.g. [1.0, 3.4, 5.6]",
+ "Error while creating field '" + field + "' from value '" + value + "'",
e);
}
+ }
- if (field.indexed()) {
- fields.add(createField(field, parsedVector));
- }
- if (field.stored()) {
- fields.ensureCapacity(parsedVector.length + 1);
- for (float vectorElement : parsedVector) {
- fields.add(getStoredField(field, vectorElement));
- }
+ @Override
+ public IndexableField createField(SchemaField field, Object vectorValue) {
+ if (vectorValue == null) return null;
+ DenseVectorParser vectorBuilder = (DenseVectorParser) vectorValue;
+ switch (vectorEncoding) {
+ case BYTE:
+ return new KnnByteVectorField(
+ field.getName(), vectorBuilder.getByteVector(), similarityFunction);
+ case FLOAT32:
+ return new KnnFloatVectorField(
+ field.getName(), vectorBuilder.getFloatVector(), similarityFunction);
+ default:
+ throw new SolrException(
+ SolrException.ErrorCode.SERVER_ERROR,
+ "Unexpected state. Vector Encoding: " + vectorEncoding);
}
- return fields;
}
@Override
- public IndexableField createField(SchemaField field, Object parsedVector) {
- if (parsedVector == null) return null;
- float[] typedVector = (float[]) parsedVector;
- return new KnnVectorField(field.getName(), typedVector, similarityFunction);
+ public Object toObject(IndexableField f) {
+ if (vectorEncoding.equals(VectorEncoding.BYTE)) {
+ BytesRef bytesRef = f.binaryValue();
+ if (bytesRef != null) {
+ List<Number> ret = new ArrayList<>(dimension);
+ for (byte b : bytesRef.bytes) {
+ ret.add((int) b);
+ }
+ return ret;
+ } else {
+ throw new AssertionError("Unexpected state. Field: '" + f + "'");
+ }
+ }
+
+ return super.toObject(f);
}
/**
@@ -205,53 +258,19 @@ public class DenseVectorField extends FloatPointField {
* org.apache.solr.handler.loader.CSVLoader} produces an ArrayList of String - {@link
* org.apache.solr.handler.loader.JsonLoader} produces an ArrayList of Double - {@link
* org.apache.solr.handler.loader.JavabinLoader} produces an ArrayList of Float
- *
- * @param inputValue - An {@link ArrayList} containing the elements of the vector
- * @return the vector parsed
*/
- float[] parseVector(Object inputValue) {
- if (!(inputValue instanceof List)) {
- throw new SolrException(
- SolrException.ErrorCode.BAD_REQUEST,
- "incorrect vector format."
- + " The expected format is an array :'[f1,f2..f3]' where each element f is a float");
+ public DenseVectorParser getVectorBuilder(
+ Object inputValue, DenseVectorParser.BuilderPhase phase) {
+ switch (vectorEncoding) {
+ case FLOAT32:
+ return new FloatDenseVectorParser(dimension, inputValue, phase);
+ case BYTE:
+ return new ByteDenseVectorParser(dimension, inputValue, phase);
+ default:
+ throw new SolrException(
+ SolrException.ErrorCode.SERVER_ERROR,
+ "Unexpected state. Vector Encoding: " + vectorEncoding);
}
- List<?> inputVector = (List<?>) inputValue;
- if (inputVector.size() != dimension) {
- throw new SolrException(
- SolrException.ErrorCode.BAD_REQUEST,
- "incorrect vector dimension."
- + " The vector value has size "
- + inputVector.size()
- + " while it is expected a vector with size "
- + dimension);
- }
-
- float[] vector = new float[dimension];
- if (inputVector.get(0) instanceof CharSequence) {
- for (int i = 0; i < dimension; i++) {
- try {
- vector[i] = Float.parseFloat(inputVector.get(i).toString());
- } catch (NumberFormatException e) {
- throw new SolrException(
- SolrException.ErrorCode.BAD_REQUEST,
- "incorrect vector element: '"
- + inputVector.get(i)
- + "'. The expected format is:'[f1,f2..f3]' where each element f is a float");
- }
- }
- } else if (inputVector.get(0) instanceof Number) {
- for (int i = 0; i < dimension; i++) {
- vector[i] = ((Number) inputVector.get(i)).floatValue();
- }
- } else {
- throw new SolrException(
- SolrException.ErrorCode.BAD_REQUEST,
- "incorrect vector format."
- + " The expected format is an array :'[f1,f2..f3]' where each element f is a float");
- }
-
- return vector;
}
@Override
@@ -267,8 +286,22 @@ public class DenseVectorField extends FloatPointField {
}
public Query getKnnVectorQuery(
- String fieldName, float[] vectorToSearch, int topK, Query filterQuery) {
- return new KnnVectorQuery(fieldName, vectorToSearch, topK, filterQuery);
+ String fieldName, String vectorToSearch, int topK, Query filterQuery) {
+
+ DenseVectorParser vectorBuilder =
+ getVectorBuilder(vectorToSearch, DenseVectorParser.BuilderPhase.QUERY);
+
+ switch (vectorEncoding) {
+ case FLOAT32:
+ return new KnnFloatVectorQuery(
+ fieldName, vectorBuilder.getFloatVector(), topK, filterQuery);
+ case BYTE:
+ return new KnnByteVectorQuery(fieldName, vectorBuilder.getByteVector(), topK, filterQuery);
+ default:
+ throw new SolrException(
+ SolrException.ErrorCode.SERVER_ERROR,
+ "Unexpected state. Vector Encoding: " + vectorEncoding);
+ }
}
/**
diff --git a/solr/core/src/java/org/apache/solr/search/neural/KnnQParser.java b/solr/core/src/java/org/apache/solr/search/neural/KnnQParser.java
index 26e3d3942eb..76af8a0033c 100644
--- a/solr/core/src/java/org/apache/solr/search/neural/KnnQParser.java
+++ b/solr/core/src/java/org/apache/solr/search/neural/KnnQParser.java
@@ -76,10 +76,9 @@ public class KnnQParser extends QParser {
}
DenseVectorField denseVectorType = (DenseVectorField) fieldType;
- float[] parsedVectorToSearch = parseVector(vectorToSearch, denseVectorType.getDimension());
return denseVectorType.getKnnVectorQuery(
- schemaField.getName(), parsedVectorToSearch, topK, getFilterQuery());
+ schemaField.getName(), vectorToSearch, topK, getFilterQuery());
}
private Query getFilterQuery() throws SolrException, SyntaxError {
@@ -99,44 +98,4 @@ public class KnnQParser extends QParser {
}
return null;
}
-
- /**
- * Parses a String vector.
- *
- * @param value with format: [f1, f2, f3, f4...fn]
- * @return a float array
- */
- private static float[] parseVector(String value, int dimension) {
-
- if (!value.startsWith("[") || !value.endsWith("]")) {
- throw new SolrException(
- SolrException.ErrorCode.BAD_REQUEST,
- "incorrect vector format."
- + " The expected format is:'[f1,f2..f3]' where each element f is a float");
- }
-
- String[] elements = value.substring(1, value.length() - 1).split(",");
- if (elements.length != dimension) {
- throw new SolrException(
- SolrException.ErrorCode.BAD_REQUEST,
- "incorrect vector dimension."
- + " The vector value has size "
- + elements.length
- + " while it is expected a vector with size "
- + dimension);
- }
- float[] vector = new float[dimension];
- for (int i = 0; i < dimension; i++) {
- try {
- vector[i] = Float.parseFloat(elements[i]);
- } catch (NumberFormatException e) {
- throw new SolrException(
- SolrException.ErrorCode.BAD_REQUEST,
- "incorrect vector element: '"
- + elements[i]
- + "'. The expected format is:'[f1,f2..f3]' where each element f is a float");
- }
- }
- return vector;
- }
}
diff --git a/solr/core/src/java/org/apache/solr/util/vector/ByteDenseVectorParser.java b/solr/core/src/java/org/apache/solr/util/vector/ByteDenseVectorParser.java
new file mode 100644
index 00000000000..0d97514f671
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/util/vector/ByteDenseVectorParser.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.util.vector;
+
+public class ByteDenseVectorParser extends DenseVectorParser {
+ private byte[] byteVector;
+ private int curPosition;
+
+ public ByteDenseVectorParser(int dimension, Object inputValue, BuilderPhase builderPhase) {
+ this.dimension = dimension;
+ this.inputValue = inputValue;
+ this.builderPhase = builderPhase;
+ this.curPosition = 0;
+ }
+
+ @Override
+ public byte[] getByteVector() {
+ if (byteVector == null) {
+ byteVector = new byte[dimension];
+ parseVector();
+ }
+ return byteVector;
+ }
+
+ @Override
+ protected void addNumberElement(Number element) {
+ byteVector[curPosition++] = element.byteValue();
+ }
+
+ @Override
+ protected void addStringElement(String element) {
+ byteVector[curPosition++] = Byte.parseByte(element);
+ }
+
+ @Override
+ protected String errorMessage() {
+ return "The expected format is:'[b1,b2..b3]' where each element b is a byte (-128 to 127)";
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/util/vector/DenseVectorParser.java b/solr/core/src/java/org/apache/solr/util/vector/DenseVectorParser.java
new file mode 100644
index 00000000000..29001b3d789
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/util/vector/DenseVectorParser.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.util.vector;
+
+import java.util.List;
+import org.apache.solr.common.SolrException;
+
+public abstract class DenseVectorParser {
+
+ public enum BuilderPhase {
+ INDEX,
+ QUERY
+ }
+
+ protected BuilderPhase builderPhase;
+
+ protected int dimension;
+ protected Object inputValue;
+
+ public float[] getFloatVector() {
+ throw new UnsupportedOperationException("Float vector parsing is not supported");
+ }
+
+ public byte[] getByteVector() {
+ throw new UnsupportedOperationException("Byte vector parsing is not supported");
+ }
+
+ protected void parseVector() {
+ switch (builderPhase) {
+ case INDEX:
+ parseIndexVector();
+ break;
+ case QUERY:
+ parseQueryVector();
+ break;
+ }
+ }
+
+ protected void parseIndexVector() {
+ if (!(inputValue instanceof List)) {
+ throw new SolrException(
+ SolrException.ErrorCode.BAD_REQUEST, "incorrect vector format. " + errorMessage());
+ }
+ List<?> inputVector = (List<?>) inputValue;
+ checkVectorDimension(inputVector.size());
+ if (inputVector.get(0) instanceof CharSequence) {
+ for (int i = 0; i < dimension; i++) {
+ try {
+ addStringElement(inputVector.get(i).toString());
+ } catch (NumberFormatException e) {
+ throw new SolrException(
+ SolrException.ErrorCode.BAD_REQUEST,
+ "incorrect vector element: '" + inputVector.get(i) + "'. " + errorMessage());
+ }
+ }
+ } else if (inputVector.get(0) instanceof Number) {
+ for (int i = 0; i < dimension; i++) {
+ addNumberElement((Number) inputVector.get(i));
+ }
+ } else {
+ throw new SolrException(
+ SolrException.ErrorCode.BAD_REQUEST, "incorrect vector format. " + errorMessage());
+ }
+ }
+
+ protected void parseQueryVector() {
+
+ String value = inputValue.toString();
+ if (!value.startsWith("[") || !value.endsWith("]")) {
+ throw new SolrException(
+ SolrException.ErrorCode.BAD_REQUEST, "incorrect vector format. " + errorMessage());
+ }
+
+ String[] elements = value.substring(1, value.length() - 1).split(",");
+ checkVectorDimension(elements.length);
+
+ for (int i = 0; i < dimension; i++) {
+ try {
+ addStringElement(elements[i].trim());
+ } catch (NumberFormatException e) {
+ throw new SolrException(
+ SolrException.ErrorCode.BAD_REQUEST,
+ "incorrect vector element: '" + elements[i] + "'. " + errorMessage());
+ }
+ }
+ }
+
+ protected void checkVectorDimension(int inputVectorDimension) {
+ if (inputVectorDimension != dimension) {
+ throw new SolrException(
+ SolrException.ErrorCode.BAD_REQUEST,
+ "incorrect vector dimension."
+ + " The vector value has size "
+ + inputVectorDimension
+ + " while it is expected a vector with size "
+ + dimension);
+ }
+ }
+
+ protected abstract void addNumberElement(Number element);
+
+ protected abstract void addStringElement(String element);
+
+ protected abstract String errorMessage();
+}
diff --git a/solr/core/src/java/org/apache/solr/util/vector/FloatDenseVectorParser.java b/solr/core/src/java/org/apache/solr/util/vector/FloatDenseVectorParser.java
new file mode 100644
index 00000000000..59af043caf7
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/util/vector/FloatDenseVectorParser.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.util.vector;
+
+public class FloatDenseVectorParser extends DenseVectorParser {
+ private float[] vector;
+ private int curPosition;
+
+ public FloatDenseVectorParser(int dimension, Object inputValue, BuilderPhase builderPhase) {
+ this.dimension = dimension;
+ this.inputValue = inputValue;
+ this.curPosition = 0;
+ this.builderPhase = builderPhase;
+ }
+
+ @Override
+ public float[] getFloatVector() {
+ if (vector == null) {
+ vector = new float[dimension];
+ parseVector();
+ }
+ return vector;
+ }
+
+ @Override
+ protected void addNumberElement(Number element) {
+ vector[curPosition++] = element.floatValue();
+ }
+
+ @Override
+ protected void addStringElement(String element) {
+ vector[curPosition++] = Float.parseFloat(element);
+ }
+
+ @Override
+ protected String errorMessage() {
+ return "The expected format is:'[f1,f2..f3]' where each element f is a float";
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/util/vector/package-info.java b/solr/core/src/java/org/apache/solr/util/vector/package-info.java
new file mode 100644
index 00000000000..0dfdab9f9ea
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/util/vector/package-info.java
@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/** Utilities for dense vector search */
+package org.apache.solr.util.vector;
diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-densevector.xml b/solr/core/src/test-files/solr/collection1/conf/schema-densevector.xml
index f03c570cb9f..7b5057c74aa 100644
--- a/solr/core/src/test-files/solr/collection1/conf/schema-densevector.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/schema-densevector.xml
@@ -20,11 +20,18 @@
<schema name="schema-densevector" version="1.0">
<fieldType name="string" class="solr.StrField" multiValued="true"/>
- <fieldType name="knn_vector" class="solr.DenseVectorField" vectorDimension="4" similarityFunction="cosine"/>
-
+ <fieldType name="knn_vector" class="solr.DenseVectorField" vectorDimension="4" similarityFunction="cosine" />
+ <fieldType name="plong" class="solr.LongPointField" useDocValuesAsStored="false"/>
+
+ <fieldType name="knn_vector_byte_encoding" class="solr.DenseVectorField" vectorDimension="4" similarityFunction="cosine" vectorEncoding="BYTE"/>
+
<field name="id" type="string" indexed="true" stored="true" multiValued="false" required="false"/>
<field name="vector" type="knn_vector" indexed="true" stored="true"/>
<field name="vector2" type="knn_vector" indexed="true" stored="true"/>
-
+ <field name="vector_byte_encoding" type="knn_vector_byte_encoding" indexed="true" stored="true" />
+ <field name="string_field" type="string" indexed="true" stored="true" multiValued="false" required="false"/>
+
+ <field name="_version_" type="plong" indexed="true" stored="true" multiValued="false" />
+
<uniqueKey>id</uniqueKey>
</schema>
diff --git a/solr/core/src/test/org/apache/solr/schema/DenseVectorFieldTest.java b/solr/core/src/test/org/apache/solr/schema/DenseVectorFieldTest.java
index ad9ab23f073..55f05efb093 100644
--- a/solr/core/src/test/org/apache/solr/schema/DenseVectorFieldTest.java
+++ b/solr/core/src/test/org/apache/solr/schema/DenseVectorFieldTest.java
@@ -19,16 +19,27 @@ package org.apache.solr.schema;
import static org.hamcrest.core.Is.is;
import java.util.Arrays;
+import java.util.Map;
+import org.apache.lucene.index.VectorEncoding;
import org.apache.lucene.index.VectorSimilarityFunction;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.core.AbstractBadConfigTestBase;
+import org.apache.solr.util.vector.DenseVectorParser;
import org.hamcrest.MatcherAssert;
+import org.junit.Before;
import org.junit.Test;
public class DenseVectorFieldTest extends AbstractBadConfigTestBase {
- private DenseVectorField toTest = new DenseVectorField();
+ private DenseVectorField toTestFloatEncoding;
+ private DenseVectorField toTestByteEncoding;
+
+ @Before
+ public void init() {
+ toTestFloatEncoding = new DenseVectorField(3, VectorEncoding.FLOAT32);
+ toTestByteEncoding = new DenseVectorField(3, VectorEncoding.BYTE);
+ }
@Test
public void fieldTypeDefinition_badVectorDimension_shouldThrowException() throws Exception {
@@ -171,57 +182,125 @@ public class DenseVectorFieldTest extends AbstractBadConfigTestBase {
"Single string value should throw an exception",
SolrException.class,
() -> {
- toTest.parseVector("string");
+ toTestFloatEncoding
+ .getVectorBuilder("string", DenseVectorParser.BuilderPhase.INDEX)
+ .getFloatVector();
+ });
+ MatcherAssert.assertThat(
+ thrown.getMessage(),
+ is(
+ "incorrect vector format. The expected format is:'[f1,f2..f3]' where each element f is a float"));
+
+ thrown =
+ assertThrows(
+ "Single string value should throw an exception",
+ SolrException.class,
+ () -> {
+ toTestByteEncoding
+ .getVectorBuilder("string", DenseVectorParser.BuilderPhase.INDEX)
+ .getByteVector();
});
MatcherAssert.assertThat(
thrown.getMessage(),
is(
- "incorrect vector format."
- + " The expected format is an array :'[f1,f2..f3]' where each element f is a float"));
+ "incorrect vector format. The expected format is:'[b1,b2..b3]' where each element b is a byte (-128 to 127)"));
thrown =
assertThrows(
"Single float value should throw an exception",
SolrException.class,
() -> {
- toTest.parseVector(1.5f);
+ toTestFloatEncoding
+ .getVectorBuilder(1.5f, DenseVectorParser.BuilderPhase.INDEX)
+ .getFloatVector();
});
MatcherAssert.assertThat(
thrown.getMessage(),
is(
- "incorrect vector format."
- + " The expected format is an array :'[f1,f2..f3]' where each element f is a float"));
+ "incorrect vector format. The expected format is:'[f1,f2..f3]' where each element f is a float"));
+
+ thrown =
+ assertThrows(
+ "Single string value should throw an exception",
+ SolrException.class,
+ () -> {
+ toTestByteEncoding
+ .getVectorBuilder("1", DenseVectorParser.BuilderPhase.INDEX)
+ .getByteVector();
+ });
+ MatcherAssert.assertThat(
+ thrown.getMessage(),
+ is(
+ "incorrect vector format. The expected format is:'[b1,b2..b3]' where each element b is a byte (-128 to 127)"));
}
@Test
public void parseVector_notNumericList_shouldThrowException() {
- toTest = new DenseVectorField(3);
-
RuntimeException thrown =
assertThrows(
"Incorrect elements should throw an exception",
SolrException.class,
() -> {
- toTest.parseVector(
- Arrays.asList(
- new DenseVectorField(3), new DenseVectorField(4), new DenseVectorField(5)));
+ toTestFloatEncoding
+ .getVectorBuilder(
+ Arrays.asList(
+ new DenseVectorField(3),
+ new DenseVectorField(4),
+ new DenseVectorField(5)),
+ DenseVectorParser.BuilderPhase.INDEX)
+ .getFloatVector();
});
MatcherAssert.assertThat(
thrown.getMessage(),
is(
- "incorrect vector format. The expected format is an array :'[f1,f2..f3]' where each element f is a float"));
+ "incorrect vector format. The expected format is:'[f1,f2..f3]' where each element f is a float"));
+
+ thrown =
+ assertThrows(
+ "Incorrect elements should throw an exception",
+ SolrException.class,
+ () -> {
+ toTestByteEncoding
+ .getVectorBuilder(
+ Arrays.asList(
+ new DenseVectorField(3),
+ new DenseVectorField(4),
+ new DenseVectorField(5)),
+ DenseVectorParser.BuilderPhase.INDEX)
+ .getByteVector();
+ });
+ MatcherAssert.assertThat(
+ thrown.getMessage(),
+ is(
+ "incorrect vector format. The expected format is:'[b1,b2..b3]' where each element b is a byte (-128 to 127)"));
}
@Test
public void parseVector_incorrectVectorDimension_shouldThrowException() {
- toTest = new DenseVectorField(3);
+ toTestFloatEncoding = new DenseVectorField(3);
RuntimeException thrown =
assertThrows(
"Incorrect vector dimension should throw an exception",
SolrException.class,
() -> {
- toTest.parseVector(Arrays.asList(1.0f, 1.5f));
+ toTestFloatEncoding
+ .getVectorBuilder(Arrays.asList(1.0f, 1.5f), DenseVectorParser.BuilderPhase.INDEX)
+ .getFloatVector();
+ });
+ MatcherAssert.assertThat(
+ thrown.getMessage(),
+ is(
+ "incorrect vector dimension. The vector value has size 2 while it is expected a vector with size 3"));
+
+ thrown =
+ assertThrows(
+ "Incorrect vector dimension should throw an exception",
+ SolrException.class,
+ () -> {
+ toTestByteEncoding
+ .getVectorBuilder(Arrays.asList(2, 1), DenseVectorParser.BuilderPhase.INDEX)
+ .getByteVector();
});
MatcherAssert.assertThat(
thrown.getMessage(),
@@ -231,19 +310,36 @@ public class DenseVectorFieldTest extends AbstractBadConfigTestBase {
@Test
public void parseVector_incorrectElement_shouldThrowException() {
- toTest = new DenseVectorField(3);
-
RuntimeException thrown =
assertThrows(
"Incorrect elements should throw an exception",
SolrException.class,
() -> {
- toTest.parseVector(Arrays.asList("1.0f", "string", "string2"));
+ toTestFloatEncoding
+ .getVectorBuilder(
+ Arrays.asList("1.0f", "string", "string2"),
+ DenseVectorParser.BuilderPhase.INDEX)
+ .getFloatVector();
});
MatcherAssert.assertThat(
thrown.getMessage(),
is(
"incorrect vector element: 'string'. The expected format is:'[f1,f2..f3]' where each element f is a float"));
+
+ thrown =
+ assertThrows(
+ "Incorrect elements should throw an exception",
+ SolrException.class,
+ () -> {
+ toTestByteEncoding
+ .getVectorBuilder(
+ Arrays.asList("1", "string", "string2"), DenseVectorParser.BuilderPhase.INDEX)
+ .getByteVector();
+ });
+ MatcherAssert.assertThat(
+ thrown.getMessage(),
+ is(
+ "incorrect vector element: 'string'. The expected format is:'[b1,b2..b3]' where each element b is a byte (-128 to 127)"));
}
/**
@@ -253,10 +349,23 @@ public class DenseVectorFieldTest extends AbstractBadConfigTestBase {
*/
@Test
public void parseVector_StringArrayList_shouldParseFloatArray() {
- toTest = new DenseVectorField(3);
float[] expected = new float[] {1.1f, 2.2f, 3.3f};
+ MatcherAssert.assertThat(
+ toTestFloatEncoding
+ .getVectorBuilder(
+ Arrays.asList("1.1", "2.2", "3.3"), DenseVectorParser.BuilderPhase.INDEX)
+ .getFloatVector(),
+ is(expected));
+ }
- MatcherAssert.assertThat(toTest.parseVector(Arrays.asList("1.1", "2.2", "3.3")), is(expected));
+ @Test
+ public void parseVector_StringArrayList_shouldParseByteArray() {
+ byte[] expected = new byte[] {1, 2, 3};
+ MatcherAssert.assertThat(
+ toTestByteEncoding
+ .getVectorBuilder(Arrays.asList("1", "2", "3"), DenseVectorParser.BuilderPhase.INDEX)
+ .getByteVector(),
+ is(expected));
}
/**
@@ -265,10 +374,12 @@ public class DenseVectorFieldTest extends AbstractBadConfigTestBase {
*/
@Test
public void parseVector_DoubleArrayList_shouldParseFloatArray() {
- toTest = new DenseVectorField(3);
float[] expected = new float[] {1.7f, 5.4f, 6.6f};
-
- MatcherAssert.assertThat(toTest.parseVector(Arrays.asList(1.7d, 5.4d, 6.6d)), is(expected));
+ MatcherAssert.assertThat(
+ toTestFloatEncoding
+ .getVectorBuilder(Arrays.asList(1.7d, 5.4d, 6.6d), DenseVectorParser.BuilderPhase.INDEX)
+ .getFloatVector(),
+ is(expected));
}
/**
@@ -277,14 +388,28 @@ public class DenseVectorFieldTest extends AbstractBadConfigTestBase {
*/
@Test
public void parseVector_FloatArrayList_shouldParseFloatArray() {
- toTest = new DenseVectorField(3);
+ toTestFloatEncoding = new DenseVectorField(3);
float[] expected = new float[] {5.5f, 7.7f, 9.8f};
- MatcherAssert.assertThat(toTest.parseVector(Arrays.asList(5.5f, 7.7f, 9.8f)), is(expected));
+ MatcherAssert.assertThat(
+ toTestFloatEncoding
+ .getVectorBuilder(Arrays.asList(5.5f, 7.7f, 9.8f), DenseVectorParser.BuilderPhase.INDEX)
+ .getFloatVector(),
+ is(expected));
}
@Test
- public void indexing_incorrectVectorFormat_shouldThrowException() throws Exception {
+ public void parseVector_IntArrayList_shouldParseByteArray() {
+ byte[] expected = new byte[] {5, 7, 9};
+ MatcherAssert.assertThat(
+ toTestByteEncoding
+ .getVectorBuilder(Arrays.asList(5, 7, 9), DenseVectorParser.BuilderPhase.INDEX)
+ .getByteVector(),
+ is(expected));
+ }
+
+ @Test
+ public void indexing_notAVectorValue_shouldThrowException() throws Exception {
try {
initCore("solrconfig-basic.xml", "schema-densevector.xml");
@@ -332,7 +457,7 @@ public class DenseVectorFieldTest extends AbstractBadConfigTestBase {
}
@Test
- public void query_storedField_shouldBeReturnedInResults() throws Exception {
+ public void query_vectorFloatEncoded_storedField_shouldBeReturnedInResults() throws Exception {
try {
initCore("solrconfig-basic.xml", "schema-densevector.xml");
@@ -345,13 +470,27 @@ public class DenseVectorFieldTest extends AbstractBadConfigTestBase {
assertJQ(
req("q", "id:0", "fl", "vector"), "/response/docs/[0]=={'vector':[1.1,2.1,3.1,4.1]}");
- assertQ(
- req("q", "id:0", "fl", "vector"),
- "*[count(//doc)=1]",
- "//result/doc[1]/arr[@name=\"vector\"]/float[1][.='" + 1.1 + "']",
- "//result/doc[1]/arr[@name=\"vector\"]/float[2][.='" + 2.1 + "']",
- "//result/doc[1]/arr[@name=\"vector\"]/float[3][.='" + 3.1 + "']",
- "//result/doc[1]/arr[@name=\"vector\"]/float[4][.='" + 4.1 + "']");
+ } finally {
+ deleteCore();
+ }
+ }
+
+ @Test
+ public void query_vectorByteEncoded_storedField_shouldBeReturnedInResults() throws Exception {
+ try {
+ initCore("solrconfig-basic.xml", "schema-densevector.xml");
+
+ SolrInputDocument doc1 = new SolrInputDocument();
+ doc1.addField("id", "0");
+ doc1.addField("vector_byte_encoding", Arrays.asList(1, 2, 3, 4));
+
+ assertU(adoc(doc1));
+ assertU(commit());
+
+ assertJQ(
+ req("q", "id:0", "fl", "vector_byte_encoding"),
+ "/response/docs/[0]=={'vector_byte_encoding':[1,2,3,4]}");
+
} finally {
deleteCore();
}
@@ -458,4 +597,159 @@ public class DenseVectorFieldTest extends AbstractBadConfigTestBase {
deleteCore();
}
}
+
+ @Test
+ public void denseVectorField_shouldBePresentAfterAtomicUpdate() throws Exception {
+ try {
+ initCore("solrconfig.xml", "schema-densevector.xml");
+ SolrInputDocument doc = new SolrInputDocument();
+ doc.addField("id", "0");
+ doc.addField("vector", Arrays.asList(1.1, 2.2, 3.3, 4.4));
+ doc.addField("vector_byte_encoding", Arrays.asList(5, 6, 7, 8));
+ doc.addField("string_field", "test");
+
+ assertU(adoc(doc));
+ assertU(commit());
+
+ assertJQ(
+ req("q", "id:0", "fl", "*"),
+ "/response/docs/[0]/vector==[1.1,2.2,3.3,4.4]",
+ "/response/docs/[0]/vector_byte_encoding==[5,6,7,8]",
+ "/response/docs/[0]/string_field==test");
+
+ SolrInputDocument updateDoc = new SolrInputDocument();
+ updateDoc.addField("id", "0");
+ updateDoc.addField("string_field", Map.of("set", "other test"));
+ assertU(adoc(updateDoc));
+ assertU(commit());
+
+ assertJQ(
+ req("q", "id:0", "fl", "*"),
+ "/response/docs/[0]/vector==[1.1,2.2,3.3,4.4]",
+ "/response/docs/[0]/vector_byte_encoding==[5,6,7,8]",
+ "/response/docs/[0]/string_field=='other test'");
+
+ } finally {
+ deleteCore();
+ }
+ }
+
+ @Test
+ public void denseVectorFieldOnAtomicUpdate_shouldBeUpdatedCorrectly() throws Exception {
+ try {
+ initCore("solrconfig.xml", "schema-densevector.xml");
+ SolrInputDocument doc = new SolrInputDocument();
+ doc.addField("id", "0");
+ doc.addField("vector", Arrays.asList(1.1, 2.2, 3.3, 4.4));
+ doc.addField("vector_byte_encoding", Arrays.asList(5, 6, 7, 8));
+ doc.addField("string_field", "test");
+
+ assertU(adoc(doc));
+ assertU(commit());
+
+ assertJQ(
+ req("q", "id:0", "fl", "*"),
+ "/response/docs/[0]/vector==[1.1,2.2,3.3,4.4]",
+ "/response/docs/[0]/vector_byte_encoding==[5,6,7,8]",
+ "/response/docs/[0]/string_field==test");
+
+ SolrInputDocument updateDoc = new SolrInputDocument();
+ updateDoc.addField("id", "0");
+ updateDoc.addField("vector", Map.of("set", Arrays.asList(9.2, 2.2, 3.3, 5.2)));
+ updateDoc.addField("vector_byte_encoding", Map.of("set", Arrays.asList(8, 3, 1, 3)));
+ assertU(adoc(updateDoc));
+ assertU(commit());
+
+ assertJQ(
+ req("q", "id:0", "fl", "*"),
+ "/response/docs/[0]/vector==[9.2,2.2,3.3,5.2]",
+ "/response/docs/[0]/vector_byte_encoding==[8,3,1,3]",
+ "/response/docs/[0]/string_field=='test'");
+
+ } finally {
+ deleteCore();
+ }
+ }
+
+ @Test
+ public void denseVectorByteEncoding_shouldRaiseExceptionWithValuesOutsideBoundaries()
+ throws Exception {
+ try {
+ initCore("solrconfig.xml", "schema-densevector.xml");
+ SolrInputDocument doc = new SolrInputDocument();
+ doc.addField("id", "0");
+ doc.addField("vector_byte_encoding", Arrays.asList(128, 6, 7, 8));
+
+ RuntimeException thrown =
+ assertThrows(
+ "Incorrect elements should throw an exception",
+ SolrException.class,
+ () -> {
+ assertU(adoc(doc));
+ });
+
+ MatcherAssert.assertThat(
+ thrown.getCause().getMessage(),
+ is(
+ "Error while creating field 'vector_byte_encoding{type=knn_vector_byte_encoding,properties=indexed,stored}' from value '[128, 6, 7, 8]'"));
+
+ MatcherAssert.assertThat(
+ thrown.getCause().getCause().getMessage(),
+ is(
+ "incorrect vector element: '128'. The expected format is:'[b1,b2..b3]' where each element b is a byte (-128 to 127)"));
+
+ SolrInputDocument doc1 = new SolrInputDocument();
+ doc1.addField("id", "1");
+ doc1.addField("vector_byte_encoding", Arrays.asList(1, -129, 7, 8));
+
+ thrown =
+ assertThrows(
+ "Incorrect elements should throw an exception",
+ SolrException.class,
+ () -> {
+ assertU(adoc(doc1));
+ });
+
+ MatcherAssert.assertThat(
+ thrown.getCause().getMessage(),
+ is(
+ "Error while creating field 'vector_byte_encoding{type=knn_vector_byte_encoding,properties=indexed,stored}' from value '[1, -129, 7, 8]'"));
+ MatcherAssert.assertThat(
+ thrown.getCause().getCause().getMessage(),
+ is(
+ "incorrect vector element: '-129'. The expected format is:'[b1,b2..b3]' where each element b is a byte (-128 to 127)"));
+ } finally {
+ deleteCore();
+ }
+ }
+
+ @Test
+ public void denseVectorByteEncoding_shouldRaiseExceptionWithFloatValues() throws Exception {
+ try {
+ initCore("solrconfig.xml", "schema-densevector.xml");
+ SolrInputDocument doc = new SolrInputDocument();
+ doc.addField("id", "0");
+ doc.addField("vector_byte_encoding", Arrays.asList(14.3, 6.2, 7.2, 8.1));
+
+ RuntimeException thrown =
+ assertThrows(
+ "Incorrect elements should throw an exception",
+ SolrException.class,
+ () -> {
+ assertU(adoc(doc));
+ });
+
+ MatcherAssert.assertThat(
+ thrown.getCause().getMessage(),
+ is(
+ "Error while creating field 'vector_byte_encoding{type=knn_vector_byte_encoding,properties=indexed,stored}' from value '[14.3, 6.2, 7.2, 8.1]'"));
+
+ MatcherAssert.assertThat(
+ thrown.getCause().getCause().getMessage(),
+ is(
+ "incorrect vector element: '14.3'. The expected format is:'[b1,b2..b3]' where each element b is a byte (-128 to 127)"));
+ } finally {
+ deleteCore();
+ }
+ }
}
diff --git a/solr/core/src/test/org/apache/solr/search/neural/KnnQParserTest.java b/solr/core/src/test/org/apache/solr/search/neural/KnnQParserTest.java
index c209e8e4165..13c694080c9 100644
--- a/solr/core/src/test/org/apache/solr/search/neural/KnnQParserTest.java
+++ b/solr/core/src/test/org/apache/solr/search/neural/KnnQParserTest.java
@@ -33,6 +33,7 @@ public class KnnQParserTest extends SolrTestCaseJ4 {
String IDField = "id";
String vectorField = "vector";
String vectorField2 = "vector2";
+ String vectorFieldByteEncoding = "vector_byte_encoding";
@Before
public void prepareIndex() throws Exception {
@@ -83,7 +84,6 @@ public class KnnQParserTest extends SolrTestCaseJ4 {
docs.get(9)
.addField(
vectorField, Arrays.asList(1.8f, 2.5f, 3.7f, 4.9f)); // cosine distance vector1= 0.997
-
docs.get(10)
.addField(vectorField2, Arrays.asList(1f, 2f, 3f, 4f)); // cosine distance vector2= 1
docs.get(11)
@@ -94,6 +94,15 @@ public class KnnQParserTest extends SolrTestCaseJ4 {
.addField(
vectorField2, Arrays.asList(1.5f, 2.5f, 3.5f, 4.5f)); // cosine distance vector2= 0.998
+ docs.get(0).addField(vectorFieldByteEncoding, Arrays.asList(1, 2, 3, 4));
+ docs.get(1).addField(vectorFieldByteEncoding, Arrays.asList(2, 2, 1, 4));
+ docs.get(2).addField(vectorFieldByteEncoding, Arrays.asList(1, 2, 1, 2));
+ docs.get(3).addField(vectorFieldByteEncoding, Arrays.asList(7, 2, 1, 3));
+ docs.get(4).addField(vectorFieldByteEncoding, Arrays.asList(19, 2, 4, 4));
+ docs.get(5).addField(vectorFieldByteEncoding, Arrays.asList(19, 2, 4, 4));
+ docs.get(6).addField(vectorFieldByteEncoding, Arrays.asList(18, 2, 4, 4));
+ docs.get(7).addField(vectorFieldByteEncoding, Arrays.asList(8, 3, 2, 4));
+
return docs;
}
@@ -205,6 +214,56 @@ public class KnnQParserTest extends SolrTestCaseJ4 {
"//result/doc[3]/str[@name='id'][.='12']");
}
+ @Test
+ public void vectorByteEncodingField_shouldSearchOnThatField() {
+ String vectorToSearch = "[2, 2, 1, 3]";
+
+ assertQ(
+ req(CommonParams.Q, "{!knn f=vector_byte_encoding topK=2}" + vectorToSearch, "fl", "id"),
+ "//result[@numFound='2']",
+ "//result/doc[1]/str[@name='id'][.='2']",
+ "//result/doc[2]/str[@name='id'][.='3']");
+
+ vectorToSearch = "[8, 3, 2, 4]";
+
+ assertQ(
+ req(CommonParams.Q, "{!knn f=vector_byte_encoding topK=2}" + vectorToSearch, "fl", "id"),
+ "//result[@numFound='2']",
+ "//result/doc[1]/str[@name='id'][.='8']",
+ "//result/doc[2]/str[@name='id'][.='4']");
+ }
+
+ @Test
+ public void vectorByteEncodingField_shouldRaiseExceptionIfQueryUsesFloatVectors() {
+ String vectorToSearch = "[8.3, 4.3, 2.1, 4.1]";
+
+ assertQEx(
+ "incorrect vector element: '8.3'. The expected format is:'[b1,b2..b3]' where each element b is a byte (-128 to 127)",
+ "incorrect vector element: '8.3'. The expected format is:'[b1,b2..b3]' where each element b is a byte (-128 to 127)",
+ req(CommonParams.Q, "{!knn f=vector_byte_encoding topK=10}" + vectorToSearch, "fl", "id"),
+ SolrException.ErrorCode.BAD_REQUEST);
+ }
+
+ @Test
+ public void
+ vectorByteEncodingField_shouldRaiseExceptionWhenQueryContainsValuesOutsideByteValueRange() {
+ String vectorToSearch = "[1, -129, 3, 5]";
+
+ assertQEx(
+ "incorrect vector element: ' -129'. The expected format is:'[b1,b2..b3]' where each element b is a byte (-128 to 127)",
+ "incorrect vector element: ' -129'. The expected format is:'[b1,b2..b3]' where each element b is a byte (-128 to 127)",
+ req(CommonParams.Q, "{!knn f=vector_byte_encoding topK=10}" + vectorToSearch, "fl", "id"),
+ SolrException.ErrorCode.BAD_REQUEST);
+
+ vectorToSearch = "[1, 3, 156, 5]";
+
+ assertQEx(
+ "incorrect vector element: ' 156'. The expected format is:'[b1,b2..b3]' where each element b is a byte (-128 to 127)",
+ "incorrect vector element: ' 156'. The expected format is:'[b1,b2..b3]' where each element b is a byte (-128 to 127)",
+ req(CommonParams.Q, "{!knn f=vector_byte_encoding topK=10}" + vectorToSearch, "fl", "id"),
+ SolrException.ErrorCode.BAD_REQUEST);
+ }
+
@Test
public void missingVectorToSearch_shouldThrowException() {
assertQEx(
diff --git a/solr/core/src/test/org/apache/solr/update/DocumentBuilderTest.java b/solr/core/src/test/org/apache/solr/update/DocumentBuilderTest.java
index e708035571c..c6310c3600c 100644
--- a/solr/core/src/test/org/apache/solr/update/DocumentBuilderTest.java
+++ b/solr/core/src/test/org/apache/solr/update/DocumentBuilderTest.java
@@ -25,6 +25,7 @@ import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
import org.apache.lucene.document.Document;
+import org.apache.lucene.document.KnnFloatVectorField;
import org.apache.lucene.document.KnnVectorField;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.VectorSimilarityFunction;
@@ -357,7 +358,7 @@ public class DocumentBuilderTest extends SolrTestCaseJ4 {
"vector", new float[] {1.1f, 2.1f, 3.1f, 4.1f}, VectorSimilarityFunction.COSINE);
MatcherAssert.assertThat(
- ((KnnVectorField) out.getField("vector")).vectorValue(),
+ ((KnnFloatVectorField) out.getField("vector")).vectorValue(),
is(expectedIndexableField.vectorValue()));
List<IndexableField> storedFields =
@@ -388,7 +389,7 @@ public class DocumentBuilderTest extends SolrTestCaseJ4 {
"vector2", new float[] {1.1f, 2.1f, 3.1f, 4.1f}, VectorSimilarityFunction.DOT_PRODUCT);
MatcherAssert.assertThat(
- ((KnnVectorField) out.getField("vector2")).vectorValue(),
+ ((KnnFloatVectorField) out.getField("vector2")).vectorValue(),
is(expectedDestination.vectorValue()));
List<IndexableField> storedFields =
@@ -442,7 +443,7 @@ public class DocumentBuilderTest extends SolrTestCaseJ4 {
MatcherAssert.assertThat(
thrown.getMessage(),
is(
- "ERROR: [doc=0] Error adding field 'vector4'='[1.1, 2.1, 3.1, 4.1]' msg=Error while creating field 'vector5{type=knn_vector5,properties=indexed,stored}' from value '[1.1, 2.1, 3.1, 4.1]', expected format:'[f1, f2, f3...fn]' e.g. [1.0, 3.4, 5.6]"));
+ "ERROR: [doc=0] Error adding field 'vector4'='[1.1, 2.1, 3.1, 4.1]' msg=Error while creating field 'vector5{type=knn_vector5,properties=indexed,stored}' from value '[1.1, 2.1, 3.1, 4.1]'"));
MatcherAssert.assertThat(
thrown.getCause().getCause().getMessage(),
is(
diff --git a/solr/solr-ref-guide/modules/query-guide/pages/dense-vector-search.adoc b/solr/solr-ref-guide/modules/query-guide/pages/dense-vector-search.adoc
index b9e6106ffec..8cc75bed709 100644
--- a/solr/solr-ref-guide/modules/query-guide/pages/dense-vector-search.adoc
+++ b/solr/solr-ref-guide/modules/query-guide/pages/dense-vector-search.adoc
@@ -126,6 +126,18 @@ Accepted values: `hnsw`.
Please note that the `knnAlgorithm` accepted values may change in future releases.
+`vectorEncoding`::
++
+[%autowidth,frame=none]
+|===
+|Optional |Default: `FLOAT32`
+|===
++
+(advanced) Specifies the underlying encoding of the dense vector elements. This affects memory/disk impact for both the indexed and stored fields (if enabled)
++
+
+Accepted values: `FLOAT32`, `BYTE`.
+
`hnswMaxConnections`::
+