You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by cp...@apache.org on 2024/07/01 16:58:13 UTC
(lucene) branch branch_9x updated (6304105c104 -> 48816b4de42)
This is an automated email from the ASF dual-hosted git repository.
cpoerschke pushed a change to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/lucene.git
from 6304105c104 Fix more vector similarity query tests (#13530)
new dce6cd0723b Remove unused segNo calculation in IndexWriter.doFlush (#13491)
new e40d5c55560 [Abstract]Knn[Byte|Float]VectorQuery tweaks: reduce duplicate method calls (#13528)
new 48816b4de42 Lucene99HnswVectorsReader.search float-vs-byte variants: reduce code duplication (#13529)
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../codecs/lucene99/Lucene99HnswVectorsReader.java | 53 ++++++++++------------
.../java/org/apache/lucene/index/IndexWriter.java | 8 +---
.../lucene/search/AbstractKnnVectorQuery.java | 15 +++---
.../apache/lucene/search/KnnByteVectorQuery.java | 13 ++++--
.../apache/lucene/search/KnnFloatVectorQuery.java | 13 ++++--
.../lucene/search/TestKnnFloatVectorQuery.java | 2 +-
6 files changed, 50 insertions(+), 54 deletions(-)
(lucene) 03/03: Lucene99HnswVectorsReader.search float-vs-byte variants: reduce code duplication (#13529)
Posted by cp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
cpoerschke pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit 48816b4de422f6330c6b4bfbdd30a3f1722b8fad
Author: Christine Poerschke <cp...@apache.org>
AuthorDate: Mon Jul 1 17:32:04 2024 +0100
Lucene99HnswVectorsReader.search float-vs-byte variants: reduce code duplication (#13529)
* Lucene99HnswVectorsReader.search float-vs-byte variants: reduce code duplication
* action review feedback: use org.apache.lucene.util.IOSupplier
(cherry picked from commit f4cd4b46fc1ccb497624c5c5454785078aca6501)
---
.../codecs/lucene99/Lucene99HnswVectorsReader.java | 53 ++++++++++------------
1 file changed, 24 insertions(+), 29 deletions(-)
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsReader.java
index aacb58a2c4a..7b9e26e630e 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsReader.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsReader.java
@@ -45,6 +45,7 @@ import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.RandomAccessInput;
import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.IOSupplier;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.hnsw.HnswGraph;
@@ -248,45 +249,39 @@ public final class Lucene99HnswVectorsReader extends KnnVectorsReader
@Override
public void search(String field, float[] target, KnnCollector knnCollector, Bits acceptDocs)
throws IOException {
- FieldEntry fieldEntry = fields.get(field);
-
- if (fieldEntry.size() == 0
- || knnCollector.k() == 0
- || fieldEntry.vectorEncoding != VectorEncoding.FLOAT32) {
- return;
- }
- final RandomVectorScorer scorer = flatVectorsReader.getRandomVectorScorer(field, target);
- final KnnCollector collector =
- new OrdinalTranslatedKnnCollector(knnCollector, scorer::ordToDoc);
- final Bits acceptedOrds = scorer.getAcceptOrds(acceptDocs);
- if (knnCollector.k() < scorer.maxOrd()) {
- HnswGraphSearcher.search(scorer, collector, getGraph(fieldEntry), acceptedOrds);
- } else {
- // if k is larger than the number of vectors, we can just iterate over all vectors
- // and collect them
- for (int i = 0; i < scorer.maxOrd(); i++) {
- if (acceptedOrds == null || acceptedOrds.get(i)) {
- if (knnCollector.earlyTerminated()) {
- break;
- }
- knnCollector.incVisitedCount(1);
- knnCollector.collect(scorer.ordToDoc(i), scorer.score(i));
- }
- }
- }
+ search(
+ fields.get(field),
+ knnCollector,
+ acceptDocs,
+ VectorEncoding.FLOAT32,
+ () -> flatVectorsReader.getRandomVectorScorer(field, target));
}
@Override
public void search(String field, byte[] target, KnnCollector knnCollector, Bits acceptDocs)
throws IOException {
- FieldEntry fieldEntry = fields.get(field);
+ search(
+ fields.get(field),
+ knnCollector,
+ acceptDocs,
+ VectorEncoding.BYTE,
+ () -> flatVectorsReader.getRandomVectorScorer(field, target));
+ }
+
+ private void search(
+ FieldEntry fieldEntry,
+ KnnCollector knnCollector,
+ Bits acceptDocs,
+ VectorEncoding vectorEncoding,
+ IOSupplier<RandomVectorScorer> scorerSupplier)
+ throws IOException {
if (fieldEntry.size() == 0
|| knnCollector.k() == 0
- || fieldEntry.vectorEncoding != VectorEncoding.BYTE) {
+ || fieldEntry.vectorEncoding != vectorEncoding) {
return;
}
- final RandomVectorScorer scorer = flatVectorsReader.getRandomVectorScorer(field, target);
+ final RandomVectorScorer scorer = scorerSupplier.get();
final KnnCollector collector =
new OrdinalTranslatedKnnCollector(knnCollector, scorer::ordToDoc);
final Bits acceptedOrds = scorer.getAcceptOrds(acceptDocs);
(lucene) 01/03: Remove unused segNo calculation in IndexWriter.doFlush (#13491)
Posted by cp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
cpoerschke pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit dce6cd0723b28377c67d134ddf56e0f5092c4ff1
Author: zhouhui <vs...@163.com>
AuthorDate: Tue Jul 2 00:29:10 2024 +0800
Remove unused segNo calculation in IndexWriter.doFlush (#13491)
(cherry picked from commit 3cd406e783739a6bdcfc13d14b12640ba63fcdfa)
---
lucene/core/src/java/org/apache/lucene/index/IndexWriter.java | 8 +-------
1 file changed, 1 insertion(+), 7 deletions(-)
diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
index 05f36a24a11..1b9fea4e041 100644
--- a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
@@ -4279,13 +4279,7 @@ public class IndexWriter
synchronized (fullFlushLock) {
boolean flushSuccess = false;
try {
- long seqNo = docWriter.flushAllThreads();
- if (seqNo < 0) {
- seqNo = -seqNo;
- anyChanges = true;
- } else {
- anyChanges = false;
- }
+ anyChanges = (docWriter.flushAllThreads() < 0);
if (!anyChanges) {
// flushCount is incremented in flushAllThreads
flushCount.incrementAndGet();
(lucene) 02/03: [Abstract]Knn[Byte|Float]VectorQuery tweaks: reduce duplicate method calls (#13528)
Posted by cp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
cpoerschke pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit e40d5c55560cedf29bc9aa259f3317781e18e34d
Author: Christine Poerschke <cp...@apache.org>
AuthorDate: Mon Jul 1 17:31:02 2024 +0100
[Abstract]Knn[Byte|Float]VectorQuery tweaks: reduce duplicate method calls (#13528)
* reduce LeafReaderContext.reader()[.maxDoc()] calls in AbstractKnnVectorQuery.getLeafResults
* reduce IndexReader.leaves() calls in AbstractKnnVectorQuery.findSegmentStarts
* reduce LeafReaderContext.reader() calls in Knn(Byte|Float)VectorQuery.approximateSearch
(cherry picked from commit 0ad270d8b0d4ca1e8ef0ddf2b32947192b51b224)
Resolved Conflicts:
lucene/core/src/java/org/apache/lucene/search/AbstractKnnVectorQuery.java
---
.../org/apache/lucene/search/AbstractKnnVectorQuery.java | 15 ++++++++-------
.../java/org/apache/lucene/search/KnnByteVectorQuery.java | 13 ++++++++-----
.../org/apache/lucene/search/KnnFloatVectorQuery.java | 13 ++++++++-----
.../org/apache/lucene/search/TestKnnFloatVectorQuery.java | 2 +-
4 files changed, 25 insertions(+), 18 deletions(-)
diff --git a/lucene/core/src/java/org/apache/lucene/search/AbstractKnnVectorQuery.java b/lucene/core/src/java/org/apache/lucene/search/AbstractKnnVectorQuery.java
index 0775a820ad6..7180995d5f1 100644
--- a/lucene/core/src/java/org/apache/lucene/search/AbstractKnnVectorQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/AbstractKnnVectorQuery.java
@@ -28,6 +28,7 @@ import java.util.concurrent.Callable;
import org.apache.lucene.codecs.KnnVectorsReader;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.QueryTimeout;
import org.apache.lucene.search.knn.KnnCollectorManager;
@@ -120,8 +121,8 @@ abstract class AbstractKnnVectorQuery extends Query {
Weight filterWeight,
TimeLimitingKnnCollectorManager timeLimitingKnnCollectorManager)
throws IOException {
- Bits liveDocs = ctx.reader().getLiveDocs();
- int maxDoc = ctx.reader().maxDoc();
+ final LeafReader reader = ctx.reader();
+ final Bits liveDocs = reader.getLiveDocs();
if (filterWeight == null) {
return approximateSearch(ctx, liveDocs, Integer.MAX_VALUE, timeLimitingKnnCollectorManager);
@@ -132,7 +133,7 @@ abstract class AbstractKnnVectorQuery extends Query {
return NO_RESULTS;
}
- BitSet acceptDocs = createBitSet(scorer.iterator(), liveDocs, maxDoc);
+ BitSet acceptDocs = createBitSet(scorer.iterator(), liveDocs, reader.maxDoc());
final int cost = acceptDocs.cardinality();
QueryTimeout queryTimeout = timeLimitingKnnCollectorManager.getQueryTimeout();
@@ -263,19 +264,19 @@ abstract class AbstractKnnVectorQuery extends Query {
docs[i] = topK.scoreDocs[i].doc;
scores[i] = topK.scoreDocs[i].score;
}
- int[] segmentStarts = findSegmentStarts(reader, docs);
+ int[] segmentStarts = findSegmentStarts(reader.leaves(), docs);
return new DocAndScoreQuery(k, docs, scores, segmentStarts, reader.getContext().id());
}
- static int[] findSegmentStarts(IndexReader reader, int[] docs) {
- int[] starts = new int[reader.leaves().size() + 1];
+ static int[] findSegmentStarts(List<LeafReaderContext> leaves, int[] docs) {
+ int[] starts = new int[leaves.size() + 1];
starts[starts.length - 1] = docs.length;
if (starts.length == 2) {
return starts;
}
int resultIndex = 0;
for (int i = 1; i < starts.length - 1; i++) {
- int upper = reader.leaves().get(i).docBase;
+ int upper = leaves.get(i).docBase;
resultIndex = Arrays.binarySearch(docs, resultIndex, docs.length, upper);
if (resultIndex < 0) {
resultIndex = -1 - resultIndex;
diff --git a/lucene/core/src/java/org/apache/lucene/search/KnnByteVectorQuery.java b/lucene/core/src/java/org/apache/lucene/search/KnnByteVectorQuery.java
index 5b60e680e10..db5ae4a0d9d 100644
--- a/lucene/core/src/java/org/apache/lucene/search/KnnByteVectorQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/KnnByteVectorQuery.java
@@ -23,6 +23,7 @@ import org.apache.lucene.codecs.KnnVectorsReader;
import org.apache.lucene.document.KnnFloatVectorField;
import org.apache.lucene.index.ByteVectorValues;
import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.search.knn.KnnCollectorManager;
import org.apache.lucene.util.ArrayUtil;
@@ -83,24 +84,26 @@ public class KnnByteVectorQuery extends AbstractKnnVectorQuery {
KnnCollectorManager knnCollectorManager)
throws IOException {
KnnCollector knnCollector = knnCollectorManager.newCollector(visitedLimit, context);
- ByteVectorValues byteVectorValues = context.reader().getByteVectorValues(field);
+ LeafReader reader = context.reader();
+ ByteVectorValues byteVectorValues = reader.getByteVectorValues(field);
if (byteVectorValues == null) {
- ByteVectorValues.checkField(context.reader(), field);
+ ByteVectorValues.checkField(reader, field);
return NO_RESULTS;
}
if (Math.min(knnCollector.k(), byteVectorValues.size()) == 0) {
return NO_RESULTS;
}
- context.reader().searchNearestVectors(field, target, knnCollector, acceptDocs);
+ reader.searchNearestVectors(field, target, knnCollector, acceptDocs);
TopDocs results = knnCollector.topDocs();
return results != null ? results : NO_RESULTS;
}
@Override
VectorScorer createVectorScorer(LeafReaderContext context, FieldInfo fi) throws IOException {
- ByteVectorValues vectorValues = context.reader().getByteVectorValues(field);
+ LeafReader reader = context.reader();
+ ByteVectorValues vectorValues = reader.getByteVectorValues(field);
if (vectorValues == null) {
- ByteVectorValues.checkField(context.reader(), field);
+ ByteVectorValues.checkField(reader, field);
return null;
}
return vectorValues.scorer(target);
diff --git a/lucene/core/src/java/org/apache/lucene/search/KnnFloatVectorQuery.java b/lucene/core/src/java/org/apache/lucene/search/KnnFloatVectorQuery.java
index b06e81eb5d8..585893fa3c2 100644
--- a/lucene/core/src/java/org/apache/lucene/search/KnnFloatVectorQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/KnnFloatVectorQuery.java
@@ -23,6 +23,7 @@ import org.apache.lucene.codecs.KnnVectorsReader;
import org.apache.lucene.document.KnnFloatVectorField;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FloatVectorValues;
+import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.search.knn.KnnCollectorManager;
import org.apache.lucene.util.ArrayUtil;
@@ -84,24 +85,26 @@ public class KnnFloatVectorQuery extends AbstractKnnVectorQuery {
KnnCollectorManager knnCollectorManager)
throws IOException {
KnnCollector knnCollector = knnCollectorManager.newCollector(visitedLimit, context);
- FloatVectorValues floatVectorValues = context.reader().getFloatVectorValues(field);
+ LeafReader reader = context.reader();
+ FloatVectorValues floatVectorValues = reader.getFloatVectorValues(field);
if (floatVectorValues == null) {
- FloatVectorValues.checkField(context.reader(), field);
+ FloatVectorValues.checkField(reader, field);
return NO_RESULTS;
}
if (Math.min(knnCollector.k(), floatVectorValues.size()) == 0) {
return NO_RESULTS;
}
- context.reader().searchNearestVectors(field, target, knnCollector, acceptDocs);
+ reader.searchNearestVectors(field, target, knnCollector, acceptDocs);
TopDocs results = knnCollector.topDocs();
return results != null ? results : NO_RESULTS;
}
@Override
VectorScorer createVectorScorer(LeafReaderContext context, FieldInfo fi) throws IOException {
- FloatVectorValues vectorValues = context.reader().getFloatVectorValues(field);
+ LeafReader reader = context.reader();
+ FloatVectorValues vectorValues = reader.getFloatVectorValues(field);
if (vectorValues == null) {
- FloatVectorValues.checkField(context.reader(), field);
+ FloatVectorValues.checkField(reader, field);
return null;
}
return vectorValues.scorer(target);
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestKnnFloatVectorQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestKnnFloatVectorQuery.java
index eb840d01e6f..6c973d5a103 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestKnnFloatVectorQuery.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestKnnFloatVectorQuery.java
@@ -212,7 +212,7 @@ public class TestKnnFloatVectorQuery extends BaseKnnVectorQueryTestCase {
maxScore = Math.max(maxScore, scores[i]);
}
IndexReader indexReader = searcher.getIndexReader();
- int[] segments = AbstractKnnVectorQuery.findSegmentStarts(indexReader, docs);
+ int[] segments = AbstractKnnVectorQuery.findSegmentStarts(indexReader.leaves(), docs);
AbstractKnnVectorQuery.DocAndScoreQuery query =
new AbstractKnnVectorQuery.DocAndScoreQuery(