You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ms...@apache.org on 2026/06/16 20:04:01 UTC
(lucene) branch branch_10x updated: Optimize FieldExistsQuery.count() when all docs have the field (#16111)
This is an automated email from the ASF dual-hosted git repository.
msfroh pushed a commit to branch branch_10x
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/branch_10x by this push:
new 6bbd64527fa Optimize FieldExistsQuery.count() when all docs have the field (#16111)
6bbd64527fa is described below
commit 6bbd64527fa61afe785a67dbdb8dec9334193084
Author: Prithvi S <pr...@gmail.com>
AuthorDate: Wed Jun 17 01:24:46 2026 +0530
Optimize FieldExistsQuery.count() when all docs have the field (#16111)
---
lucene/CHANGES.txt | 3 +-
.../org/apache/lucene/search/FieldExistsQuery.java | 40 +++++++-----
.../apache/lucene/search/TestFieldExistsQuery.java | 72 ++++++++++++++++++++++
3 files changed, 100 insertions(+), 15 deletions(-)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index d77f7634f00..d4c8c4d6b92 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -141,11 +141,12 @@ Improvements
Optimizations
---------------------
-
* GITHUB#16254: Enable the vectorized findNextGEQ path on aarch64, where it was disabled on cores
with fewer than 8 int lanes (Graviton2 and Graviton4). The vector compare beats the scalar
fallback by +75% on Graviton4 and +35% on Graviton2 (AdvanceBenchmark). (Shitij Bhargava)
+* GITHUB#16111: Optimize FieldExistsQuery.count() when all docs have the field. (Prithvi S)
+
* GITHUB#16222: MultiTermQuery constant-score wrapper now defers term collection to ScorerSupplier#get()
for queries with an unknown term count (automaton queries such as wildcard/regexp/prefix/range),
instead of scanning the term dictionary while building the ScorerSupplier. This keeps the
diff --git a/lucene/core/src/java/org/apache/lucene/search/FieldExistsQuery.java b/lucene/core/src/java/org/apache/lucene/search/FieldExistsQuery.java
index d4232efef08..c6c6f28c607 100644
--- a/lucene/core/src/java/org/apache/lucene/search/FieldExistsQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/FieldExistsQuery.java
@@ -231,32 +231,44 @@ public class FieldExistsQuery extends Query {
if (reader.getDocCount(field) == reader.maxDoc()) {
return reader.numDocs();
}
-
- return super.count(context);
- } else if (fieldInfo.hasVectorValues()) { // the field indexes vectors
- if (reader.hasDeletions() == false) {
- return getVectorValuesSize(fieldInfo, reader);
- }
return super.count(context);
+ }
+
+ int count = -1;
+ if (fieldInfo.hasVectorValues()) { // the field indexes vectors
+ count = getVectorValuesSize(fieldInfo, reader);
} else if (fieldInfo.getDocValuesType()
!= DocValuesType.NONE) { // the field indexes doc values
- if (reader.hasDeletions() == false) {
+ if (fieldInfo.docValuesSkipIndexType() != DocValuesSkipIndexType.NONE) {
+ DocValuesSkipper docValuesSkipper = reader.getDocValuesSkipper(field);
+ count = (docValuesSkipper == null ? 0 : docValuesSkipper.docCount());
+ } else if (reader.hasDeletions() == false) {
+ // No deletions: we can use points or terms doc count as a proxy for doc values.
if (fieldInfo.getPointDimensionCount() > 0) {
PointValues pointValues = reader.getPointValues(field);
- return pointValues == null ? 0 : pointValues.getDocCount();
+ count = (pointValues == null ? 0 : pointValues.getDocCount());
} else if (fieldInfo.getIndexOptions() != IndexOptions.NONE) {
Terms terms = reader.terms(field);
- return terms == null ? 0 : terms.getDocCount();
- } else if (fieldInfo.docValuesSkipIndexType() != DocValuesSkipIndexType.NONE) {
- DocValuesSkipper docValuesSkipper = reader.getDocValuesSkipper(field);
- return docValuesSkipper == null ? 0 : docValuesSkipper.docCount();
+ count = (terms == null ? 0 : terms.getDocCount());
}
}
-
- return super.count(context);
} else {
throw new IllegalStateException(buildErrorMsg(fieldInfo));
}
+
+ if (count == 0) {
+ // One of the above cases shows the field is not present on this leaf
+ return 0;
+ } else if (count == reader.maxDoc()) {
+ // All docs in the leaf (live or deleted) have the field. Return the count of live docs.
+ return reader.numDocs();
+ } else if (count >= 0 && reader.hasDeletions() == false) {
+ // No deleted docs. The computed count can be trusted.
+ return count;
+ }
+ // Some docs don't have the field and some docs are deleted.
+ // Need to scan to get the correct intersection between field exists docs and live docs.
+ return super.count(context);
}
@Override
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestFieldExistsQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestFieldExistsQuery.java
index 24535f227ac..9cb26fbd5dc 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestFieldExistsQuery.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestFieldExistsQuery.java
@@ -875,4 +875,76 @@ public class TestFieldExistsQuery extends LuceneTestCase {
final Weight weight = searcher.createWeight(testQuery, ScoreMode.COMPLETE, 1);
assertEquals(weight.count(reader.leaves().get(0)), numMatchingDocs);
}
+
+ /**
+ * When every document in a segment has a doc values field and some documents are deleted, count()
+ * should still return the number of live docs in O(1) instead of falling back to iteration.
+ */
+ public void testDocValuesCountWithDeletionsAllDocsHaveField() throws IOException {
+ Directory dir = newDirectory();
+ RandomIndexWriter w = new RandomIndexWriter(random(), dir);
+ int numDocs = atLeast(20);
+ for (int i = 0; i < numDocs; i++) {
+ Document doc = new Document();
+ // Use indexedField to ensure a DocValuesSkipper is available, which provides
+ // an exact doc count for doc values that we can use even with deletions.
+ doc.add(NumericDocValuesField.indexedField("dv", i));
+ doc.add(new StringField("id", Integer.toString(i), Store.NO));
+ w.addDocument(doc);
+ }
+ w.forceMerge(1);
+ // Delete some documents, but every remaining doc still has the field
+ w.w.getConfig().setMergePolicy(NoMergePolicy.INSTANCE);
+ w.deleteDocuments(new Term("id", "0"));
+ w.deleteDocuments(new Term("id", "1"));
+
+ DirectoryReader reader = w.getReader();
+ IndexSearcher searcher = new IndexSearcher(reader);
+ assertTrue("Expected deletions in the segment", reader.hasDeletions());
+ assertEquals(1, reader.leaves().size());
+
+ Query query = new FieldExistsQuery("dv");
+ Weight weight = searcher.createWeight(query, ScoreMode.COMPLETE, 1);
+ // All docs had the field, so even with deletions we can count in O(1)
+ int count = weight.count(reader.leaves().get(0));
+ assertEquals(reader.numDocs(), count);
+ assertEquals(numDocs - 2, count);
+
+ IOUtils.close(reader, w, dir);
+ }
+
+ /**
+ * When every document in a segment has a vector field and some documents are deleted, count()
+ * should still return the number of live docs in O(1) instead of falling back to iteration.
+ */
+ public void testVectorCountWithDeletionsAllDocsHaveField() throws IOException {
+ Directory dir = newDirectory();
+ RandomIndexWriter w = new RandomIndexWriter(random(), dir);
+ int numDocs = atLeast(20);
+ for (int i = 0; i < numDocs; i++) {
+ Document doc = new Document();
+ float[] vector = new float[] {i + 1, i + 2, i + 3, i + 4};
+ VectorUtil.l2normalize(vector);
+ doc.add(new KnnFloatVectorField("vector", vector));
+ doc.add(new StringField("id", Integer.toString(i), Store.NO));
+ w.addDocument(doc);
+ }
+ w.forceMerge(1);
+ w.w.getConfig().setMergePolicy(NoMergePolicy.INSTANCE);
+ w.deleteDocuments(new Term("id", "0"));
+
+ DirectoryReader reader = w.getReader();
+ IndexSearcher searcher = new IndexSearcher(reader);
+ assertTrue("Expected deletions in the segment", reader.hasDeletions());
+ assertEquals(1, reader.leaves().size());
+
+ Query query = new FieldExistsQuery("vector");
+ Weight weight = searcher.createWeight(query, ScoreMode.COMPLETE, 1);
+ // All docs had the vector field, so even with deletions we can count in O(1)
+ int count = weight.count(reader.leaves().get(0));
+ assertEquals(reader.numDocs(), count);
+ assertEquals(numDocs - 1, count);
+
+ IOUtils.close(reader, w, dir);
+ }
}