You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ro...@apache.org on 2026/02/06 11:38:06 UTC
(lucene) branch branch_10x updated: Allow NumericComparator subclasses to implement their own competitive iterators (#15663)
This is an automated email from the ASF dual-hosted git repository.
romseygeek pushed a commit to branch branch_10x
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/branch_10x by this push:
new cef73d404fe Allow NumericComparator subclasses to implement their own competitive iterators (#15663)
cef73d404fe is described below
commit cef73d404fe59abeff4ca70d4f495d7313d74a28
Author: Alan Woodward <ro...@apache.org>
AuthorDate: Fri Feb 6 11:28:54 2026 +0000
Allow NumericComparator subclasses to implement their own competitive iterators (#15663)
This commit changes NumericComparator.CompetitiveDISIBuilder from
private to protected to allow subclasses to implement their own
iterators, and adds a test class illustrating how this would work.
---
lucene/CHANGES.txt | 3 +-
.../core/tests/TestComparatorSubclassing.java | 102 ++++++++++++++++++++
.../search/comparators/NumericComparator.java | 107 +++++++++++++--------
3 files changed, 170 insertions(+), 42 deletions(-)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 08c51862bf0..28dfd13eec9 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -7,7 +7,8 @@ http://s.apache.org/luceneversions
API Changes
---------------------
-(No changes)
+* GITHUB#15663: Allow subclasses of NumericComparator to implement their own
+ CompetitiveDISIBuilder subtypes. (Alan Woodward)
New Features
---------------------
diff --git a/lucene/core.tests/src/test/org/apache/lucene/core/tests/TestComparatorSubclassing.java b/lucene/core.tests/src/test/org/apache/lucene/core/tests/TestComparatorSubclassing.java
new file mode 100644
index 00000000000..63e1ad5c3e9
--- /dev/null
+++ b/lucene/core.tests/src/test/org/apache/lucene/core/tests/TestComparatorSubclassing.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.core.tests;
+
+import java.io.IOException;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.FieldComparator;
+import org.apache.lucene.search.FieldComparatorSource;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.LeafFieldComparator;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.Pruning;
+import org.apache.lucene.search.Sort;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.TotalHits;
+import org.apache.lucene.search.comparators.LongComparator;
+import org.apache.lucene.search.comparators.NumericComparator;
+import org.apache.lucene.tests.index.RandomIndexWriter;
+import org.apache.lucene.tests.util.LuceneTestCase;
+
+public class TestComparatorSubclassing extends LuceneTestCase {
+
+ public void testPluggableCompetitiveIterators() throws IOException {
+
+ boolean[] updated = new boolean[] {false};
+
+ LongComparator extendedComparator =
+ new LongComparator(10, "field", 0L, true, Pruning.NONE) {
+ @Override
+ public LeafFieldComparator getLeafComparator(LeafReaderContext context)
+ throws IOException {
+ return new LongLeafComparator(context) {
+ @Override
+ protected NumericComparator<Long>.CompetitiveDISIBuilder buildCompetitiveDISIBuilder()
+ throws IOException {
+ int maxDoc = context.reader().maxDoc();
+ int jumpTo = Math.max(0, maxDoc - 10);
+ return new CompetitiveDISIBuilder(this) {
+ @Override
+ protected boolean hasMissingDocs() {
+ return false;
+ }
+
+ @Override
+ protected void doUpdateCompetitiveIterator() {
+ updateCompetitiveIterator(DocIdSetIterator.range(jumpTo, maxDoc));
+ updated[0] = true;
+ }
+ };
+ }
+ };
+ }
+ };
+
+ try (var dir = newDirectory();
+ var writer = new RandomIndexWriter(random(), dir)) {
+ for (int i = 0; i < 2000; i++) {
+ Document doc = new Document();
+ doc.add(NumericDocValuesField.indexedField("field", i));
+ writer.addDocument(doc);
+ }
+ writer.commit();
+
+ try (var reader = writer.getReader()) {
+ IndexSearcher searcher = new IndexSearcher(reader);
+ Sort sort =
+ new Sort(
+ new SortField(
+ "field",
+ new FieldComparatorSource() {
+ @Override
+ public FieldComparator<?> newComparator(
+ String fieldname, int numHits, Pruning pruning, boolean reversed) {
+ return extendedComparator;
+ }
+ }));
+
+ TopDocs td = searcher.search(MatchAllDocsQuery.INSTANCE, 10, sort);
+ assertEquals(TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO, td.totalHits.relation());
+ assertTrue(updated[0]);
+ }
+ }
+ }
+}
diff --git a/lucene/core/src/java/org/apache/lucene/search/comparators/NumericComparator.java b/lucene/core/src/java/org/apache/lucene/search/comparators/NumericComparator.java
index 1aa704b36c5..5b62de7345b 100644
--- a/lucene/core/src/java/org/apache/lucene/search/comparators/NumericComparator.java
+++ b/lucene/core/src/java/org/apache/lucene/search/comparators/NumericComparator.java
@@ -103,20 +103,23 @@ public abstract class NumericComparator<T extends Number> extends FieldComparato
public NumericLeafComparator(LeafReaderContext context) throws IOException {
this.context = context;
this.docValues = getNumericDocValues(context, field);
- CompetitiveDISIBuilder builder = null;
- if (pruning != Pruning.NONE) {
- LeafReader reader = context.reader();
- PointValues pointValues = reader.getPointValues(field);
- if (pointValues != null) {
- builder = new PointsCompetitiveDISIBuilder(pointValues, this);
- } else {
- DocValuesSkipper skipper = reader.getDocValuesSkipper(field);
- if (skipper != null) {
- builder = new DVSkipperCompetitiveDISIBuilder(skipper, this);
- }
- }
+ this.competitiveDISIBuilder = buildCompetitiveDISIBuilder();
+ }
+
+ protected CompetitiveDISIBuilder buildCompetitiveDISIBuilder() throws IOException {
+ if (pruning == Pruning.NONE) {
+ return null;
}
- competitiveDISIBuilder = builder;
+ LeafReader reader = context.reader();
+ PointValues pointValues = reader.getPointValues(field);
+ if (pointValues != null) {
+ return new PointsCompetitiveDISIBuilder(pointValues, this);
+ }
+ DocValuesSkipper skipper = reader.getDocValuesSkipper(field);
+ if (skipper != null) {
+ return new DVSkipperCompetitiveDISIBuilder(skipper, this);
+ }
+ return null;
}
/**
@@ -176,22 +179,47 @@ public abstract class NumericComparator<T extends Number> extends FieldComparato
protected abstract long topAsComparableLong();
}
- private abstract class CompetitiveDISIBuilder {
+ /** Constructs a competitive iterator which can be updated during collection */
+ protected abstract class CompetitiveDISIBuilder {
+
+ /** maxDoc of the current segment */
+ protected final int maxDoc;
- final int maxDoc;
- final NumericLeafComparator leafComparator;
+ /** The LeafComparator for the current segment */
+ protected final NumericLeafComparator leafComparator;
/** According to {@link FieldComparator#setTopValue}, topValueSet is final in leafComparator */
final boolean leafTopSet = topValueSet;
- final UpdateableDocIdSetIterator competitiveIterator = new UpdateableDocIdSetIterator();
- long minValueAsLong = Long.MIN_VALUE;
- long maxValueAsLong = Long.MAX_VALUE;
+ private final UpdateableDocIdSetIterator competitiveIterator = new UpdateableDocIdSetIterator();
+
+ /** The current minimum value encoded as a long */
+ protected long minValueAsLong = Long.MIN_VALUE;
+
+ /** The current maximum value encoded as a long */
+ protected long maxValueAsLong = Long.MAX_VALUE;
+
int maxDocVisited = -1;
int updateCounter = 0;
int currentSkipInterval = MIN_SKIP_INTERVAL;
- CompetitiveDISIBuilder(NumericLeafComparator leafComparator) {
+ /** Are there documents in this segment with no value for the comparator to use */
+ protected abstract boolean hasMissingDocs();
+
+ /**
+ * Build a new DocIdSetIterator to use as a competitive iterator, if the new iterator will
+ * filter out enough documents. Implementations should pass their newly built iterators to
+ * {@link #updateCompetitiveIterator(DocIdSetIterator)}
+ */
+ protected abstract void doUpdateCompetitiveIterator() throws IOException;
+
+ /** Use a new iterator as the competitive iterator for collection */
+ protected final void updateCompetitiveIterator(DocIdSetIterator iterator) {
+ this.competitiveIterator.update(iterator);
+ }
+
+ /** Create a new CompetitiveDISIBuilder */
+ protected CompetitiveDISIBuilder(NumericLeafComparator leafComparator) {
this.leafComparator = leafComparator;
this.maxDoc = leafComparator.context.reader().maxDoc();
this.competitiveIterator.update(DocIdSetIterator.all(maxDoc));
@@ -202,8 +230,6 @@ public abstract class NumericComparator<T extends Number> extends FieldComparato
void setScorer(Scorable scorer) throws IOException {}
- abstract int docCount();
-
final void updateCompetitiveIterator() throws IOException {
if (hitsThresholdReached == false) {
return;
@@ -213,7 +239,7 @@ public abstract class NumericComparator<T extends Number> extends FieldComparato
}
// if some documents have missing points, check that missing values prohibits optimization
- if (docCount() < maxDoc && isMissingValueCompetitive()) {
+ if (hasMissingDocs() && isMissingValueCompetitive()) {
return;
}
@@ -231,8 +257,6 @@ public abstract class NumericComparator<T extends Number> extends FieldComparato
doUpdateCompetitiveIterator();
}
- abstract void doUpdateCompetitiveIterator() throws IOException;
-
private void setMaxDocVisited(int maxDocVisited) {
this.maxDocVisited = maxDocVisited;
}
@@ -361,8 +385,8 @@ public abstract class NumericComparator<T extends Number> extends FieldComparato
}
@Override
- int docCount() {
- return pointValues.getDocCount();
+ protected boolean hasMissingDocs() {
+ return pointValues.getDocCount() != maxDoc;
}
/**
@@ -375,22 +399,22 @@ public abstract class NumericComparator<T extends Number> extends FieldComparato
if (queueFull && hitsThresholdReached) {
// if some documents have missing points, then check that missing values prohibits
// optimization
- if (docCount() < maxDoc && isMissingValueCompetitive()) {
+ if (hasMissingDocs() && isMissingValueCompetitive()) {
return;
}
long bottom = leafComparator.bottomAsComparableLong();
long minValue = sortableBytesToLong(pointValues.getMinPackedValue());
long maxValue = sortableBytesToLong(pointValues.getMaxPackedValue());
if (reverse == false && bottom < minValue) {
- competitiveIterator.update(DocIdSetIterator.empty());
+ updateCompetitiveIterator(DocIdSetIterator.empty());
} else if (reverse && bottom > maxValue) {
- competitiveIterator.update(DocIdSetIterator.empty());
+ updateCompetitiveIterator(DocIdSetIterator.empty());
}
}
}
@Override
- void doUpdateCompetitiveIterator() throws IOException {
+ protected void doUpdateCompetitiveIterator() throws IOException {
DocIdSetBuilder result = new DocIdSetBuilder(maxDoc);
PointValues.IntersectVisitor visitor =
new PointValues.IntersectVisitor() {
@@ -462,15 +486,16 @@ public abstract class NumericComparator<T extends Number> extends FieldComparato
updateSkipInterval(false);
if (pointValues.getDocCount() < iteratorCost) {
// Use the set of doc with values to help drive iteration
- competitiveIterator.update(
+ updateCompetitiveIterator(
leafComparator.getNumericDocValues(leafComparator.context, field));
iteratorCost = pointValues.getDocCount();
}
return;
}
pointValues.intersect(visitor);
- competitiveIterator.update(result.build().iterator());
- iteratorCost = competitiveIterator.cost();
+ DocIdSetIterator newIterator = result.build().iterator();
+ updateCompetitiveIterator(newIterator);
+ iteratorCost = newIterator.cost();
updateSkipInterval(true);
}
@@ -510,8 +535,8 @@ public abstract class NumericComparator<T extends Number> extends FieldComparato
}
@Override
- int docCount() {
- return skipper.docCount();
+ protected boolean hasMissingDocs() {
+ return skipper.docCount() != maxDoc;
}
/**
@@ -522,21 +547,21 @@ public abstract class NumericComparator<T extends Number> extends FieldComparato
if (queueFull && hitsThresholdReached) {
// if some documents have missing doc values, check that missing values prohibits
// optimization
- if (docCount() < maxDoc && isMissingValueCompetitive()) {
+ if (hasMissingDocs() && isMissingValueCompetitive()) {
return;
}
long bottom = leafComparator.bottomAsComparableLong();
if (reverse == false && bottom < skipper.minValue()) {
- competitiveIterator.update(DocIdSetIterator.empty());
+ updateCompetitiveIterator(DocIdSetIterator.empty());
} else if (reverse && bottom > skipper.maxValue()) {
- competitiveIterator.update(DocIdSetIterator.empty());
+ updateCompetitiveIterator(DocIdSetIterator.empty());
}
}
}
@Override
- void doUpdateCompetitiveIterator() {
- competitiveIterator.update(
+ protected void doUpdateCompetitiveIterator() {
+ updateCompetitiveIterator(
new SkipBlockRangeIterator(skipper, minValueAsLong, maxValueAsLong));
}
}