You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by gu...@apache.org on 2025/03/20 05:22:52 UTC
(lucene) branch main updated: Implement bulk adding methods for dynamic pruning. (#14365)
This is an automated email from the ASF dual-hosted git repository.
guofeng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/main by this push:
new b75602428f7 Implement bulk adding methods for dynamic pruning. (#14365)
b75602428f7 is described below
commit b75602428f78bd79296eb3b6be27465961b63d09
Author: Guo Feng <52...@users.noreply.github.com>
AuthorDate: Thu Mar 20 13:22:47 2025 +0800
Implement bulk adding methods for dynamic pruning. (#14365)
---
lucene/CHANGES.txt | 4 +++-
.../java/org/apache/lucene/index/PointValues.java | 2 ++
.../search/comparators/NumericComparator.java | 14 +++++++++++
.../org/apache/lucene/util/DocIdSetBuilder.java | 27 +++++++++++++++++++++-
4 files changed, 45 insertions(+), 2 deletions(-)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index c215696ad98..2d44a18b030 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -185,7 +185,9 @@ Optimizations
* GITHUB#14301: Avoid unnecessary evaluations and skipping documents. (hanbj)
-# GITHUB#14203: Decode doc ids in BKD leaves with auto-vectorized loops when using DEFAULT_MAX_POINTS_IN_LEAF_NODE. (Guo Feng)
+* GITHUB#14203: Decode doc ids in BKD leaves with auto-vectorized loops when using DEFAULT_MAX_POINTS_IN_LEAF_NODE. (Guo Feng)
+
+* GITHUB#14176: Avoid the per-doc virtual call when building competitive iterator for dynamic pruning. (Guo Feng)
# GITHUB#14361: Introduce new encoding of BPV 21 for DocIdsWriter used in BKD Tree. (Aniketh Jain, Guo Feng)
diff --git a/lucene/core/src/java/org/apache/lucene/index/PointValues.java b/lucene/core/src/java/org/apache/lucene/index/PointValues.java
index aebcaa90e2a..cbecad2d98b 100644
--- a/lucene/core/src/java/org/apache/lucene/index/PointValues.java
+++ b/lucene/core/src/java/org/apache/lucene/index/PointValues.java
@@ -291,6 +291,8 @@ public abstract class PointValues {
/**
* Similar to {@link IntersectVisitor#visit(int)}, but a bulk visit and implementations may have
* their optimizations.
+ *
+ * <p>It is guaranteed that the given iterator is not positioned;
*/
default void visit(DocIdSetIterator iterator) throws IOException {
int docID;
diff --git a/lucene/core/src/java/org/apache/lucene/search/comparators/NumericComparator.java b/lucene/core/src/java/org/apache/lucene/search/comparators/NumericComparator.java
index 1fae9a940e4..af18c293a7e 100644
--- a/lucene/core/src/java/org/apache/lucene/search/comparators/NumericComparator.java
+++ b/lucene/core/src/java/org/apache/lucene/search/comparators/NumericComparator.java
@@ -31,6 +31,7 @@ import org.apache.lucene.search.Scorable;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.util.DocIdSetBuilder;
import org.apache.lucene.util.FixedBitSet;
+import org.apache.lucene.util.IntsRef;
/**
* Abstract numeric comparator for comparing numeric values. This comparator provides a skipping
@@ -251,6 +252,19 @@ public abstract class NumericComparator<T extends Number> extends FieldComparato
}
}
+ @Override
+ public void visit(DocIdSetIterator iterator) throws IOException {
+ if (iterator.advance(maxDocVisited + 1) != DocIdSetIterator.NO_MORE_DOCS) {
+ adder.add(iterator.docID());
+ adder.add(iterator);
+ }
+ }
+
+ @Override
+ public void visit(IntsRef ref) {
+ adder.add(ref, maxDocVisited + 1);
+ }
+
@Override
public PointValues.Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {
long min = sortableBytesToLong(minPackedValue);
diff --git a/lucene/core/src/java/org/apache/lucene/util/DocIdSetBuilder.java b/lucene/core/src/java/org/apache/lucene/util/DocIdSetBuilder.java
index 6d0b9f3b860..2521a16727d 100644
--- a/lucene/core/src/java/org/apache/lucene/util/DocIdSetBuilder.java
+++ b/lucene/core/src/java/org/apache/lucene/util/DocIdSetBuilder.java
@@ -47,6 +47,8 @@ public final class DocIdSetBuilder {
void add(IntsRef docs);
void add(DocIdSetIterator iterator) throws IOException;
+
+ void add(IntsRef docs, int docLowerBoundInclusive);
}
private record FixedBitSetAdder(FixedBitSet bitSet) implements BulkAdder {
@@ -65,7 +67,18 @@ public final class DocIdSetBuilder {
@Override
public void add(DocIdSetIterator iterator) throws IOException {
- bitSet.or(iterator);
+ iterator.nextDoc();
+ iterator.intoBitSet(DocIdSetIterator.NO_MORE_DOCS, bitSet, 0);
+ }
+
+ @Override
+ public void add(IntsRef docs, int docLowerBoundInclusive) {
+ for (int i = docs.offset, to = docs.offset + docs.length; i < to; i++) {
+ int doc = docs.ints[i];
+ if (doc >= docLowerBoundInclusive) {
+ bitSet.set(doc);
+ }
+ }
}
}
@@ -104,6 +117,18 @@ public final class DocIdSetBuilder {
add(docID);
}
}
+
+ @Override
+ public void add(IntsRef docs, int docLowerBoundInclusive) {
+ int index = buffer.length;
+ for (int i = docs.offset, to = docs.offset + docs.length; i < to; i++) {
+ int doc = docs.ints[i];
+ if (doc >= docLowerBoundInclusive) {
+ buffer.array[index++] = doc;
+ }
+ }
+ buffer.length = index;
+ }
}
private final int maxDoc;