You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by gs...@apache.org on 2023/12/08 23:39:58 UTC
(lucene) branch branch_9x updated: Ensure #finish is called on all drill-sideways FacetCollectors even when no hits are scored (#12853)
This is an automated email from the ASF dual-hosted git repository.
gsmiller pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/branch_9x by this push:
new 06002e015dc Ensure #finish is called on all drill-sideways FacetCollectors even when no hits are scored (#12853)
06002e015dc is described below
commit 06002e015dc2827e1695d82f43e0039072d46490
Author: Greg Miller <gs...@gmail.com>
AuthorDate: Fri Dec 8 15:25:57 2023 -0800
Ensure #finish is called on all drill-sideways FacetCollectors even when no hits are scored (#12853)
---
lucene/CHANGES.txt | 3 +
.../org/apache/lucene/facet/DrillSideways.java | 15 ++-
.../apache/lucene/facet/DrillSidewaysQuery.java | 56 ++++-----
.../apache/lucene/facet/DrillSidewaysScorer.java | 33 ++---
.../org/apache/lucene/facet/FacetsCollector.java | 10 +-
.../org/apache/lucene/facet/TestDrillSideways.java | 136 ++++++++++++++++++---
6 files changed, 177 insertions(+), 76 deletions(-)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 2f5da5f09da..d1e31df0e6d 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -31,6 +31,9 @@ Bug Fixes
---------------------
* GITHUB#12866: Prevent extra similarity computation for single-level HNSW graphs. (Kaival Parikh)
+* GITHUB#12558: Ensure #finish is called on all drill-sideways FacetsCollectors even when no hits are scored.
+ (Greg Miller)
+
Other
---------------------
* GITHUB#11023: Removing some dead code in CheckIndex. (Jakub Slowinski)
diff --git a/lucene/facet/src/java/org/apache/lucene/facet/DrillSideways.java b/lucene/facet/src/java/org/apache/lucene/facet/DrillSideways.java
index c59e76a25d4..e5e111b5e30 100644
--- a/lucene/facet/src/java/org/apache/lucene/facet/DrillSideways.java
+++ b/lucene/facet/src/java/org/apache/lucene/facet/DrillSideways.java
@@ -151,6 +151,14 @@ public class DrillSideways {
return new FacetsCollectorManager();
}
+ /**
+ * Subclass can override to customize drill sideways facets collector. This should not return
+ * {@code null} as we assume drill sideways is being used to collect "sideways" hits:
+ */
+ protected FacetsCollectorManager createDrillSidewaysFacetsCollectorManager() {
+ return new FacetsCollectorManager();
+ }
+
/** Subclass can override to customize per-dim Facets impl. */
protected Facets buildFacetsResult(
FacetsCollector drillDowns, FacetsCollector[] drillSideways, String[] drillSidewaysDims)
@@ -501,7 +509,7 @@ public class DrillSideways {
FacetsCollectorManager[] drillSidewaysFacetsCollectorManagers =
new FacetsCollectorManager[numDims];
for (int i = 0; i < numDims; i++) {
- drillSidewaysFacetsCollectorManagers[i] = new FacetsCollectorManager();
+ drillSidewaysFacetsCollectorManagers[i] = createDrillSidewaysFacetsCollectorManager();
}
DrillSidewaysQuery dsq =
@@ -571,7 +579,10 @@ public class DrillSideways {
for (String dim : drillDownDims.keySet())
callableCollectors.add(
new CallableCollector(
- i++, searcher, getDrillDownQuery(query, filters, dim), new FacetsCollectorManager()));
+ i++,
+ searcher,
+ getDrillDownQuery(query, filters, dim),
+ createDrillSidewaysFacetsCollectorManager()));
final FacetsCollector mainFacetsCollector;
final FacetsCollector[] facetsCollectors = new FacetsCollector[drillDownDims.size()];
diff --git a/lucene/facet/src/java/org/apache/lucene/facet/DrillSidewaysQuery.java b/lucene/facet/src/java/org/apache/lucene/facet/DrillSidewaysQuery.java
index 88850df057c..e8a4b15d979 100644
--- a/lucene/facet/src/java/org/apache/lucene/facet/DrillSidewaysQuery.java
+++ b/lucene/facet/src/java/org/apache/lucene/facet/DrillSidewaysQuery.java
@@ -23,13 +23,13 @@ import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Objects;
-import org.apache.lucene.facet.DrillSidewaysScorer.DocsAndCost;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.search.BulkScorer;
import org.apache.lucene.search.ConstantScoreScorer;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.LeafCollector;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.QueryVisitor;
import org.apache.lucene.search.ScoreMode;
@@ -175,6 +175,17 @@ class DrillSidewaysQuery extends Query {
int drillDownCount = drillDowns.length;
+ FacetsCollector drillDownCollector;
+ LeafCollector drillDownLeafCollector;
+ if (drillDownCollectorManager != null) {
+ drillDownCollector = drillDownCollectorManager.newCollector();
+ managedDrillDownCollectors.add(drillDownCollector);
+ drillDownLeafCollector = drillDownCollector.getLeafCollector(context);
+ } else {
+ drillDownCollector = null;
+ drillDownLeafCollector = null;
+ }
+
FacetsCollector[] sidewaysCollectors = new FacetsCollector[drillDownCount];
managedDrillSidewaysCollectors.add(sidewaysCollectors);
@@ -193,42 +204,29 @@ class DrillSidewaysQuery extends Query {
FacetsCollector sidewaysCollector = drillSidewaysCollectorManagers[dim].newCollector();
sidewaysCollectors[dim] = sidewaysCollector;
- dims[dim] = new DrillSidewaysScorer.DocsAndCost(scorer, sidewaysCollector);
+ dims[dim] =
+ new DrillSidewaysScorer.DocsAndCost(
+ scorer, sidewaysCollector.getLeafCollector(context));
}
- // If more than one dim has no matches, then there
- // are no hits nor drill-sideways counts. Or, if we
- // have only one dim and that dim has no matches,
- // same thing.
- // if (nullCount > 1 || (nullCount == 1 && dims.length == 1)) {
- if (nullCount > 1) {
+ // If baseScorer is null or the dim nullCount > 1, then we have nothing to score. We return
+ // a null scorer in this case, but we need to make sure #finish gets called on all facet
+ // collectors since IndexSearcher won't handle this for us:
+ if (baseScorer == null || nullCount > 1) {
+ if (drillDownCollector != null) {
+ drillDownCollector.finish();
+ }
+ for (FacetsCollector fc : sidewaysCollectors) {
+ fc.finish();
+ }
return null;
}
// Sort drill-downs by most restrictive first:
- Arrays.sort(
- dims,
- new Comparator<DrillSidewaysScorer.DocsAndCost>() {
- @Override
- public int compare(DocsAndCost o1, DocsAndCost o2) {
- return Long.compare(o1.approximation.cost(), o2.approximation.cost());
- }
- });
-
- if (baseScorer == null) {
- return null;
- }
-
- FacetsCollector drillDownCollector;
- if (drillDownCollectorManager != null) {
- drillDownCollector = drillDownCollectorManager.newCollector();
- managedDrillDownCollectors.add(drillDownCollector);
- } else {
- drillDownCollector = null;
- }
+ Arrays.sort(dims, Comparator.comparingLong(o -> o.approximation.cost()));
return new DrillSidewaysScorer(
- context, baseScorer, drillDownCollector, dims, scoreSubDocsAtOnce);
+ context, baseScorer, drillDownLeafCollector, dims, scoreSubDocsAtOnce);
}
};
}
diff --git a/lucene/facet/src/java/org/apache/lucene/facet/DrillSidewaysScorer.java b/lucene/facet/src/java/org/apache/lucene/facet/DrillSidewaysScorer.java
index 85e1160eb81..372765215ff 100644
--- a/lucene/facet/src/java/org/apache/lucene/facet/DrillSidewaysScorer.java
+++ b/lucene/facet/src/java/org/apache/lucene/facet/DrillSidewaysScorer.java
@@ -24,7 +24,6 @@ import java.util.Comparator;
import java.util.List;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.search.BulkScorer;
-import org.apache.lucene.search.Collector;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.LeafCollector;
import org.apache.lucene.search.Scorable;
@@ -46,8 +45,7 @@ class DrillSidewaysScorer extends BulkScorer {
// private static boolean DEBUG = false;
- private final Collector drillDownCollector;
- private LeafCollector drillDownLeafCollector;
+ private final LeafCollector drillDownLeafCollector;
private final DocsAndCost[] dims;
@@ -70,7 +68,7 @@ class DrillSidewaysScorer extends BulkScorer {
DrillSidewaysScorer(
LeafReaderContext context,
Scorer baseScorer,
- Collector drillDownCollector,
+ LeafCollector drillDownLeafCollector,
DocsAndCost[] dims,
boolean scoreSubDocsAtOnce) {
this.dims = dims;
@@ -83,7 +81,7 @@ class DrillSidewaysScorer extends BulkScorer {
} else {
this.baseApproximation = baseIterator;
}
- this.drillDownCollector = drillDownCollector;
+ this.drillDownLeafCollector = drillDownLeafCollector;
this.scoreSubDocsAtOnce = scoreSubDocsAtOnce;
}
@@ -101,18 +99,6 @@ class DrillSidewaysScorer extends BulkScorer {
if (maxDoc != Integer.MAX_VALUE) {
throw new IllegalArgumentException("maxDoc must be Integer.MAX_VALUE");
}
- // if (DEBUG) {
- // System.out.println("\nscore: reader=" + context.reader());
- // }
- // System.out.println("score r=" + context.reader());
- if (drillDownCollector != null) {
- drillDownLeafCollector = drillDownCollector.getLeafCollector(context);
- } else {
- drillDownLeafCollector = null;
- }
- for (DocsAndCost dim : dims) {
- dim.sidewaysLeafCollector = dim.sidewaysCollector.getLeafCollector(context);
- }
// some scorers, eg ReqExlScorer, can hit NPE if cost is called after nextDoc
long baseQueryCost = baseIterator.cost();
@@ -722,7 +708,7 @@ class DrillSidewaysScorer extends BulkScorer {
// }
collector.collect(collectDocID);
- if (drillDownCollector != null) {
+ if (drillDownLeafCollector != null) {
drillDownLeafCollector.collect(collectDocID);
}
@@ -738,7 +724,7 @@ class DrillSidewaysScorer extends BulkScorer {
private void collectHit(LeafCollector collector, DocsAndCost dim) throws IOException {
collector.collect(collectDocID);
- if (drillDownCollector != null) {
+ if (drillDownLeafCollector != null) {
drillDownLeafCollector.collect(collectDocID);
}
@@ -748,7 +734,7 @@ class DrillSidewaysScorer extends BulkScorer {
private void collectHit(LeafCollector collector, List<DocsAndCost> dims) throws IOException {
collector.collect(collectDocID);
- if (drillDownCollector != null) {
+ if (drillDownLeafCollector != null) {
drillDownLeafCollector.collect(collectDocID);
}
@@ -812,10 +798,9 @@ class DrillSidewaysScorer extends BulkScorer {
// two-phase confirmation, or null if the approximation is accurate
final TwoPhaseIterator twoPhase;
final float matchCost;
- final Collector sidewaysCollector;
- LeafCollector sidewaysLeafCollector;
+ final LeafCollector sidewaysLeafCollector;
- DocsAndCost(Scorer scorer, Collector sidewaysCollector) {
+ DocsAndCost(Scorer scorer, LeafCollector sidewaysLeafCollector) {
final TwoPhaseIterator twoPhase = scorer.twoPhaseIterator();
if (twoPhase == null) {
this.approximation = scorer.iterator();
@@ -827,7 +812,7 @@ class DrillSidewaysScorer extends BulkScorer {
this.matchCost = twoPhase.matchCost();
}
this.cost = approximation.cost();
- this.sidewaysCollector = sidewaysCollector;
+ this.sidewaysLeafCollector = sidewaysLeafCollector;
}
}
}
diff --git a/lucene/facet/src/java/org/apache/lucene/facet/FacetsCollector.java b/lucene/facet/src/java/org/apache/lucene/facet/FacetsCollector.java
index 2bce78e22b3..41db7b6df45 100644
--- a/lucene/facet/src/java/org/apache/lucene/facet/FacetsCollector.java
+++ b/lucene/facet/src/java/org/apache/lucene/facet/FacetsCollector.java
@@ -143,8 +143,14 @@ public class FacetsCollector extends SimpleCollector {
@Override
public void finish() throws IOException {
- matchingDocs.add(new MatchingDocs(this.context, docsBuilder.build(), totalHits, scores));
- docsBuilder = null;
+ DocIdSet bits;
+ if (docsBuilder != null) {
+ bits = docsBuilder.build();
+ docsBuilder = null;
+ } else {
+ bits = DocIdSet.EMPTY;
+ }
+ matchingDocs.add(new MatchingDocs(this.context, bits, totalHits, scores));
scores = null;
context = null;
}
diff --git a/lucene/facet/src/test/org/apache/lucene/facet/TestDrillSideways.java b/lucene/facet/src/test/org/apache/lucene/facet/TestDrillSideways.java
index 29eba1b7e07..a4b7879e742 100644
--- a/lucene/facet/src/test/org/apache/lucene/facet/TestDrillSideways.java
+++ b/lucene/facet/src/test/org/apache/lucene/facet/TestDrillSideways.java
@@ -64,6 +64,7 @@ import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.LeafCollector;
import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.MatchNoDocsQuery;
import org.apache.lucene.search.PhraseQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.QueryCachingPolicy;
@@ -289,7 +290,7 @@ public class TestDrillSideways extends FacetTestCase {
FacetsCollector baseFC = new FacetsCollector();
FacetsCollector dimFC = new FacetsCollector();
DrillSidewaysScorer.DocsAndCost docsAndCost =
- new DrillSidewaysScorer.DocsAndCost(dimScorer, dimFC);
+ new DrillSidewaysScorer.DocsAndCost(dimScorer, dimFC.getLeafCollector(ctx));
LeafCollector baseCollector =
new LeafCollector() {
@@ -313,7 +314,7 @@ public class TestDrillSideways extends FacetTestCase {
new DrillSidewaysScorer(
ctx,
baseScorer,
- baseFC,
+ baseFC.getLeafCollector(ctx),
new DrillSidewaysScorer.DocsAndCost[] {docsAndCost},
scoreSubDocsAtOnce);
expectThrows(CollectionTerminatedException.class, () -> scorer.score(baseCollector, null));
@@ -407,23 +408,7 @@ public class TestDrillSideways extends FacetTestCase {
try (IndexReader r = w.getReader();
TaxonomyReader taxoR = new DirectoryTaxonomyReader(taxoW)) {
-
- // We can't use AssertingIndexSearcher unfortunately since it may randomly decide to bulk
- // score a sub-range of docs instead of all docs at once. This is incompatible will drill
- // sideways, so we have to do our own check here. This just makes sure we call #finish on
- // the last leaf. It's too bad we need to do this and maybe some day we can clean this up
- // by rethinking drill-sideways:
- IndexSearcher searcher =
- new IndexSearcher(r) {
- @Override
- protected void search(
- List<LeafReaderContext> leaves, Weight weight, Collector collector)
- throws IOException {
- AssertingCollector assertingCollector = AssertingCollector.wrap(collector);
- super.search(leaves, weight, assertingCollector);
- assert assertingCollector.hasFinishedCollectingPreviousLeaf;
- }
- };
+ IndexSearcher searcher = new DrillSidewaysAssertingIndexSearcher(r);
Query baseQuery = new MatchAllDocsQuery();
DrillDownQuery ddq = new DrillDownQuery(facetsConfig, baseQuery);
@@ -1483,6 +1468,66 @@ public class TestDrillSideways extends FacetTestCase {
IOUtils.close(r, tr, tw, d, td);
}
+ public void testFinishOnAllDimsNoHitsQuery() throws Exception {
+ try (Directory dir = newDirectory();
+ Directory taxoDir = newDirectory();
+ RandomIndexWriter w = new RandomIndexWriter(random(), dir);
+ DirectoryTaxonomyWriter taxoW =
+ new DirectoryTaxonomyWriter(taxoDir, IndexWriterConfig.OpenMode.CREATE)) {
+ FacetsConfig facetsConfig = new FacetsConfig();
+
+ Document d = new Document();
+ d.add(new FacetField("foo", "bar"));
+ w.addDocument(facetsConfig.build(taxoW, d));
+
+ try (IndexReader r = w.getReader();
+ TaxonomyReader taxoR = new DirectoryTaxonomyReader(taxoW)) {
+ IndexSearcher searcher = new DrillSidewaysAssertingIndexSearcher(r);
+
+ // Creating a query that matches nothing to make sure #finish still gets called on all
+ // facet collectors:
+ Query baseQuery = new MatchNoDocsQuery();
+ DrillDownQuery ddq = new DrillDownQuery(facetsConfig, baseQuery);
+ ddq.add("foo", "bar");
+ DrillSideways drillSideways =
+ new DrillSideways(searcher, facetsConfig, taxoR) {
+ @Override
+ protected FacetsCollectorManager createDrillDownFacetsCollectorManager() {
+ return new AssertingFacetsCollectorManager();
+ }
+
+ @Override
+ protected FacetsCollectorManager createDrillSidewaysFacetsCollectorManager() {
+ return new AssertingFacetsCollectorManager();
+ }
+ };
+
+ SimpleCollectorManager cm =
+ new SimpleCollectorManager(10, Comparator.comparingInt(a -> a.docAndScore.doc));
+ DrillSideways.ConcurrentDrillSidewaysResult<List<DocAndScore>> result =
+ drillSideways.search(ddq, cm);
+ assertEquals(0, result.collectorResult.size());
+
+ // Make sure the "matching docs" are still populated with the appropriate leaf reader
+ // context, which happens as part of #finish getting called:
+ assertEquals(1, result.drillDownFacetsCollector.getMatchingDocs().size());
+ assertEquals(
+ 1, result.drillDownFacetsCollector.getMatchingDocs().get(0).context.reader().maxDoc());
+ assertEquals(1, result.drillSidewaysFacetsCollector.length);
+ assertEquals(1, result.drillSidewaysFacetsCollector[0].getMatchingDocs().size());
+ assertEquals(
+ 1,
+ result
+ .drillSidewaysFacetsCollector[0]
+ .getMatchingDocs()
+ .get(0)
+ .context
+ .reader()
+ .maxDoc());
+ }
+ }
+ }
+
private static class Counters {
int[][] counts;
@@ -1634,6 +1679,59 @@ public class TestDrillSideways extends FacetTestCase {
}
}
+ // We can't use AssertingIndexSearcher unfortunately since it may randomly decide to bulk
+ // score a sub-range of docs instead of all docs at once. This is incompatible will drill
+ // sideways, so we have to do our own check here. This just makes sure we call #finish on
+ // the last leaf. It's too bad we need to do this and maybe some day we can clean this up
+ // by rethinking drill-sideways:
+ private static final class DrillSidewaysAssertingIndexSearcher extends IndexSearcher {
+
+ DrillSidewaysAssertingIndexSearcher(IndexReader r) {
+ super(r);
+ }
+
+ @Override
+ protected void search(List<LeafReaderContext> leaves, Weight weight, Collector collector)
+ throws IOException {
+ AssertingCollector assertingCollector = AssertingCollector.wrap(collector);
+ super.search(leaves, weight, assertingCollector);
+ assert assertingCollector.hasFinishedCollectingPreviousLeaf;
+ }
+ }
+
+ private static final class AssertingFacetsCollectorManager extends FacetsCollectorManager {
+ @Override
+ public FacetsCollector newCollector() throws IOException {
+ return new AssertingFacetsCollector();
+ }
+
+ @Override
+ public FacetsCollector reduce(Collection<FacetsCollector> collectors) throws IOException {
+ for (FacetsCollector fc : collectors) {
+ assert fc instanceof AssertingFacetsCollector;
+ assert ((AssertingFacetsCollector) fc).hasFinishedLastLeaf == true;
+ }
+ return super.reduce(collectors);
+ }
+ }
+
+ private static final class AssertingFacetsCollector extends FacetsCollector {
+ private boolean hasFinishedLastLeaf = true;
+
+ @Override
+ protected void doSetNextReader(LeafReaderContext context) throws IOException {
+ assert hasFinishedLastLeaf;
+ hasFinishedLastLeaf = false;
+ super.doSetNextReader(context);
+ }
+
+ @Override
+ public void finish() throws IOException {
+ hasFinishedLastLeaf = true;
+ super.finish();
+ }
+ }
+
private int[] getTopNOrds(final int[] counts, final String[] values, int topN) {
final int[] ids = new int[counts.length];
for (int i = 0; i < ids.length; i++) {