You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ja...@apache.org on 2026/06/15 12:06:16 UTC
(lucene) 03/03: Add DistinctValuesCollectorManager to enable concurrent search (#16217)
This is an automated email from the ASF dual-hosted git repository.
javanna pushed a commit to branch branch_10x
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit dc9c9bd3438dbc1f2bbc67882064cacb4af23162
Author: Luca Cavanna <ja...@apache.org>
AuthorDate: Mon Jun 15 13:28:39 2026 +0200
Add DistinctValuesCollectorManager to enable concurrent search (#16217)
Introduces DistinctValuesCollectorManager, a CollectorManager for DistinctValuesCollector that
creates one collector per segment slice and merges results by taking the union of distinct
values per group.
Updates TestDistinctValuesCollector to use the new manager together with
FirstPassGroupingCollectorManager, removing all deprecated IndexSearcher#search(Query, Collector) usages.
Relates to #12892
---
lucene/CHANGES.txt | 4 +
.../grouping/DistinctValuesCollectorManager.java | 89 +++++++++++++++
.../grouping/TestDistinctValuesCollector.java | 127 +++++++++++----------
3 files changed, 157 insertions(+), 63 deletions(-)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index c61a0e5a422..e1b15c98406 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -130,6 +130,10 @@ Improvements
* GITHUB#16144: Clarify LeafCollector batch collection scoring contract. (Costin Leau)
+* GITHUB#16229: BooleanWeight.explain prints failing optional clauses when query fails due to insufficient SHOULD matches. (Jakub Slowinski)
+
+* GITHUB#16217: Introduce DistinctValuesCollectorManager to parallelize search when using DistinctValuesCollector. (Luca Cavanna)
+
Optimizations
---------------------
diff --git a/lucene/grouping/src/java/org/apache/lucene/search/grouping/DistinctValuesCollectorManager.java b/lucene/grouping/src/java/org/apache/lucene/search/grouping/DistinctValuesCollectorManager.java
new file mode 100644
index 00000000000..7d13335cee9
--- /dev/null
+++ b/lucene/grouping/src/java/org/apache/lucene/search/grouping/DistinctValuesCollectorManager.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.search.grouping;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.function.Supplier;
+import org.apache.lucene.search.CollectorManager;
+
+/**
+ * A {@link CollectorManager} implementation for {@link DistinctValuesCollector} that supports
+ * parallel collection and merges results by taking the union of distinct values per group across
+ * segments.
+ *
+ * @lucene.experimental
+ */
+public class DistinctValuesCollectorManager<T, R>
+ implements CollectorManager<
+ DistinctValuesCollector<T, R>, List<DistinctValuesCollector.GroupCount<T, R>>> {
+
+ private final Supplier<GroupSelector<T>> groupSelectorFactory;
+ private final Collection<SearchGroup<T>> searchGroups;
+ private final Supplier<GroupSelector<R>> valueSelectorFactory;
+
+ /**
+ * Creates a new DistinctValuesCollectorManager.
+ *
+ * @param groupSelectorFactory factory to create group selectors for each collector
+ * @param searchGroups the search groups from the first pass
+ * @param valueSelectorFactory factory to create value selectors for each collector
+ */
+ public DistinctValuesCollectorManager(
+ Supplier<GroupSelector<T>> groupSelectorFactory,
+ Collection<SearchGroup<T>> searchGroups,
+ Supplier<GroupSelector<R>> valueSelectorFactory) {
+ this.groupSelectorFactory = groupSelectorFactory;
+ this.searchGroups = searchGroups;
+ this.valueSelectorFactory = valueSelectorFactory;
+ }
+
+ @Override
+ public DistinctValuesCollector<T, R> newCollector() throws IOException {
+ return new DistinctValuesCollector<>(
+ groupSelectorFactory.get(), searchGroups, valueSelectorFactory.get());
+ }
+
+ @Override
+ public List<DistinctValuesCollector.GroupCount<T, R>> reduce(
+ Collection<DistinctValuesCollector<T, R>> collectors) {
+ List<List<DistinctValuesCollector.GroupCount<T, R>>> allGroupsList = new ArrayList<>();
+ for (DistinctValuesCollector<T, R> collector : collectors) {
+ allGroupsList.add(collector.getGroups());
+ }
+ if (allGroupsList.isEmpty()) {
+ return List.of();
+ }
+ // Merge by taking the union of uniqueValues for each group across all collectors.
+ // All collectors share the same searchGroups so position j in each list is the same group.
+ List<DistinctValuesCollector.GroupCount<T, R>> first = allGroupsList.get(0);
+ List<DistinctValuesCollector.GroupCount<T, R>> merged = new ArrayList<>(first.size());
+ for (int j = 0; j < first.size(); j++) {
+ Set<R> union = new HashSet<>(first.get(j).uniqueValues);
+ for (int i = 1; i < allGroupsList.size(); i++) {
+ assert allGroupsList.get(i).size() == first.size();
+ union.addAll(allGroupsList.get(i).get(j).uniqueValues);
+ }
+ merged.add(new DistinctValuesCollector.GroupCount<>(first.get(j).groupValue, union));
+ }
+ return merged;
+ }
+}
diff --git a/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestDistinctValuesCollector.java b/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestDistinctValuesCollector.java
index a55ba667694..6d844f5bd88 100644
--- a/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestDistinctValuesCollector.java
+++ b/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestDistinctValuesCollector.java
@@ -16,7 +16,6 @@
*/
package org.apache.lucene.search.grouping;
-import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -29,6 +28,7 @@ import java.util.Locale;
import java.util.Map;
import java.util.Random;
import java.util.Set;
+import java.util.function.Supplier;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.SortedDocValuesField;
@@ -134,16 +134,22 @@ public class TestDistinctValuesCollector extends AbstractGroupingTestCase {
}
};
- // === Search for content:random
- FirstPassGroupingCollector<Comparable<Object>> firstCollector =
- createRandomFirstPassCollector(new Sort(), GROUP_FIELD, 10);
- indexSearcher.search(new TermQuery(new Term("content", "random")), firstCollector);
- DistinctValuesCollector<Comparable<Object>, Comparable<Object>> distinctValuesCollector =
- createDistinctCountCollector(firstCollector, COUNT_FIELD);
- indexSearcher.search(new TermQuery(new Term("content", "random")), distinctValuesCollector);
+ boolean useValueSource = random.nextBoolean();
+ Supplier<GroupSelector<Comparable<Object>>> groupSelectorFactory =
+ createGroupSelectorFactory(GROUP_FIELD, useValueSource);
+ Supplier<GroupSelector<Comparable<Object>>> valueSelectorFactory =
+ createGroupSelectorFactory(COUNT_FIELD, useValueSource);
+ // === Search for content:random
+ Collection<SearchGroup<Comparable<Object>>> searchGroups =
+ indexSearcher.search(
+ new TermQuery(new Term("content", "random")),
+ new FirstPassGroupingCollectorManager<>(groupSelectorFactory, new Sort(), 0, 10));
List<DistinctValuesCollector.GroupCount<Comparable<Object>, Comparable<Object>>> gcs =
- distinctValuesCollector.getGroups();
+ indexSearcher.search(
+ new TermQuery(new Term("content", "random")),
+ new DistinctValuesCollectorManager<>(
+ groupSelectorFactory, searchGroups, valueSelectorFactory));
Collections.sort(gcs, cmp);
assertEquals(4, gcs.size());
@@ -170,12 +176,15 @@ public class TestDistinctValuesCollector extends AbstractGroupingTestCase {
compare("1", countValues.get(0));
// === Search for content:some
- firstCollector = createRandomFirstPassCollector(new Sort(), GROUP_FIELD, 10);
- indexSearcher.search(new TermQuery(new Term("content", "some")), firstCollector);
- distinctValuesCollector = createDistinctCountCollector(firstCollector, COUNT_FIELD);
- indexSearcher.search(new TermQuery(new Term("content", "some")), distinctValuesCollector);
-
- gcs = distinctValuesCollector.getGroups();
+ searchGroups =
+ indexSearcher.search(
+ new TermQuery(new Term("content", "some")),
+ new FirstPassGroupingCollectorManager<>(groupSelectorFactory, new Sort(), 0, 10));
+ gcs =
+ indexSearcher.search(
+ new TermQuery(new Term("content", "some")),
+ new DistinctValuesCollectorManager<>(
+ groupSelectorFactory, searchGroups, valueSelectorFactory));
Collections.sort(gcs, cmp);
assertEquals(3, gcs.size());
@@ -197,12 +206,15 @@ public class TestDistinctValuesCollector extends AbstractGroupingTestCase {
compare("1", countValues.get(0));
// === Search for content:blob
- firstCollector = createRandomFirstPassCollector(new Sort(), GROUP_FIELD, 10);
- indexSearcher.search(new TermQuery(new Term("content", "blob")), firstCollector);
- distinctValuesCollector = createDistinctCountCollector(firstCollector, COUNT_FIELD);
- indexSearcher.search(new TermQuery(new Term("content", "blob")), distinctValuesCollector);
-
- gcs = distinctValuesCollector.getGroups();
+ searchGroups =
+ indexSearcher.search(
+ new TermQuery(new Term("content", "blob")),
+ new FirstPassGroupingCollectorManager<>(groupSelectorFactory, new Sort(), 0, 10));
+ gcs =
+ indexSearcher.search(
+ new TermQuery(new Term("content", "blob")),
+ new DistinctValuesCollectorManager<>(
+ groupSelectorFactory, searchGroups, valueSelectorFactory));
Collections.sort(gcs, cmp);
assertEquals(2, gcs.size());
@@ -235,25 +247,36 @@ public class TestDistinctValuesCollector extends AbstractGroupingTestCase {
List<DistinctValuesCollector.GroupCount<Comparable<Object>, Comparable<Object>>>
expectedResult = createExpectedResult(context, term, groupSort, topN);
- FirstPassGroupingCollector<Comparable<Object>> firstCollector =
- createRandomFirstPassCollector(groupSort, GROUP_FIELD, topN);
- searcher.search(new TermQuery(new Term("content", term)), firstCollector);
- DistinctValuesCollector<Comparable<Object>, Comparable<Object>> distinctValuesCollector =
- createDistinctCountCollector(firstCollector, COUNT_FIELD);
- searcher.search(new TermQuery(new Term("content", term)), distinctValuesCollector);
+ boolean useValueSource = random.nextBoolean();
+ Supplier<GroupSelector<Comparable<Object>>> groupSelectorFactory =
+ createGroupSelectorFactory(GROUP_FIELD, useValueSource);
+ Supplier<GroupSelector<Comparable<Object>>> valueSelectorFactory =
+ createGroupSelectorFactory(COUNT_FIELD, useValueSource);
+
+ Collection<SearchGroup<Comparable<Object>>> searchGroups =
+ searcher.search(
+ new TermQuery(new Term("content", term)),
+ new FirstPassGroupingCollectorManager<>(groupSelectorFactory, groupSort, 0, topN));
+
+ if (searchGroups == null || searchGroups.isEmpty()) {
+ assertEquals(0, expectedResult.size());
+ continue;
+ }
+
@SuppressWarnings("unchecked")
List<DistinctValuesCollector.GroupCount<Comparable<Object>, Comparable<Object>>>
- actualResult = distinctValuesCollector.getGroups();
+ actualResult =
+ searcher.search(
+ new TermQuery(new Term("content", term)),
+ new DistinctValuesCollectorManager<>(
+ groupSelectorFactory, searchGroups, valueSelectorFactory));
if (VERBOSE) {
System.out.println("Index iter=" + indexIter);
System.out.println("Search iter=" + searchIter);
- System.out.println(
- "1st pass collector class name=" + firstCollector.getClass().getName());
- System.out.println(
- "2nd pass collector class name=" + distinctValuesCollector.getClass().getName());
+ System.out.println("useValueSource=" + useValueSource);
System.out.println("Search term=" + term);
- System.out.println("1st pass groups=" + firstCollector.getTopGroups(0));
+ System.out.println("1st pass groups=" + searchGroups);
System.out.println("Expected:");
printGroups(expectedResult);
System.out.println("Actual:");
@@ -350,37 +373,15 @@ public class TestDistinctValuesCollector extends AbstractGroupingTestCase {
doc.add(new SortedDocValuesField(field, new BytesRef(value)));
}
- @SuppressWarnings({"unchecked", "rawtypes"})
- private <T extends Comparable<Object>, R extends Comparable<Object>>
- DistinctValuesCollector<T, R> createDistinctCountCollector(
- FirstPassGroupingCollector<T> firstPassGroupingCollector, String countField)
- throws IOException {
- Collection<SearchGroup<T>> searchGroups = firstPassGroupingCollector.getTopGroups(0);
- GroupSelector<T> selector = firstPassGroupingCollector.getGroupSelector();
- if (ValueSourceGroupSelector.class.isAssignableFrom(selector.getClass())) {
- GroupSelector gs =
- new ValueSourceGroupSelector(new BytesRefFieldSource(countField), new HashMap<>());
- return new DistinctValuesCollector<>(selector, searchGroups, gs);
- } else {
- GroupSelector ts = new TermGroupSelector(countField);
- return new DistinctValuesCollector<>(selector, searchGroups, ts);
- }
- }
-
- @SuppressWarnings({"unchecked", "rawtypes"})
- private <T> FirstPassGroupingCollector<T> createRandomFirstPassCollector(
- Sort groupSort, String groupField, int topNGroups) throws IOException {
- Random random = random();
- if (random.nextBoolean()) {
- return (FirstPassGroupingCollector<T>)
- new FirstPassGroupingCollector<>(
- new ValueSourceGroupSelector(new BytesRefFieldSource(groupField), new HashMap<>()),
- groupSort,
- topNGroups);
+ @SuppressWarnings({"unchecked"})
+ private static <T> Supplier<GroupSelector<T>> createGroupSelectorFactory(
+ String field, boolean useValueSource) {
+ if (useValueSource) {
+ return () ->
+ (GroupSelector<T>)
+ new ValueSourceGroupSelector(new BytesRefFieldSource(field), new HashMap<>());
} else {
- return (FirstPassGroupingCollector<T>)
- new FirstPassGroupingCollector<>(
- new TermGroupSelector(groupField), groupSort, topNGroups);
+ return () -> (GroupSelector<T>) new TermGroupSelector(field);
}
}