You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@solr.apache.org by ds...@apache.org on 2026/04/18 04:07:42 UTC
(solr) branch main updated: SOLR-18195: CombinedQueryComponent supports Collapse now (#4277)
This is an automated email from the ASF dual-hosted git repository.
dsmiley pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/main by this push:
new ef75ab82a5a SOLR-18195: CombinedQueryComponent supports Collapse now (#4277)
ef75ab82a5a is described below
commit ef75ab82a5a8ee56a64e2bc70302904fcf86c8dd
Author: Sonu Sharma <so...@gmail.com>
AuthorDate: Sat Apr 18 09:37:36 2026 +0530
SOLR-18195: CombinedQueryComponent supports Collapse now (#4277)
Support for using {!collapse} with CombinedQueryComponent (RRF)
---
.../SOLR-18195-collapse-results-combined-query.yml | 9 ++
.../handler/component/CombinedQueryComponent.java | 49 ++++++-
.../combine/QueryAndResponseCombiner.java | 153 ++++++++++++++++++++-
.../collection1/conf/schema-vector-catchall.xml | 8 +-
.../component/CombinedQuerySolrCloudTest.java | 93 ++++++++++++-
.../DistributedCombinedQueryComponentTest.java | 139 +++++++++++++++++++
.../combine/QueryAndResponseCombinerTest.java | 106 +++++++++++++-
.../combine/ReciprocalRankFusionTest.java | 2 +-
8 files changed, 542 insertions(+), 17 deletions(-)
diff --git a/changelog/unreleased/SOLR-18195-collapse-results-combined-query.yml b/changelog/unreleased/SOLR-18195-collapse-results-combined-query.yml
new file mode 100644
index 00000000000..5ddc729b3dc
--- /dev/null
+++ b/changelog/unreleased/SOLR-18195-collapse-results-combined-query.yml
@@ -0,0 +1,9 @@
+# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc
+title: Support for using {!collapse} with CombinedQueryComponent (RRF)
+type: added
+authors:
+ - name: Sonu Sharma
+ nick: ercsonusharma
+links:
+ - name: SOLR-18195
+ url: https://issues.apache.org/jira/browse/SOLR-18195
diff --git a/solr/core/src/java/org/apache/solr/handler/component/CombinedQueryComponent.java b/solr/core/src/java/org/apache/solr/handler/component/CombinedQueryComponent.java
index 9831281d2b2..8b097bc304d 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/CombinedQueryComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/CombinedQueryComponent.java
@@ -31,6 +31,7 @@ import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import org.apache.lucene.search.Explanation;
+import org.apache.lucene.search.Query;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.solr.client.solrj.SolrServerException;
@@ -42,6 +43,7 @@ import org.apache.solr.common.params.CursorMarkParams;
import org.apache.solr.common.params.GroupParams;
import org.apache.solr.common.params.ShardParams;
import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.CollectionUtil;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.common.util.StrUtils;
@@ -53,8 +55,10 @@ import org.apache.solr.response.ResultContext;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.SchemaField;
+import org.apache.solr.search.CollapsingQParserPlugin;
import org.apache.solr.search.DocListAndSet;
import org.apache.solr.search.QueryResult;
+import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.search.SolrReturnFields;
import org.apache.solr.search.SortSpec;
import org.apache.solr.util.SolrResponseUtil;
@@ -236,7 +240,10 @@ public class CombinedQueryComponent extends QueryComponent implements SolrCoreAw
boolean partialResults,
boolean segmentTerminatedEarly,
Boolean setMaxHitsTerminatedEarly) {
- QueryResult combinedQueryResult = QueryAndResponseCombiner.simpleCombine(queryResults);
+ SolrIndexSearcher searcher = crb.req.getSearcher();
+ List<Query> collapseFilters = getCollapseFilters(crb.getFilters());
+ QueryResult combinedQueryResult =
+ QueryAndResponseCombiner.simpleCombine(queryResults, collapseFilters, searcher);
combinedQueryResult.setPartialResults(partialResults);
combinedQueryResult.setSegmentTerminatedEarly(segmentTerminatedEarly);
combinedQueryResult.setMaxHitsTerminatedEarly(setMaxHitsTerminatedEarly);
@@ -258,6 +265,21 @@ public class CombinedQueryComponent extends QueryComponent implements SolrCoreAw
}
}
+ /** Extracts the list of CollapsingPostFilter query from the filter list, if present. */
+ private static List<Query> getCollapseFilters(List<Query> filters) {
+ if (CollectionUtil.isNotEmpty(filters)) {
+ return filters.stream()
+ .filter(q -> q instanceof CollapsingQParserPlugin.CollapsingPostFilter)
+ .toList();
+ }
+ return List.of();
+ }
+
+ /**
+ * Each shard response contains both "response" and "response_per_query". Only "response" is
+ * deduplicated across sub-queries, so any processing of "response_per_query" must exclude docs
+ * not present in "response" to avoid reintroducing docs that were eliminated while deduplication.
+ */
@Override
protected void mergeIds(ResponseBuilder rb, ShardRequest sreq) {
SortSpec ss = rb.getSortSpec();
@@ -296,6 +318,10 @@ public class CombinedQueryComponent extends QueryComponent implements SolrCoreAw
long approximateTotalHits = 0;
Map<String, List<ShardDoc>> shardDocMap = new HashMap<>();
String[] queriesToCombineKeys = rb.req.getParams().getParams(CombinerParams.COMBINER_QUERY);
+ // Build per-shard set of doc IDs from the shard's combined (deduplicated) response.
+ // Used to filter per-query docs so that RRF doesn't reintroduce docs
+ // excluded by collapse at the shard level.
+ Map<String, Set<Object>> combinedDocIdsPerShard = HashMap.newHashMap(sreq.responses.size());
// TODO: to be parallelized outer loop
for (int queryIndex = 0; queryIndex < queriesToCombineKeys.length; queryIndex++) {
int failedShardCount = 0;
@@ -377,9 +403,15 @@ public class CombinedQueryComponent extends QueryComponent implements SolrCoreAw
: new NamedList<>();
// go through every doc in this response, construct a ShardDoc, and
// put it in the uniqueDoc to dedup
+ Set<Object> thisShardCombinedIds =
+ combinedDocIdsPerShard.computeIfAbsent(
+ srsp.getShard(), shard -> extractIdsFromCombinedResponse(rb, srsp, uniqueKeyField));
for (int i = 0; i < docs.size(); i++) {
SolrDocument doc = docs.get(i);
Object id = doc.getFieldValue(uniqueKeyField.getName());
+ if (!thisShardCombinedIds.contains(id)) {
+ continue;
+ }
ShardDoc shardDoc = new ShardDoc();
shardDoc.id = id;
shardDoc.orderInShard = i;
@@ -610,4 +642,19 @@ public class CombinedQueryComponent extends QueryComponent implements SolrCoreAw
for (int i = 0; i < resultSize; i++) responseDocs.add(null);
return resultIds;
}
+
+ /**
+ * Extracts the set of doc IDs from the shard's combined response (produced by simpleCombine).
+ * Returns an empty set if the combined response is not available.
+ */
+ private static Set<Object> extractIdsFromCombinedResponse(
+ ResponseBuilder rb, ShardResponse srsp, SchemaField uniqueKeyField) {
+ Object response = SolrResponseUtil.getSubsectionFromShardResponse(rb, srsp, "response", false);
+ if (response instanceof SolrDocumentList docList) {
+ return docList.stream()
+ .map(doc -> doc.getFieldValue(uniqueKeyField.getName()))
+ .collect(Collectors.toSet());
+ }
+ return Set.of();
+ }
}
diff --git a/solr/core/src/java/org/apache/solr/handler/component/combine/QueryAndResponseCombiner.java b/solr/core/src/java/org/apache/solr/handler/component/combine/QueryAndResponseCombiner.java
index c7d357fb12b..0c023231939 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/combine/QueryAndResponseCombiner.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/combine/QueryAndResponseCombiner.java
@@ -16,19 +16,33 @@
*/
package org.apache.solr.handler.component.combine;
+import java.io.IOException;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.internal.hppc.IntDoubleHashMap;
+import org.apache.lucene.queries.function.FunctionScoreQuery;
+import org.apache.lucene.search.DoubleValues;
+import org.apache.lucene.search.DoubleValuesSource;
import org.apache.lucene.search.Explanation;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
import org.apache.lucene.search.TotalHits;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.CollectionUtil;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.handler.component.ShardDoc;
import org.apache.solr.search.DocIterator;
import org.apache.solr.search.DocSet;
import org.apache.solr.search.DocSlice;
+import org.apache.solr.search.QueryCommand;
import org.apache.solr.search.QueryResult;
+import org.apache.solr.search.SolrIndexSearcher;
+import org.apache.solr.search.SortedIntDocSet;
import org.apache.solr.util.plugin.NamedListInitializedPlugin;
/**
@@ -49,12 +63,19 @@ public abstract class QueryAndResponseCombiner implements NamedListInitializedPl
Map<String, List<ShardDoc>> queriesDocMap, SolrParams solrParams);
/**
- * Simple combine query result list as a union.
+ * Combine query result list as a union, optionally deduplicating by a collapse field. When a
+ * collapse filter is provided, only one document per unique field value is kept (based on the
+ * collapse sort/score selection). This ensures that collapse semantics are preserved across
+ * combined queries.
*
* @param queryResults the query results to be combined
+ * @param collapseFilters the collapse post filters, or empty if no collapse dedup is needed
+ * @param searcher the searcher to read field values from, required when collapseFilters is
+ * non-empty
* @return the combined query result
*/
- public static QueryResult simpleCombine(List<QueryResult> queryResults) {
+ public static QueryResult simpleCombine(
+ List<QueryResult> queryResults, List<Query> collapseFilters, SolrIndexSearcher searcher) {
QueryResult combinedQueryResults = new QueryResult();
DocSet combinedDocSet = null;
Map<Integer, Float> uniqueDocIds = new HashMap<>();
@@ -71,6 +92,19 @@ public abstract class QueryAndResponseCombiner implements NamedListInitializedPl
combinedDocSet = combinedDocSet.union(queryResult.getDocSet());
}
}
+
+ // If collapse fields are specified, deduplicate by field value across combined queries.
+ // Each sub-query already collapsed individually, but different sub-queries may have
+ // selected different group heads for the same field value.
+ int removedByCollapse = 0;
+ if (CollectionUtil.isNotEmpty(collapseFilters) && searcher != null && queryResults.size() > 1) {
+ int preCollapseSize = uniqueDocIds.size();
+ combinedDocSet =
+ removeCollapsedDuplicatesViaSearcher(
+ collapseFilters, searcher, uniqueDocIds, combinedDocSet);
+ removedByCollapse = preCollapseSize - uniqueDocIds.size();
+ }
+
int combinedResultsLength = uniqueDocIds.size();
int[] combinedResultsDocIds = new int[combinedResultsLength];
float[] combinedResultScores = new float[combinedResultsLength];
@@ -87,7 +121,7 @@ public abstract class QueryAndResponseCombiner implements NamedListInitializedPl
combinedResultsLength,
combinedResultsDocIds,
combinedResultScores,
- Math.max(combinedResultsLength, totalMatches),
+ Math.max(combinedResultsLength, totalMatches - removedByCollapse),
combinedResultScores.length > 0 ? combinedResultScores[0] : 0,
TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO);
combinedQueryResults.setDocList(combinedResultSlice);
@@ -95,6 +129,56 @@ public abstract class QueryAndResponseCombiner implements NamedListInitializedPl
return combinedQueryResults;
}
+ /**
+ * Removes collapsed duplicates across combined sub-queries. Ensures that only one document per
+ * collapse field value retained across the merged results. Entries removed by collapsing are also
+ * removed from {@code uniqueDocIds} (mutated in place).
+ *
+ * @return the collapsed combined DocSet, or null if combinedDocSet was null
+ */
+ private static DocSet removeCollapsedDuplicatesViaSearcher(
+ List<Query> collapseFilters,
+ SolrIndexSearcher searcher,
+ Map<Integer, Float> uniqueDocIds,
+ DocSet combinedDocSet) {
+ IntDoubleHashMap scoreMap = new IntDoubleHashMap(uniqueDocIds.size());
+ uniqueDocIds.forEach((doc, score) -> scoreMap.put(doc, score.doubleValue()));
+ Query baseQuery;
+ boolean needDocSet;
+ if (combinedDocSet != null) {
+ baseQuery = combinedDocSet.makeQuery();
+ needDocSet = true;
+ } else {
+ int[] queryDocIds =
+ uniqueDocIds.keySet().stream().mapToInt(Integer::intValue).sorted().toArray();
+ baseQuery = new SortedIntDocSet(queryDocIds).makeQuery();
+ needDocSet = false;
+ }
+ Query scoredQuery =
+ FunctionScoreQuery.boostByValue(baseQuery, new PrecomputedScoreValuesSource(scoreMap));
+
+ try {
+ QueryCommand cmd =
+ new QueryCommand()
+ .setQuery(scoredQuery)
+ .setFilterList(collapseFilters)
+ .setLen(uniqueDocIds.size())
+ .setNeedDocSet(needDocSet);
+ QueryResult result = searcher.search(cmd);
+
+ Set<Integer> retainedDocIds = HashSet.newHashSet(result.getDocList().size());
+ DocIterator iter = result.getDocList().iterator();
+ while (iter.hasNext()) {
+ retainedDocIds.add(iter.nextDoc());
+ }
+
+ uniqueDocIds.keySet().retainAll(retainedDocIds);
+ return needDocSet ? result.getDocSet() : null;
+ } catch (IOException e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
+ }
+ }
+
/**
* Retrieves a list of explanations for the given queries and results.
*
@@ -127,4 +211,67 @@ public abstract class QueryAndResponseCombiner implements NamedListInitializedPl
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST, "Unknown Combining algorithm: " + algorithm);
}
+
+ /**
+ * A {@link DoubleValuesSource} backed by a global doc ID to score map. Returns pre-computed
+ * scores for specific document IDs.
+ */
+ private static class PrecomputedScoreValuesSource extends DoubleValuesSource {
+
+ private final IntDoubleHashMap scoreByDoc;
+
+ PrecomputedScoreValuesSource(IntDoubleHashMap scoreByDoc) {
+ this.scoreByDoc = scoreByDoc;
+ }
+
+ @Override
+ public DoubleValues getValues(LeafReaderContext ctx, DoubleValues existing) {
+ int base = ctx.docBase;
+ return new DoubleValues() {
+ private double currentScore;
+
+ @Override
+ public double doubleValue() {
+ return currentScore;
+ }
+
+ @Override
+ public boolean advanceExact(int doc) {
+ int globalDoc = base + doc;
+ currentScore = scoreByDoc.get(globalDoc);
+ return true;
+ }
+ };
+ }
+
+ @Override
+ public boolean needsScores() {
+ return false;
+ }
+
+ @Override
+ public DoubleValuesSource rewrite(IndexSearcher searcher) {
+ return this;
+ }
+
+ @Override
+ public boolean isCacheable(LeafReaderContext ctx) {
+ return false;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ return o instanceof PrecomputedScoreValuesSource other && scoreByDoc.equals(other.scoreByDoc);
+ }
+
+ @Override
+ public int hashCode() {
+ return scoreByDoc.hashCode();
+ }
+
+ @Override
+ public String toString() {
+ return "PrecomputedScoreValuesSource(docs=" + scoreByDoc.size() + ")";
+ }
+ }
}
diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-vector-catchall.xml b/solr/core/src/test-files/solr/collection1/conf/schema-vector-catchall.xml
index 37bb2b98557..7716a8ee0f5 100644
--- a/solr/core/src/test-files/solr/collection1/conf/schema-vector-catchall.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/schema-vector-catchall.xml
@@ -504,7 +504,7 @@
</analyzer>
</fieldType>
<copyField source="*_commongrams" dest="*_commongrams_stop" />
-
+
<fieldType name="binary" class="solr.BinaryField" />
<fieldType name="collation" class="solr.CollationField" language="en" />
<fieldType name="icuCollation" class="solr.ICUCollationField" locale="en" />
@@ -691,6 +691,7 @@
<dynamicField name="*_s" type="string" indexed="true" stored="true"/>
+ <dynamicField name="*_sdv" type="string" indexed="true" stored="true" docValues="true" multiValued="false"/>
<dynamicField name="*_s1" type="string" indexed="true" stored="true" multiValued="false"/>
<dynamicField name="*_s1_ns" type="string" indexed="true" stored="false" multiValued="false"/>
<dynamicField name="*_l" type="long" indexed="true" stored="true"/>
@@ -857,14 +858,9 @@
<copyField source="lowerfilt1" dest="lowerfilt1and2"/>
<copyField source="lowerfilt" dest="lowerfilt1and2"/>
- <copyField source="*" dest="text"/>
- <copyField source="id" dest="range_facet_l"/>
- <copyField source="id" dest="id_i1"/>
<copyField source="range_facet_f" dest="range_facet_d"/>
<copyField source="range_facet_f1" dest="range_facet_f1_dv"/>
- <copyField source="id" dest="range_facet_l_dv"/>
- <copyField source="id" dest="range_facet_i_dv"/>
<copyField source="range_facet_f" dest="range_facet_f_dv"/>
<copyField source="range_facet_f" dest="range_facet_d_dv"/>
<copyField source="bday" dest="range_facet_dt_dv"/>
diff --git a/solr/core/src/test/org/apache/solr/handler/component/CombinedQuerySolrCloudTest.java b/solr/core/src/test/org/apache/solr/handler/component/CombinedQuerySolrCloudTest.java
index 4fbbb7f17e5..448ced51254 100644
--- a/solr/core/src/test/org/apache/solr/handler/component/CombinedQuerySolrCloudTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/component/CombinedQuerySolrCloudTest.java
@@ -45,6 +45,8 @@ public class CombinedQuerySolrCloudTest extends AbstractFullDistribZkTestBase {
super();
sliceCount = 2;
fixShardCount(2);
+ schemaString = "schema-vector-catchall.xml";
+ configString = "solrconfig-combined-query.xml";
}
@Override
@@ -58,6 +60,28 @@ public class CombinedQuerySolrCloudTest extends AbstractFullDistribZkTestBase {
}
private synchronized void prepareIndexDocs() throws Exception {
+ List<SolrInputDocument> docs = getSolrDocuments();
+ del("*:*");
+ for (SolrInputDocument doc : docs) {
+ indexDoc(doc);
+ }
+ commit();
+ }
+
+ /** Index all docs to the same shard for co-location (limitation for collapse). */
+ private synchronized void prepareIndexDocsColocated() throws Exception {
+ del("*:*");
+ List<SolrInputDocument> docs = getSolrDocuments();
+ for (SolrInputDocument doc : docs) {
+ doc.setField("id", doc.getFieldValue("mod3_idv") + "!" + doc.getField("id").getValue());
+ }
+ for (SolrInputDocument doc : docs) {
+ indexDoc(doc);
+ }
+ commit();
+ }
+
+ private static synchronized List<SolrInputDocument> getSolrDocuments() {
List<SolrInputDocument> docs = new ArrayList<>();
for (int i = 1; i <= NUM_DOCS; i++) {
SolrInputDocument doc = new SolrInputDocument();
@@ -87,11 +111,7 @@ public class CombinedQuerySolrCloudTest extends AbstractFullDistribZkTestBase {
docs.get(8).addField(vectorField, Arrays.asList(200f, 50f, 100f, 25f));
// cosine distance vector1= 0.997
docs.get(9).addField(vectorField, Arrays.asList(1.8f, 2.5f, 3.7f, 4.9f));
- del("*:*");
- for (SolrInputDocument doc : docs) {
- indexDoc(doc);
- }
- commit();
+ return docs;
}
@Test
@@ -249,6 +269,69 @@ public class CombinedQuerySolrCloudTest extends AbstractFullDistribZkTestBase {
rsp.getHighlighting().get("5").get("title").getFirst());
}
+ /**
+ * Tests the combined query feature with faceting, highlighting and collapse.
+ *
+ * @throws Exception if any unexpected error occurs during the test execution.
+ */
+ @Test
+ public void testQueriesWithFacetAndHighlightsCollapse() throws Exception {
+ // Re-index all docs to the first shard for co-location (required for collapse).
+ prepareIndexDocsColocated();
+ String jsonQuery =
+ """
+ {
+ "queries": {
+ "lexical1": {
+ "lucene": {
+ "query": "id:(2!2^2 OR 0!3^1 OR 0!6^2 OR 2!5^1)"
+ }
+ },
+ "lexical2": {
+ "lucene": {
+ "query": "id:(2!8^1 OR 2!5^2 OR 1!7^3 OR 1!10^2)"
+ }
+ }
+ },
+ "limit": 1,
+ "fields": [
+ "id",
+ "score",
+ "title"
+ ],
+ "params": {
+ "combiner": true,
+ "facet": true,
+ "facet.field": "id",
+ "fq": [
+ "{!collapse field=mod3_idv sort='id asc'}"
+ ],
+ "expand": true,
+ "expand.q": "*:*",
+ "combiner.query": [
+ "lexical1",
+ "lexical2"
+ ],
+ "hl": true,
+ "hl.fl": "title",
+ "hl.q": "test doc"
+ }
+ }""";
+ handle.put("expanded", UNORDERED);
+ QueryResponse rsp = query(CommonParams.JSON, jsonQuery, CommonParams.QT, "/search");
+ assertEquals(1, rsp.getResults().size());
+ assertFieldValues(rsp.getResults(), id, "2!2");
+ assertEquals("id", rsp.getFacetFields().getFirst().getName());
+ assertEquals(
+ "[0!3 (1), 1!10 (1), 2!2 (1), 0!6 (0), 0!9 (0), 1!1 (0), 1!4 (0), 1!7 (0), 2!5 (0), 2!8 (0)]",
+ rsp.getFacetFields().getFirst().getValues().toString());
+ assertEquals(1, rsp.getHighlighting().size());
+ assertEquals(
+ "title <em>test</em> for <em>doc</em> 2",
+ rsp.getHighlighting().get("2!2").get("title").getFirst());
+ assertEquals(1, rsp.getExpandedResults().size());
+ }
+
/** To test that we can force distrib */
public static class ForcedDistribSearchHandler extends SearchHandler {
@Override
diff --git a/solr/core/src/test/org/apache/solr/handler/component/DistributedCombinedQueryComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/DistributedCombinedQueryComponentTest.java
index 46441382c30..bb9105cad30 100644
--- a/solr/core/src/test/org/apache/solr/handler/component/DistributedCombinedQueryComponentTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/component/DistributedCombinedQueryComponentTest.java
@@ -290,4 +290,143 @@ public class DistributedCombinedQueryComponentTest extends BaseDistributedSearch
QueryResponse rsp = query("qt", "/forcedDistribTest", "q", "*:*", "rows", "0");
// ForcedDistribSearchHandler would trigger a failure if this didn't work
}
+
+ /**
+ * Reproduces duplicate documents in combined query results when using collapse. Two different
+ * sub-queries may select different group heads for the same collapse field value, and
+ * simpleCombine() merges by Lucene doc ID — not by collapse field value — so both survive.
+ *
+ * <p>Data setup (mod3_idv is the collapse field, values cycle 1,2,0,1,2,0,...):
+ *
+ * <pre>
+ * id=1 mod3_idv=1 text="alpha bravo" title="alpha bravo"
+ * id=2 mod3_idv=2 text="alpha charlie" title="alpha charlie"
+ * id=3 mod3_idv=0 text="bravo delta" title="bravo delta"
+ * id=4 mod3_idv=1 text="charlie delta" title="charlie delta"
+ * id=5 mod3_idv=2 text="alpha delta" title="alpha delta"
+ * id=6 mod3_idv=0 text="bravo charlie" title="bravo charlie"
+ * </pre>
+ *
+ * <p>Query 1: "alpha bravo" → matches docs 1,2,3,5,6 with varying scores. After collapse on
+ * mod3_idv, picks one group head per value (0,1,2).
+ *
+ * <p>Query 2: "charlie delta" → matches docs 2,3,4,5,6 with varying scores. After collapse on
+ * mod3_idv, picks one group head per value (0,1,2).
+ *
+ * <p>Because the queries score differently, they may pick DIFFERENT group heads for the same
+ * mod3_idv value. After simpleCombine() merges by doc ID, both heads appear → duplicates on the
+ * collapse field.
+ */
+ @Test
+ public void testCollapseWithCombinedQueryProducesDuplicates() throws Exception {
+ del("*:*");
+
+ // Index 6 docs where mod3_idv groups docs: {3,6}→0, {1,4}→1, {2,5}→2
+ // All docs indexed to the same shard (first client) to ensure co-location,
+ // which is a requirement for collapse in SolrCloud.
+ List<SolrInputDocument> docs = new ArrayList<>();
+ String[][] data = {
+ {"1", "alpha bravo", "alpha bravo"},
+ {"2", "alpha charlie", "alpha charlie"},
+ {"3", "bravo delta", "bravo delta"},
+ {"4", "charlie delta", "charlie delta"},
+ {"5", "alpha delta", "alpha delta"},
+ {"6", "bravo charlie", "bravo charlie"},
+ };
+ for (String[] row : data) {
+ SolrInputDocument doc = new SolrInputDocument();
+ doc.addField("id", row[0]);
+ doc.addField("text", row[1]);
+ doc.addField("title", row[2]);
+ int idVal = Integer.parseInt(row[0]);
+ doc.addField("mod3_idv", idVal % 3);
+ doc.addField("mod3_sdv", String.valueOf(idVal % 3));
+ docs.add(doc);
+ }
+ // Index all docs to the first shard (co-location for collapse).
+ // indexDoc(client, doc) also indexes to controlClient automatically.
+ for (SolrInputDocument doc : docs) {
+ indexDoc(clients.getFirst(), doc);
+ }
+ commit();
+
+ // Two queries that score docs differently, both collapsed on mod3_idv.
+ // Query 1 "alpha bravo" favours doc 1 (mod3_idv=1) and doc 3 (mod3_idv=0).
+ // Query 2 "charlie delta" favours doc 4 (mod3_idv=1) and doc 3 or 6 (mod3_idv=0).
+ // After collapse, each query returns 3 docs (one per mod3_idv value 0,1,2).
+ // But the group heads for mod3_idv=1 differ: query1 picks id=1, query2 picks id=4.
+ // simpleCombine() merges by doc ID, so both id=1 and id=4 survive → duplicate on mod3_idv=1.
+ // q1 -> 1,2,5,3,6 q2 -> 2,4,6,3,5
+ // q1 -> 1,2,2,0,0 q2 -> 2,1,0,0,2
+ // q1 -> 1 for sure, 2,3 => 4 for sure, 3 and 2
+ String jsonQuery =
+ """
+ {
+ "queries": {
+ "q1": {
+ "lucene": {
+ "query": "text:alpha OR text:bravo"
+ }
+ },
+ "q2": {
+ "lucene": {
+ "query": "text:charlie OR text:delta"
+ }
+ }
+ },
+ "limit": 10,
+ "fields": [
+ "id",
+ "score",
+ "mod3_idv"
+ ],
+ "params": {
+ "combiner": true,
+ "combiner.query": [
+ "q1",
+ "q2"
+ ],
+ "fq": [
+ "{!collapse field=mod3_idv sort='mod3_sdv asc, id desc'}"
+ ],
+ "facet": true,
+ "facet.field": "id",
+ "hl": true,
+ "hl.fl": "title",
+ "hl.q": "alpha delta",
+ "expand": true,
+ "expand.q": "text:alpha OR text:bravo OR text:charlie OR text:delta"
+ }
+ }""";
+
+ handle.put("expanded", UNORDERED);
+ QueryResponse rsp = query(CommonParams.JSON, jsonQuery, CommonParams.QT, "/search");
+
+ // Collect mod3_idv values from the result to check for duplicates
+ List<Integer> collapseValues = new ArrayList<>();
+ for (int i = 0; i < rsp.getResults().size(); i++) {
+ Object val = rsp.getResults().get(i).getFieldValue("mod3_idv");
+ collapseValues.add(((Number) val).intValue());
+ }
+
+ // Without the fix: more than 3 results because different group heads survive for the same
+ // mod3_idv value. With the fix: exactly 3 results (one per mod3_idv value 0, 1, 2).
+ long uniqueCollapseValues = collapseValues.stream().distinct().count();
+ assertEquals(
+ "Expected no duplicate collapse field values in combined results, "
+ + "but got collapseValues="
+ + collapseValues,
+ uniqueCollapseValues,
+ collapseValues.size());
+ assertEquals("Expected exactly 3 groups (mod3_idv values 0, 1, 2)", 3, collapseValues.size());
+ assertEquals("id", rsp.getFacetFields().getFirst().getName());
+ assertEquals(
+ "[4 (1), 5 (1), 6 (1), 1 (0), 2 (0), 3 (0)]",
+ rsp.getFacetFields().getFirst().getValues().toString());
+ assertEquals(3, rsp.getHighlighting().size());
+ assertEquals("charlie <em>delta</em>", rsp.getHighlighting().get("4").get("title").getFirst());
+ assertEquals(
+ "<em>alpha</em> <em>delta</em>", rsp.getHighlighting().get("5").get("title").getFirst());
+ assertEquals(3, rsp.getExpandedResults().size());
+ }
}
diff --git a/solr/core/src/test/org/apache/solr/handler/component/combine/QueryAndResponseCombinerTest.java b/solr/core/src/test/org/apache/solr/handler/component/combine/QueryAndResponseCombinerTest.java
index aece94c5ea3..a03aa7219e5 100644
--- a/solr/core/src/test/org/apache/solr/handler/component/combine/QueryAndResponseCombinerTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/component/combine/QueryAndResponseCombinerTest.java
@@ -17,15 +17,29 @@
package org.apache.solr.handler.component.combine;
import java.util.List;
+import org.apache.lucene.search.Query;
import org.apache.lucene.search.TotalHits;
import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.request.SolrRequestInfo;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.search.CollapsingQParserPlugin.CollapsingPostFilter;
import org.apache.solr.search.DocSlice;
+import org.apache.solr.search.QParser;
import org.apache.solr.search.QueryResult;
+import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.search.SortedIntDocSet;
+import org.apache.solr.util.RefCounted;
+import org.junit.BeforeClass;
import org.junit.Test;
public class QueryAndResponseCombinerTest extends SolrTestCaseJ4 {
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ initCore("solrconfig.xml", "schema11.xml");
+ }
+
public static List<QueryResult> getQueryResults() {
QueryResult r1 = new QueryResult();
r1.setDocList(
@@ -52,10 +66,100 @@ public class QueryAndResponseCombinerTest extends SolrTestCaseJ4 {
return List.of(r1, r2);
}
+ /** Parses a collapse fq string and returns the CollapsingPostFilter. */
+ private static CollapsingPostFilter parseCollapseFilter(String collapseFq) throws Exception {
+ SolrQueryRequest solrReq = req();
+ SolrQueryResponse rsp = new SolrQueryResponse();
+ SolrRequestInfo.setRequestInfo(new SolrRequestInfo(solrReq, rsp));
+ try {
+ Query q = QParser.getParser(collapseFq, solrReq).getQuery();
+ assertTrue(
+ "Expected CollapsingPostFilter but got " + q.getClass(),
+ q instanceof CollapsingPostFilter);
+ return (CollapsingPostFilter) q;
+ } finally {
+ SolrRequestInfo.clearRequestInfo();
+ solrReq.close();
+ }
+ }
+
+ /** Tests that simpleCombine without using collapse. */
@Test
public void simpleCombine() {
- QueryResult queryResult = QueryAndResponseCombiner.simpleCombine(getQueryResults());
+ QueryResult queryResult = QueryAndResponseCombiner.simpleCombine(getQueryResults(), null, null);
assertEquals(3, queryResult.getDocList().size());
assertEquals(4, queryResult.getDocSet().size());
}
+
+ /**
+ * Tests that simpleCombine deduplicates by collapse field value when a collapse field is
+ * provided. Uses the same query results from getQueryResults() but indexes docs so that Lucene
+ * doc IDs 0 and 1 share the same collapse field value — only the higher-scoring doc should
+ * survive.
+ */
+ @Test
+ public void simpleCombineWithCollapseField() throws Exception {
+ // getQueryResults() returns:
+ // r1: docs [1, 2] scores [0.67, 0.62]
+ // r2: docs [0] scores [0.87]
+ assertU(delQ("*:*"));
+ assertU(adoc("id", "0", "group_ti_dv", "1"));
+ assertU(adoc("id", "1", "group_ti_dv", "1"));
+ assertU(adoc("id", "2", "group_ti_dv", "2"));
+ assertU(commit());
+
+ CollapsingPostFilter collapseFilter = parseCollapseFilter("{!collapse field=group_ti_dv}");
+
+ RefCounted<SolrIndexSearcher> searcherRef = h.getCore().getSearcher();
+ try {
+ SolrIndexSearcher searcher = searcherRef.get();
+
+ QueryResult combined =
+ QueryAndResponseCombiner.simpleCombine(
+ getQueryResults(), List.of(collapseFilter), searcher);
+
+ // group=1: doc 0 (0.87) beats doc 1 (0.67) → keep doc 0
+ // group=2: doc 2 (0.62) → kept
+ assertEquals(
+ "Expected 2 docs after collapse dedup (one per group value)",
+ 2,
+ combined.getDocList().size());
+ assertEquals(0, combined.getDocList().iterator().nextDoc());
+ } finally {
+ searcherRef.decref();
+ }
+ }
+
+ /**
+ * Tests that docs with missing collapse field values are kept (null policy pass-through) and not
+ * considered duplicates.
+ */
+ @Test
+ public void simpleCombineWithCollapseFieldNullPolicy() throws Exception {
+ assertU(delQ("*:*"));
+ assertU(adoc("id", "0", "group_ti_dv", "1"));
+ assertU(adoc("id", "1", "group_ti_dv", "1"));
+ assertU(adoc("id", "2")); // no collapse field
+ assertU(commit());
+
+ CollapsingPostFilter collapseFilter =
+ parseCollapseFilter("{!collapse field=group_ti_dv nullPolicy=collapse sort='score asc'}");
+
+ RefCounted<SolrIndexSearcher> searcherRef = h.getCore().getSearcher();
+ try {
+ SolrIndexSearcher searcher = searcherRef.get();
+
+ QueryResult combined =
+ QueryAndResponseCombiner.simpleCombine(
+ getQueryResults(), List.of(collapseFilter), searcher);
+
+ // group=1: doc 0 (0.87) > doc 1 (0.67) → keep doc 1
+ // doc 2 has no group value → kept (null policy)
+ assertEquals(
+ "Expected 2 docs: one group head + one null-field doc", 2, combined.getDocList().size());
+ assertEquals(1, combined.getDocList().iterator().nextDoc());
+ } finally {
+ searcherRef.decref();
+ }
+ }
}
diff --git a/solr/core/src/test/org/apache/solr/handler/component/combine/ReciprocalRankFusionTest.java b/solr/core/src/test/org/apache/solr/handler/component/combine/ReciprocalRankFusionTest.java
index 27ab8c5a312..ac416c49aab 100644
--- a/solr/core/src/test/org/apache/solr/handler/component/combine/ReciprocalRankFusionTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/component/combine/ReciprocalRankFusionTest.java
@@ -55,7 +55,7 @@ public class ReciprocalRankFusionTest extends SolrTestCaseJ4 {
@Test
public void testSimpleCombine() {
List<QueryResult> rankedList = QueryAndResponseCombinerTest.getQueryResults();
- QueryResult result = QueryAndResponseCombiner.simpleCombine(rankedList);
+ QueryResult result = QueryAndResponseCombiner.simpleCombine(rankedList, null, null);
assertEquals(3, result.getDocList().size());
assertEquals(4, result.getDocSet().size());
}