You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by yo...@apache.org on 2010/10/31 15:06:12 UTC
svn commit: r1029349 - in /lucene/dev/trunk/solr/src:
common/org/apache/solr/common/params/
java/org/apache/solr/handler/component/ java/org/apache/solr/search/
test/org/apache/solr/
Author: yonik
Date: Sun Oct 31 14:06:12 2010
New Revision: 1029349
URL: http://svn.apache.org/viewvc?rev=1029349&view=rev
Log:
SOLR-2207: add paging to result grouping
Modified:
lucene/dev/trunk/solr/src/common/org/apache/solr/common/params/GroupParams.java
lucene/dev/trunk/solr/src/java/org/apache/solr/handler/component/QueryComponent.java
lucene/dev/trunk/solr/src/java/org/apache/solr/search/Grouping.java
lucene/dev/trunk/solr/src/java/org/apache/solr/search/SolrIndexSearcher.java
lucene/dev/trunk/solr/src/test/org/apache/solr/TestGroupingSearch.java
Modified: lucene/dev/trunk/solr/src/common/org/apache/solr/common/params/GroupParams.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/common/org/apache/solr/common/params/GroupParams.java?rev=1029349&r1=1029348&r2=1029349&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/common/org/apache/solr/common/params/GroupParams.java (original)
+++ lucene/dev/trunk/solr/src/common/org/apache/solr/common/params/GroupParams.java Sun Oct 31 14:06:12 2010
@@ -18,7 +18,7 @@
package org.apache.solr.common.params;
/**
- * Facet parameters
+ * Group parameters
*/
public interface GroupParams {
public static final String GROUP = "group";
@@ -30,5 +30,7 @@ public interface GroupParams {
/** the limit for the number of documents in each group */
public static final String GROUP_LIMIT = GROUP + ".limit";
+ /** the offset for the doclist of each group */
+ public static final String GROUP_OFFSET = GROUP + ".offset";
}
Modified: lucene/dev/trunk/solr/src/java/org/apache/solr/handler/component/QueryComponent.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/java/org/apache/solr/handler/component/QueryComponent.java?rev=1029349&r1=1029348&r2=1029349&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/java/org/apache/solr/handler/component/QueryComponent.java (original)
+++ lucene/dev/trunk/solr/src/java/org/apache/solr/handler/component/QueryComponent.java Sun Oct 31 14:06:12 2010
@@ -311,6 +311,7 @@ public class QueryComponent extends Sear
Sort groupSort = groupSortStr != null ? QueryParsing.parseSort(groupSortStr, req) : null;
int limitDefault = cmd.getLen(); // this is normally from "rows"
+ int groupOffsetDefault = params.getInt(GroupParams.GROUP_OFFSET, 0);
int docsPerGroupDefault = params.getInt(GroupParams.GROUP_LIMIT, 1);
// temporary: implement all group-by-field as group-by-func
@@ -340,6 +341,8 @@ public class QueryComponent extends Sear
gc.key = groupByStr;
gc.numGroups = limitDefault;
gc.docsPerGroup = docsPerGroupDefault;
+ gc.groupOffset = groupOffsetDefault;
+ gc.offset = cmd.getOffset();
cmd.groupCommands.add(gc);
}
@@ -355,6 +358,7 @@ public class QueryComponent extends Sear
gc.key = groupByStr;
gc.numGroups = limitDefault;
gc.docsPerGroup = docsPerGroupDefault;
+ gc.groupOffset = groupOffsetDefault;
cmd.groupCommands.add(gc);
}
Modified: lucene/dev/trunk/solr/src/java/org/apache/solr/search/Grouping.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/java/org/apache/solr/search/Grouping.java?rev=1029349&r1=1029348&r2=1029349&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/java/org/apache/solr/search/Grouping.java (original)
+++ lucene/dev/trunk/solr/src/java/org/apache/solr/search/Grouping.java Sun Oct 31 14:06:12 2010
@@ -31,7 +31,9 @@ public class Grouping {
public String key; // the name to use for this group in the response
public Sort groupSort; // the sort of the documents *within* a single group.
public int docsPerGroup; // how many docs in each group - from "group.limit" param, default=1
+ public int groupOffset; // the offset within each group (for paging within each group)
public int numGroups; // how many groups - defaults to the "rows" parameter
+ public int offset; // offset into the list of groups
}
public static class CommandQuery extends Command {
@@ -511,11 +513,18 @@ class Phase2GroupCollector extends Colle
int docBase;
// TODO: may want to decouple from the phase1 collector
- public Phase2GroupCollector(TopGroupCollector topGroups, ValueSource groupByVS, Map vsContext, Sort sort, int docsPerGroup, boolean getScores) throws IOException {
+ public Phase2GroupCollector(TopGroupCollector topGroups, ValueSource groupByVS, Map vsContext, Sort sort, int docsPerGroup, boolean getScores, int offset) throws IOException {
boolean getSortFields = false;
+ if (topGroups.orderedGroups == null)
+ topGroups.buildSet();
+
groupMap = new HashMap<MutableValue, SearchGroupDocs>(topGroups.groupMap.size());
- for (SearchGroup group : topGroups.groupMap.values()) {
+ for (SearchGroup group : topGroups.orderedGroups) {
+ if (offset > 0) {
+ offset--;
+ continue;
+ }
SearchGroupDocs groupDocs = new SearchGroupDocs();
groupDocs.groupValue = group.groupValue;
groupDocs.collector = TopFieldCollector.create(sort, docsPerGroup, getSortFields, getScores, getScores, true);
Modified: lucene/dev/trunk/solr/src/java/org/apache/solr/search/SolrIndexSearcher.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/java/org/apache/solr/search/SolrIndexSearcher.java?rev=1029349&r1=1029348&r2=1029349&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/java/org/apache/solr/search/SolrIndexSearcher.java (original)
+++ lucene/dev/trunk/solr/src/java/org/apache/solr/search/SolrIndexSearcher.java Sun Oct 31 14:06:12 2010
@@ -997,8 +997,7 @@ public class SolrIndexSearcher extends I
DocSet filter = cmd.getFilter()!=null ? cmd.getFilter() : getDocSet(cmd.getFilterList());
- int last = cmd.getOffset() + cmd.getLen();
- if (last < 0 || last > maxDoc()) last=maxDoc();
+ int maxDoc = maxDoc();
boolean needScores = (cmd.getFlags() & GET_SCORES) != 0;
boolean getDocSet = (cmd.getFlags() & GET_DOCSET) != 0;
@@ -1018,10 +1017,14 @@ public class SolrIndexSearcher extends I
Map context = ValueSource.newContext();
gc.groupBy.createWeight(context, this);
TopGroupCollector collector;
+
+ int groupsToCollect = gc.numGroups<0 ? maxDoc : gc.offset + gc.numGroups;
+ if (groupsToCollect < 0 || groupsToCollect > maxDoc) groupsToCollect = maxDoc;
+
if (gc.groupSort != null && gc.groupSort != sort) {
- collector = new TopGroupSortCollector(gc.groupBy, context, sort, gc.groupSort, last);
+ collector = new TopGroupSortCollector(gc.groupBy, context, sort, gc.groupSort, groupsToCollect);
} else {
- collector = new TopGroupCollector(gc.groupBy, context, sort, last);
+ collector = new TopGroupCollector(gc.groupBy, context, sort, groupsToCollect);
}
collectors.add(collector);
@@ -1031,8 +1034,11 @@ public class SolrIndexSearcher extends I
}
if (groupCommand instanceof Grouping.CommandQuery) {
+ int docsToCollect = groupCommand.docsPerGroup<0 ? maxDoc : groupCommand.groupOffset + groupCommand.docsPerGroup;
+ if (docsToCollect < 0 || docsToCollect > maxDoc) docsToCollect = maxDoc;
+
DocSet groupFilt = getDocSet(((Grouping.CommandQuery)groupCommand).query);
- TopFieldCollector collector = TopFieldCollector.create(groupCommand.groupSort==null ? sort : groupCommand.groupSort, groupCommand.docsPerGroup, false, needScores, needScores, true);
+ TopFieldCollector collector = TopFieldCollector.create(groupCommand.groupSort==null ? sort : groupCommand.groupSort, docsToCollect, false, needScores, needScores, true);
collectors.add(new FilterCollector(groupFilt, collector));
}
}
@@ -1058,7 +1064,11 @@ public class SolrIndexSearcher extends I
if (groupCommand instanceof Grouping.CommandFunc) {
Grouping.CommandFunc gc = (Grouping.CommandFunc)groupCommand;
Sort collectorSort = gc.groupSort == null ? sort : gc.groupSort;
- Phase2GroupCollector collector = new Phase2GroupCollector((TopGroupCollector)gc.collector, gc.groupBy, gc.context, collectorSort, gc.docsPerGroup, needScores);
+
+ int docsToCollect = groupCommand.docsPerGroup<0 ? maxDoc : groupCommand.groupOffset + groupCommand.docsPerGroup;
+ if (docsToCollect < 0 || docsToCollect > maxDoc) docsToCollect = maxDoc;
+
+ Phase2GroupCollector collector = new Phase2GroupCollector((TopGroupCollector)gc.collector, gc.groupBy, gc.context, collectorSort, docsToCollect, needScores, groupCommand.offset);
phase2Collectors.add(collector);
numPhase2++;
} else if (groupCommand instanceof Grouping.CommandQuery) {
@@ -1090,7 +1100,11 @@ public class SolrIndexSearcher extends I
// TODO: refactor this
if (groupCommand instanceof Grouping.CommandQuery) {
- TopDocs topDocs = ((FilterCollector)gcollector).getTopFieldCollector().topDocs(0, groupCommand.docsPerGroup);
+
+ int docsToCollect = groupCommand.docsPerGroup<0 ? maxDoc : groupCommand.groupOffset + groupCommand.docsPerGroup;
+ if (docsToCollect < 0 || docsToCollect > maxDoc) docsToCollect = maxDoc;
+
+ TopDocs topDocs = ((FilterCollector)gcollector).getTopFieldCollector().topDocs(0, docsToCollect);
// TODO: refactor
@@ -1105,12 +1119,13 @@ public class SolrIndexSearcher extends I
float score = topDocs.getMaxScore();
maxScore = Math.max(maxScore, score);
- DocSlice docs = new DocSlice(0, ids.length, ids, scores, topDocs.totalHits, score);
+ DocSlice docs = new DocSlice(groupCommand.groupOffset, Math.max(0, ids.length - groupCommand.groupOffset), ids, scores, topDocs.totalHits, score);
groupResult.add("doclist", docs);
if (getDocList) {
- for (int id : ids)
- idSet.add(id);
+ DocIterator iter = docs.iterator();
+ while (iter.hasNext())
+ idSet.add(iter.nextDoc());
}
continue;
@@ -1125,7 +1140,12 @@ public class SolrIndexSearcher extends I
List groupList = new ArrayList();
groupResult.add("groups", groupList); // grouped={ key={ groups=[
+ int skipCount = groupCommand.offset;
for (SearchGroup group : collector.orderedGroups) {
+ if (skipCount > 0) {
+ skipCount--;
+ continue;
+ }
NamedList nl = new SimpleOrderedMap();
groupList.add(nl); // grouped={ key={ groups=[ {
@@ -1134,7 +1154,10 @@ public class SolrIndexSearcher extends I
SearchGroupDocs groupDocs = collector2.groupMap.get(group.groupValue);
// nl.add("matches", groupDocs.matches); // redundant with doclist.numFound from the doc list
- TopDocs topDocs = groupDocs.collector.topDocs(0, groupCommandFunc.docsPerGroup);
+ int docsToCollect = groupCommand.docsPerGroup<0 ? maxDoc : groupCommand.groupOffset + groupCommand.docsPerGroup;
+ if (docsToCollect < 0 || docsToCollect > maxDoc) docsToCollect = maxDoc;
+
+ TopDocs topDocs = groupDocs.collector.topDocs(0, docsToCollect);
//topDocs.totalHits
int ids[] = new int[topDocs.scoreDocs.length];
float[] scores = needScores ? new float[topDocs.scoreDocs.length] : null;
@@ -1146,14 +1169,14 @@ public class SolrIndexSearcher extends I
float score = topDocs.getMaxScore();
maxScore = Math.max(maxScore, score);
- DocSlice docs = new DocSlice(0, ids.length, ids, scores, topDocs.totalHits, score);
+ DocSlice docs = new DocSlice(groupCommand.groupOffset, Math.max(0, ids.length - groupCommand.groupOffset), ids, scores, topDocs.totalHits, score);
nl.add("doclist", docs);
if (getDocList) {
- for (int id : ids)
- idSet.add(id);
+ DocIterator iter = docs.iterator();
+ while (iter.hasNext())
+ idSet.add(iter.nextDoc());
}
-
/*** values from stage 1
DocSlice docs = new DocSlice(0, 1, new int[] {group.topDoc}, null, 1, 0);
nl.add("docs", docs);
Modified: lucene/dev/trunk/solr/src/test/org/apache/solr/TestGroupingSearch.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/test/org/apache/solr/TestGroupingSearch.java?rev=1029349&r1=1029348&r2=1029349&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/test/org/apache/solr/TestGroupingSearch.java (original)
+++ lucene/dev/trunk/solr/src/test/org/apache/solr/TestGroupingSearch.java Sun Oct 31 14:06:12 2010
@@ -201,6 +201,19 @@ public class TestGroupingSearch extends
"]}}"
);
+ // test offset into group list
+ assertJQ(req("fq",filt, "q","{!func}"+f2, "group","true", "group.field",f, "fl","id", "rows","1", "start","1")
+ ,"/grouped=={'"+f+"':{'matches':10,'groups':[" +
+ "{'groupValue':3,'doclist':{'numFound':2,'start':0,'docs':[{'id':'3'}]}}" +
+ "]}}"
+ );
+
+ // test big offset into group list
+ assertJQ(req("fq",filt, "q","{!func}"+f2, "group","true", "group.field",f, "fl","id", "rows","1", "start","100")
+ ,"/grouped=={'"+f+"':{'matches':10,'groups':[" +
+ "]}}"
+ );
+
// test increasing the docs per group returned
assertJQ(req("fq",filt, "q","{!func}"+f2, "group","true", "group.field",f, "fl","id", "rows","2", "group.limit","3")
,"/grouped=={'"+f+"':{'matches':10,'groups':[" +
@@ -209,6 +222,22 @@ public class TestGroupingSearch extends
"]}}"
);
+ // test offset into each group
+ assertJQ(req("fq",filt, "q","{!func}"+f2, "group","true", "group.field",f, "fl","id", "rows","2", "group.limit","3", "group.offset","1")
+ ,"/grouped=={'"+f+"':{'matches':10,'groups':[" +
+ "{'groupValue':1,'doclist':{'numFound':3,'start':1,'docs':[{'id':'10'},{'id':'5'}]}}," +
+ "{'groupValue':3,'doclist':{'numFound':2,'start':1,'docs':[{'id':'6'}]}}" +
+ "]}}"
+ );
+
+ // test big offset into each group
+ assertJQ(req("fq",filt, "q","{!func}"+f2, "group","true", "group.field",f, "fl","id", "rows","2", "group.limit","3", "group.offset","10")
+ ,"/grouped=={'"+f+"':{'matches':10,'groups':[" +
+ "{'groupValue':1,'doclist':{'numFound':3,'start':10,'docs':[]}}," +
+ "{'groupValue':3,'doclist':{'numFound':2,'start':10,'docs':[]}}" +
+ "]}}"
+ );
+
// test adding in scores
assertJQ(req("fq",filt, "q","{!func}"+f2, "group","true", "group.field",f, "fl","id,score", "rows","2", "group.limit","2", "indent","off")
,"/grouped/"+f+"/groups==" +
@@ -257,6 +286,18 @@ public class TestGroupingSearch extends
"'doclist':{'numFound':4,'start':0,'docs':[{'id':'3'},{'id':'4'},{'id':'2'}]}}}"
);
+ // group.query and offset
+ assertJQ(req("fq",filt, "q","{!func}"+f2, "group","true", "group.query","id:[2 TO 5]", "fl","id", "group.limit","3", "group.offset","2")
+ ,"/grouped=={'id:[2 TO 5]':{'matches':10," +
+ "'doclist':{'numFound':4,'start':2,'docs':[{'id':'2'},{'id':'5'}]}}}"
+ );
+
+ // group.query and big offset
+ assertJQ(req("fq",filt, "q","{!func}"+f2, "group","true", "group.query","id:[2 TO 5]", "fl","id", "group.limit","3", "group.offset","10")
+ ,"/grouped=={'id:[2 TO 5]':{'matches':10," +
+ "'doclist':{'numFound':4,'start':10,'docs':[]}}}"
+ );
+
// multiple at once
assertJQ(req("fq",filt, "q","{!func}"+f2, "group","true",
"group.query","id:[2 TO 5]",