You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mv...@apache.org on 2011/08/07 11:01:58 UTC
svn commit: r1154675 - in /lucene/dev/branches/branch_3x:
lucene/contrib/grouping/src/java/org/apache/lucene/search/grouping/ solr/
solr/core/src/java/org/apache/solr/handler/component/
solr/core/src/java/org/apache/solr/search/ solr/core/src/test/org/...
Author: mvg
Date: Sun Aug 7 09:01:57 2011
New Revision: 1154675
URL: http://svn.apache.org/viewvc?rev=1154675&view=rev
Log:
SOLR-2665: Added post group faceting.
Modified:
lucene/dev/branches/branch_3x/lucene/contrib/grouping/src/java/org/apache/lucene/search/grouping/AbstractAllGroupHeadsCollector.java
lucene/dev/branches/branch_3x/lucene/contrib/grouping/src/java/org/apache/lucene/search/grouping/TermAllGroupHeadsCollector.java
lucene/dev/branches/branch_3x/solr/CHANGES.txt
lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/Grouping.java
lucene/dev/branches/branch_3x/solr/core/src/test/org/apache/solr/TestGroupingSearch.java
lucene/dev/branches/branch_3x/solr/solrj/src/java/org/apache/solr/common/params/GroupParams.java
Modified: lucene/dev/branches/branch_3x/lucene/contrib/grouping/src/java/org/apache/lucene/search/grouping/AbstractAllGroupHeadsCollector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/grouping/src/java/org/apache/lucene/search/grouping/AbstractAllGroupHeadsCollector.java?rev=1154675&r1=1154674&r2=1154675&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/grouping/src/java/org/apache/lucene/search/grouping/AbstractAllGroupHeadsCollector.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/grouping/src/java/org/apache/lucene/search/grouping/AbstractAllGroupHeadsCollector.java Sun Aug 7 09:01:57 2011
@@ -134,8 +134,8 @@ public abstract class AbstractAllGroupHe
*/
protected class TemporalResult {
- protected GH groupHead;
- protected boolean stop;
+ public GH groupHead;
+ public boolean stop;
}
Modified: lucene/dev/branches/branch_3x/lucene/contrib/grouping/src/java/org/apache/lucene/search/grouping/TermAllGroupHeadsCollector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/grouping/src/java/org/apache/lucene/search/grouping/TermAllGroupHeadsCollector.java?rev=1154675&r1=1154674&r2=1154675&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/grouping/src/java/org/apache/lucene/search/grouping/TermAllGroupHeadsCollector.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/grouping/src/java/org/apache/lucene/search/grouping/TermAllGroupHeadsCollector.java Sun Aug 7 09:01:57 2011
@@ -307,7 +307,17 @@ public abstract class TermAllGroupHeadsC
} else {
if (sortOrds[compIDX] < 0) {
// The current segment doesn't contain the sort value we encountered before. Therefore the ord is negative.
- return sortValues[compIDX].compareTo(sortsIndex[compIDX].lookup[sortsIndex[compIDX].order[doc]]);
+ final String val1 = sortValues[compIDX];
+ final String val2 = sortsIndex[compIDX].lookup[sortsIndex[compIDX].order[doc]];
+ if (val1 == null) {
+ if (val2 == null) {
+ return 0;
+ }
+ return -1;
+ } else if (val2 == null) {
+ return 1;
+ }
+ return val1.compareTo(val2);
} else {
return sortOrds[compIDX] - sortsIndex[compIDX].order[doc];
}
@@ -421,7 +431,17 @@ public abstract class TermAllGroupHeadsC
public int compare(int compIDX, int doc) throws IOException {
if (sortOrds[compIDX] < 0) {
// The current segment doesn't contain the sort value we encountered before. Therefore the ord is negative.
- return sortValues[compIDX].compareTo(sortsIndex[compIDX].lookup[sortsIndex[compIDX].order[doc]]);
+ final String val1 = sortValues[compIDX];
+ final String val2 = sortsIndex[compIDX].lookup[sortsIndex[compIDX].order[doc]];
+ if (val1 == null) {
+ if (val2 == null) {
+ return 0;
+ }
+ return -1;
+ } else if (val2 == null) {
+ return 1;
+ }
+ return val1.compareTo(val2);
} else {
return sortOrds[compIDX] - sortsIndex[compIDX].order[doc];
}
Modified: lucene/dev/branches/branch_3x/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/CHANGES.txt?rev=1154675&r1=1154674&r2=1154675&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/CHANGES.txt (original)
+++ lucene/dev/branches/branch_3x/solr/CHANGES.txt Sun Aug 7 09:01:57 2011
@@ -90,6 +90,10 @@ New Features
* SOLR-2637: Added support for group result parsing in SolrJ.
(Tao Cheng, Martijn van Groningen)
+* SOLR-2665: Added post group faceting. Facet counts are based on the most
+ relevant document of each group matching the query. This feature has the
+ same impact on the StatsComponent. (Martijn van Groningen)
+
Optimizations
----------------------
Modified: lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java?rev=1154675&r1=1154674&r2=1154675&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java (original)
+++ lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java Sun Aug 7 09:01:57 2011
@@ -190,6 +190,7 @@ public class QueryComponent extends Sear
String[] queries = params.getParams(GroupParams.GROUP_QUERY);
String groupSortStr = params.get(GroupParams.GROUP_SORT);
boolean main = params.getBool(GroupParams.GROUP_MAIN, false);
+ boolean truncateGroups = params.getBool(GroupParams.GROUP_TRUNCATE, false);
String formatStr = params.get(GroupParams.GROUP_FORMAT, Grouping.Format.grouped.name());
Grouping.Format defaultFormat;
@@ -216,7 +217,8 @@ public class QueryComponent extends Sear
.setLimitDefault(limitDefault)
.setDefaultTotalCount(defaultTotalCount)
.setDocsPerGroupDefault(docsPerGroupDefault)
- .setGroupOffsetDefault(groupOffsetDefault);
+ .setGroupOffsetDefault(groupOffsetDefault)
+ .setGetGroupedDocSet(truncateGroups);
if (fields != null) {
for (String field : fields) {
Modified: lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/Grouping.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/Grouping.java?rev=1154675&r1=1154674&r2=1154675&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/Grouping.java (original)
+++ lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/search/Grouping.java Sun Aug 7 09:01:57 2011
@@ -23,6 +23,8 @@ import org.apache.lucene.index.IndexRead
import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.search.*;
import org.apache.lucene.search.grouping.*;
+import org.apache.lucene.util.FixedBitSet;
+import org.apache.lucene.util.OpenBitSet;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
@@ -68,6 +70,7 @@ public class Grouping {
private int maxDoc;
private boolean needScores;
private boolean getDocSet;
+ private boolean getGroupedDocSet;
private boolean getDocList; // doclist needed for debugging or highlighting
private Query query;
private DocSet filter;
@@ -210,6 +213,11 @@ public class Grouping {
return this;
}
+ public Grouping setGetGroupedDocSet(boolean getGroupedDocSet) {
+ this.getGroupedDocSet = getGroupedDocSet;
+ return this;
+ }
+
public List<Command> getCommands() {
return commands;
}
@@ -251,16 +259,21 @@ public class Grouping {
cmd.prepare();
}
+ AbstractAllGroupHeadsCollector<?> allGroupHeadsCollector = null;
List<Collector> collectors = new ArrayList<Collector>(commands.size());
for (Command cmd : commands) {
Collector collector = cmd.createFirstPassCollector();
- if (collector != null)
+ if (collector != null) {
collectors.add(collector);
+ }
+ if (getGroupedDocSet && allGroupHeadsCollector == null) {
+ collectors.add(allGroupHeadsCollector = cmd.createAllGroupCollector());
+ }
}
Collector allCollectors = MultiCollector.wrap(collectors.toArray(new Collector[collectors.size()]));
DocSetCollector setCollector = null;
- if (getDocSet) {
+ if (getDocSet && allGroupHeadsCollector == null) {
setCollector = new DocSetDelegateCollector(maxDoc >> 6, maxDoc, allCollectors);
allCollectors = setCollector;
}
@@ -284,7 +297,12 @@ public class Grouping {
searcher.search(query, luceneFilter, allCollectors);
}
- if (getDocSet) {
+ if (getGroupedDocSet && allGroupHeadsCollector != null) {
+ FixedBitSet fixedBitSet = allGroupHeadsCollector.retrieveGroupHeads(maxDoc);
+ long[] bits = fixedBitSet.getBits();
+ OpenBitSet openBitSet = new OpenBitSet(bits, bits.length);
+ qr.setDocSet(new BitDocSet(openBitSet));
+ } else if (getDocSet) {
qr.setDocSet(setCollector.getDocSet());
}
@@ -439,6 +457,17 @@ public class Grouping {
}
/**
+ * Returns a collector that is able to return the most relevant document of all groups.
+ * Returns <code>null</code> if the command doesn't support this type of collector.
+ *
+ * @return a collector that is able to return the most relevant document of all groups.
+ * @throws IOException If I/O related errors occur
+ */
+ public AbstractAllGroupHeadsCollector<?> createAllGroupCollector() throws IOException {
+ return null;
+ }
+
+ /**
* Performs any necessary post actions to prepare the response.
*
* @throws IOException If I/O related errors occur
@@ -631,6 +660,15 @@ public class Grouping {
/**
* {@inheritDoc}
*/
+ @Override
+ public AbstractAllGroupHeadsCollector<?> createAllGroupCollector() throws IOException {
+ Sort sortWithinGroup = groupSort != null ? groupSort : new Sort();
+ return TermAllGroupHeadsCollector.create(groupBy, sortWithinGroup);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
protected void finish() throws IOException {
result = secondPass != null ? secondPass.getTopGroups(0) : null;
if (main) {
Modified: lucene/dev/branches/branch_3x/solr/core/src/test/org/apache/solr/TestGroupingSearch.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/core/src/test/org/apache/solr/TestGroupingSearch.java?rev=1154675&r1=1154674&r2=1154675&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/core/src/test/org/apache/solr/TestGroupingSearch.java (original)
+++ lucene/dev/branches/branch_3x/solr/core/src/test/org/apache/solr/TestGroupingSearch.java Sun Aug 7 09:01:57 2011
@@ -262,6 +262,44 @@ public class TestGroupingSearch extends
);
}
+ @Test
+ public void testGroupingGroupedBasedFaceting() throws Exception {
+ assertU(add(doc("id", "1", "value1_s1", "1", "value2_i", "1", "value3_s1", "a", "value4_s1", "1")));
+ assertU(add(doc("id", "2", "value1_s1", "1", "value2_i", "2", "value3_s1", "a", "value4_s1", "1")));
+ assertU(commit());
+ assertU(add(doc("id", "3", "value1_s1", "2", "value2_i", "3", "value3_s1", "b", "value4_s1", "2")));
+ assertU(add(doc("id", "4", "value1_s1", "1", "value2_i", "4", "value3_s1", "a", "value4_s1", "1")));
+ assertU(add(doc("id", "5", "value1_s1", "2", "value2_i", "5", "value3_s1", "b", "value4_s1", "2")));
+ assertU(commit());
+
+ // Facet counts based on documents
+ SolrQueryRequest req = req("q", "*:*", "sort", "value2_i asc", "rows", "1", "group", "true", "group.field",
+ "value1_s1", "fl", "id", "facet", "true", "facet.field", "value3_s1", "group.truncate", "false");
+ assertJQ(
+ req,
+ "/grouped=={'value1_s1':{'matches':5,'groups':[{'groupValue':'1','doclist':{'numFound':3,'start':0,'docs':[{'id':'1'}]}}]}}",
+ "/facet_counts=={'facet_queries':{},'facet_fields':{'value3_s1':['a',3,'b',2]},'facet_dates':{},'facet_ranges':{}}"
+ );
+
+ // Facet counts based on groups
+ req = req("q", "*:*", "sort", "value2_i asc", "rows", "1", "group", "true", "group.field",
+ "value1_s1", "fl", "id", "facet", "true", "facet.field", "value3_s1", "group.truncate", "true");
+ assertJQ(
+ req,
+ "/grouped=={'value1_s1':{'matches':5,'groups':[{'groupValue':'1','doclist':{'numFound':3,'start':0,'docs':[{'id':'1'}]}}]}}",
+ "/facet_counts=={'facet_queries':{},'facet_fields':{'value3_s1':['a',1,'b',1]},'facet_dates':{},'facet_ranges':{}}"
+ );
+
+ // Facet counts based on groups without sort on an int field.
+ req = req("q", "*:*", "rows", "1", "group", "true", "group.field", "value4_s1", "fl", "id", "facet", "true",
+ "facet.field", "value3_s1", "group.truncate", "true");
+ assertJQ(
+ req,
+ "/grouped=={'value4_s1':{'matches':5,'groups':[{'groupValue':'1','doclist':{'numFound':3,'start':0,'docs':[{'id':'1'}]}}]}}",
+ "/facet_counts=={'facet_queries':{},'facet_fields':{'value3_s1':['a',1,'b',1]},'facet_dates':{},'facet_ranges':{}}"
+ );
+ }
+
static String f = "foo_s1";
static String f2 = "foo2_i";
@@ -500,154 +538,196 @@ public class TestGroupingSearch extends
@Test
public void testRandomGrouping() throws Exception {
- /**
- updateJ("{\"add\":{\"doc\":{\"id\":\"77\"}}}", params("commit","true"));
- assertJQ(req("q","id:77"), "/response/numFound==1");
-
- Doc doc = createDocObj(types);
- updateJ(toJSON(doc), params("commit","true"));
-
- assertJQ(req("q","id:"+doc.id), "/response/numFound==1");
- **/
-
- int indexIter=50 * RANDOM_MULTIPLIER; // make >0 to enable test
- int queryIter=100 * RANDOM_MULTIPLIER;
-
- while (--indexIter >= 0) {
-
- int indexSize = random.nextInt(25 * RANDOM_MULTIPLIER);
-//indexSize=2;
- List<FldType> types = new ArrayList<FldType>();
- types.add(new FldType("id",ONE_ONE, new SVal('A','Z',4,4)));
- types.add(new FldType("score_s1",ONE_ONE, new SVal('a','c',1,1))); // field used to score
- types.add(new FldType("bar_s1",ONE_ONE, new SVal('a','z',3,5)));
- types.add(new FldType(FOO_STRING_FIELD,ZERO_ONE, new SVal('a','z',1,2)));
- types.add(new FldType(SMALL_STRING_FIELD,ZERO_ONE, new SVal('a',(char)('c'+indexSize/10),1,1)));
-// types.add(new FldType(SMALL_INT_FIELD,ZERO_ONE, new IRange(0,5+indexSize/10)));
-
- clearIndex();
- Map<Comparable, Doc> model = indexDocs(types, null, indexSize);
- //System.out.println("############### model=" + model);
+ try {
+ int indexIter=50 * RANDOM_MULTIPLIER; // make >0 to enable test
+ int queryIter=100 * RANDOM_MULTIPLIER;
+
+ while (--indexIter >= 0) {
+
+ int indexSize = random.nextInt(25 * RANDOM_MULTIPLIER);
+ List<FldType> types = new ArrayList<FldType>();
+ types.add(new FldType("id",ONE_ONE, new SVal('A','Z',4,4)));
+ types.add(new FldType("score_s1",ONE_ONE, new SVal('a','c',1,1))); // field used to score
+ types.add(new FldType("bar_s1",ONE_ONE, new SVal('a','z',3,5)));
+ types.add(new FldType(FOO_STRING_FIELD,ONE_ONE, new SVal('a','z',1,2)));
+ types.add(new FldType(SMALL_STRING_FIELD,ZERO_ONE, new SVal('a',(char)('c'+indexSize/10),1,1)));
- // test with specific docs
- if (false) {
clearIndex();
- model.clear();
- Doc d1 = createDoc(types);
- d1.getValues(SMALL_STRING_FIELD).set(0,"c");
- d1.getValues(SMALL_INT_FIELD).set(0,5);
- d1.order = 0;
- updateJ(toJSON(d1), params("commit","true"));
- model.put(d1.id, d1);
-
- d1 = createDoc(types);
- d1.getValues(SMALL_STRING_FIELD).set(0,"b");
- d1.getValues(SMALL_INT_FIELD).set(0,5);
- d1.order = 1;
- updateJ(toJSON(d1), params("commit","false"));
- model.put(d1.id, d1);
-
- d1 = createDoc(types);
- d1.getValues(SMALL_STRING_FIELD).set(0,"c");
- d1.getValues(SMALL_INT_FIELD).set(0,5);
- d1.order = 2;
- updateJ(toJSON(d1), params("commit","false"));
- model.put(d1.id, d1);
-
- d1 = createDoc(types);
- d1.getValues(SMALL_STRING_FIELD).set(0,"c");
- d1.getValues(SMALL_INT_FIELD).set(0,5);
- d1.order = 3;
- updateJ(toJSON(d1), params("commit","false"));
- model.put(d1.id, d1);
-
- d1 = createDoc(types);
- d1.getValues(SMALL_STRING_FIELD).set(0,"b");
- d1.getValues(SMALL_INT_FIELD).set(0,2);
- d1.order = 4;
- updateJ(toJSON(d1), params("commit","true"));
- model.put(d1.id, d1);
- }
+ Map<Comparable, Doc> model = indexDocs(types, null, indexSize);
+ // test with specific docs
+ if (false) {
+ clearIndex();
+ model.clear();
+ Doc d1 = createDoc(types);
+ d1.getValues(SMALL_STRING_FIELD).set(0,"c");
+ d1.getValues(SMALL_INT_FIELD).set(0,5);
+ d1.order = 0;
+ updateJ(toJSON(d1), params("commit","true"));
+ model.put(d1.id, d1);
+
+ d1 = createDoc(types);
+ d1.getValues(SMALL_STRING_FIELD).set(0,"b");
+ d1.getValues(SMALL_INT_FIELD).set(0,5);
+ d1.order = 1;
+ updateJ(toJSON(d1), params("commit","false"));
+ model.put(d1.id, d1);
+
+ d1 = createDoc(types);
+ d1.getValues(SMALL_STRING_FIELD).set(0,"c");
+ d1.getValues(SMALL_INT_FIELD).set(0,5);
+ d1.order = 2;
+ updateJ(toJSON(d1), params("commit","false"));
+ model.put(d1.id, d1);
+
+ d1 = createDoc(types);
+ d1.getValues(SMALL_STRING_FIELD).set(0,"c");
+ d1.getValues(SMALL_INT_FIELD).set(0,5);
+ d1.order = 3;
+ updateJ(toJSON(d1), params("commit","false"));
+ model.put(d1.id, d1);
+
+ d1 = createDoc(types);
+ d1.getValues(SMALL_STRING_FIELD).set(0,"b");
+ d1.getValues(SMALL_INT_FIELD).set(0,2);
+ d1.order = 4;
+ updateJ(toJSON(d1), params("commit","true"));
+ model.put(d1.id, d1);
+ }
- for (int qiter=0; qiter<queryIter; qiter++) {
- String groupField = types.get(random.nextInt(types.size())).fname;
- int rows = random.nextInt(10)==0 ? random.nextInt(model.size()+2) : random.nextInt(11)-1;
- int start = random.nextInt(5)==0 ? random.nextInt(model.size()+2) : random.nextInt(5); // pick a small start normally for better coverage
- int group_limit = random.nextInt(10)==0 ? random.nextInt(model.size()+2) : random.nextInt(11)-1;
- int group_offset = random.nextInt(10)==0 ? random.nextInt(model.size()+2) : random.nextInt(2); // pick a small start normally for better coverage
-
- String[] stringSortA = new String[1];
- Comparator<Doc> sortComparator = createSort(h.getCore().getSchema(), types, stringSortA);
- String sortStr = stringSortA[0];
- Comparator<Doc> groupComparator = random.nextBoolean() ? sortComparator : createSort(h.getCore().getSchema(), types, stringSortA);
- String groupSortStr = stringSortA[0];
-
- // since groupSortStr defaults to sortStr, we need to normalize null to "score desc" if
- // sortStr != null.
- if (groupSortStr == null && groupSortStr != sortStr) {
- groupSortStr = "score desc";
- }
+ for (int qiter=0; qiter<queryIter; qiter++) {
+ String groupField = types.get(random.nextInt(types.size())).fname;
- // Test specific case
- if (false) {
- groupField=SMALL_INT_FIELD;
- sortComparator=createComparator(Arrays.asList(createComparator(SMALL_STRING_FIELD, true, true, false, true)));
- sortStr = SMALL_STRING_FIELD + " asc";
- groupComparator = createComparator(Arrays.asList(createComparator(SMALL_STRING_FIELD, true, true, false, false)));
- groupSortStr = SMALL_STRING_FIELD + " asc";
- rows=1; start=0; group_offset=1; group_limit=1;
- }
+ int rows = random.nextInt(10)==0 ? random.nextInt(model.size()+2) : random.nextInt(11)-1;
+ int start = random.nextInt(5)==0 ? random.nextInt(model.size()+2) : random.nextInt(5); // pick a small start normally for better coverage
+ int group_limit = random.nextInt(10)==0 ? random.nextInt(model.size()+2) : random.nextInt(11)-1;
+ int group_offset = random.nextInt(10)==0 ? random.nextInt(model.size()+2) : random.nextInt(2); // pick a small start normally for better coverage
+
+ String[] stringSortA = new String[1];
+ Comparator<Doc> sortComparator = createSort(h.getCore().getSchema(), types, stringSortA);
+ String sortStr = stringSortA[0];
+ Comparator<Doc> groupComparator = random.nextBoolean() ? sortComparator : createSort(h.getCore().getSchema(), types, stringSortA);
+ String groupSortStr = stringSortA[0];
+
+ // since groupSortStr defaults to sortStr, we need to normalize null to "score desc" if
+ // sortStr != null.
+ if (groupSortStr == null && groupSortStr != sortStr) {
+ groupSortStr = "score desc";
+ }
+
+ // Test specific case
+ if (false) {
+ groupField=SMALL_INT_FIELD;
+ sortComparator=createComparator(Arrays.asList(createComparator(SMALL_STRING_FIELD, true, true, false, true)));
+ sortStr = SMALL_STRING_FIELD + " asc";
+ groupComparator = createComparator(Arrays.asList(createComparator(SMALL_STRING_FIELD, true, true, false, false)));
+ groupSortStr = SMALL_STRING_FIELD + " asc";
+ rows=1; start=0; group_offset=1; group_limit=1;
+ }
- Map<Comparable, Grp> groups = groupBy(model.values(), groupField);
+ Map<Comparable, Grp> groups = groupBy(model.values(), groupField);
- // first sort the docs in each group
- for (Grp grp : groups.values()) {
- Collections.sort(grp.docs, groupComparator);
- }
+ // first sort the docs in each group
+ for (Grp grp : groups.values()) {
+ Collections.sort(grp.docs, groupComparator);
+ }
- // now sort the groups
+ // now sort the groups
- // if sort != group.sort, we need to find the max doc by "sort"
- if (groupComparator != sortComparator) {
- for (Grp grp : groups.values()) grp.setMaxDoc(sortComparator);
- }
+ // if sort != group.sort, we need to find the max doc by "sort"
+ if (groupComparator != sortComparator) {
+ for (Grp grp : groups.values()) grp.setMaxDoc(sortComparator);
+ }
+
+ List<Grp> sortedGroups = new ArrayList<Grp>(groups.values());
+ Collections.sort(sortedGroups, groupComparator==sortComparator ? createFirstDocComparator(sortComparator) : createMaxDocComparator(sortComparator));
- List<Grp> sortedGroups = new ArrayList(groups.values());
- Collections.sort(sortedGroups, groupComparator==sortComparator ? createFirstDocComparator(sortComparator) : createMaxDocComparator(sortComparator));
+ boolean includeNGroups = random.nextBoolean();
+ Object modelResponse = buildGroupedResult(h.getCore().getSchema(), sortedGroups, start, rows, group_offset, group_limit, includeNGroups);
- boolean includeNGroups = random.nextBoolean();
- Object modelResponse = buildGroupedResult(h.getCore().getSchema(), sortedGroups, start, rows, group_offset, group_limit, includeNGroups);
+ boolean truncateGroups = random.nextBoolean();
+ Map<String, Integer> facetCounts = new TreeMap<String, Integer>();
+ if (truncateGroups) {
+ for (Grp grp : sortedGroups) {
+ Doc doc = grp.docs.get(0);
+ if (doc.getValues(FOO_STRING_FIELD) == null) {
+ continue;
+ }
+
+ String key = doc.getFirstValue(FOO_STRING_FIELD).toString();
+ boolean exists = facetCounts.containsKey(key);
+ int count = exists ? facetCounts.get(key) : 0;
+ facetCounts.put(key, ++count);
+ }
+ } else {
+ for (Doc doc : model.values()) {
+ if (doc.getValues(FOO_STRING_FIELD) == null) {
+ continue;
+ }
+
+ for (Comparable field : doc.getValues(FOO_STRING_FIELD)) {
+ String key = field.toString();
+ boolean exists = facetCounts.containsKey(key);
+ int count = exists ? facetCounts.get(key) : 0;
+ facetCounts.put(key, ++count);
+ }
+ }
+ }
+ List<Comparable> expectedFacetResponse = new ArrayList<Comparable>();
+ for (Map.Entry<String, Integer> stringIntegerEntry : facetCounts.entrySet()) {
+ expectedFacetResponse.add(stringIntegerEntry.getKey());
+ expectedFacetResponse.add(stringIntegerEntry.getValue());
+ }
- int randomPercentage = random.nextInt(101);
- // TODO: create a random filter too
- SolrQueryRequest req = req("group","true","wt","json","indent","true", "echoParams","all", "q","{!func}score_f", "group.field",groupField
- ,sortStr==null ? "nosort":"sort", sortStr ==null ? "": sortStr
- ,(groupSortStr==null || groupSortStr==sortStr) ? "noGroupsort":"group.sort", groupSortStr==null ? "": groupSortStr
- ,"rows",""+rows, "start",""+start, "group.offset",""+group_offset, "group.limit",""+group_limit,
- GroupParams.GROUP_CACHE_PERCENTAGE, Integer.toString(randomPercentage), GroupParams.GROUP_TOTAL_COUNT, includeNGroups ? "true" : "false"
- );
-
- String strResponse = h.query(req);
-
- Object realResponse = ObjectBuilder.fromJSON(strResponse);
- String err = JSONTestUtil.matchObj("/grouped/"+groupField, realResponse, modelResponse);
- if (err != null) {
- log.error("GROUPING MISMATCH: " + err
- + "\n\trequest="+req
- + "\n\tresult="+strResponse
- + "\n\texpected="+ JSONUtil.toJSON(modelResponse)
- + "\n\tsorted_model="+ sortedGroups
+ int randomPercentage = random.nextInt(101);
+ // TODO: create a random filter too
+ SolrQueryRequest req = req("group","true","wt","json","indent","true", "echoParams","all", "q","{!func}score_f", "group.field",groupField
+ ,sortStr==null ? "nosort":"sort", sortStr ==null ? "": sortStr
+ ,(groupSortStr==null || groupSortStr==sortStr) ? "noGroupsort":"group.sort", groupSortStr==null ? "": groupSortStr
+ ,"rows",""+rows, "start",""+start, "group.offset",""+group_offset, "group.limit",""+group_limit,
+ GroupParams.GROUP_CACHE_PERCENTAGE, Integer.toString(randomPercentage), GroupParams.GROUP_TOTAL_COUNT, includeNGroups ? "true" : "false",
+ "facet", "true", "facet.sort", "index", "facet.limit", "-1", "facet.field", FOO_STRING_FIELD,
+ GroupParams.GROUP_TRUNCATE, truncateGroups ? "true" : "false", "facet.mincount", "1"
);
- // re-execute the request... good for putting a breakpoint here for debugging
- String rsp = h.query(req);
+ String strResponse = h.query(req);
- fail(err);
- }
- } // end query iter
- } // end index iter
+ Object realResponse = ObjectBuilder.fromJSON(strResponse);
+ String err = JSONTestUtil.matchObj("/grouped/"+groupField, realResponse, modelResponse);
+ if (err != null) {
+ log.error("GROUPING MISMATCH: " + err
+ + "\n\trequest="+req
+ + "\n\tresult="+strResponse
+ + "\n\texpected="+ JSONUtil.toJSON(modelResponse)
+ + "\n\tsorted_model="+ sortedGroups
+ );
+
+ // re-execute the request... good for putting a breakpoint here for debugging
+ String rsp = h.query(req);
+
+ fail(err);
+ }
+
+ // assert post / pre grouping facets
+ err = JSONTestUtil.matchObj("/facet_counts/facet_fields/"+FOO_STRING_FIELD, realResponse, expectedFacetResponse);
+ if (err != null) {
+ log.error("GROUPING MISMATCH: " + err
+ + "\n\trequest="+req
+ + "\n\tresult="+strResponse
+ + "\n\texpected="+ JSONUtil.toJSON(expectedFacetResponse)
+ );
+
+ // re-execute the request... good for putting a breakpoint here for debugging
+ h.query(req);
+ fail(err);
+ }
+ } // end query iter
+ } // end index iter
+ } finally {
+ // B/c the facet.field is also used of grouping we have the purge the FC to avoid FC insanity
+ FieldCache.DEFAULT.purgeAllCaches();
+ }
}
Modified: lucene/dev/branches/branch_3x/solr/solrj/src/java/org/apache/solr/common/params/GroupParams.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/solrj/src/java/org/apache/solr/common/params/GroupParams.java?rev=1154675&r1=1154674&r2=1154675&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/solrj/src/java/org/apache/solr/common/params/GroupParams.java (original)
+++ lucene/dev/branches/branch_3x/solr/solrj/src/java/org/apache/solr/common/params/GroupParams.java Sun Aug 7 09:01:57 2011
@@ -48,8 +48,9 @@ public interface GroupParams {
// Note: Since you can supply multiple fields to group on, but only have a facets for the whole result. It only makes
// sense to me to support these parameters for the first group.
- /** Whether the docSet (for example for faceting) should be based on plain documents (a.k.a UNGROUPED) or on the groups (a.k.a GROUPED). */
- public static final String GROUP_COLLAPSE = GROUP + ".collapse";
+ /** Whether the docSet (for example for faceting) should be based on plain documents (a.k.a UNGROUPED) or on the groups (a.k.a GROUPED).
+ * The docSet will only the most relevant documents per group. It is if you query for everything with group.limit=1 */
+ public static final String GROUP_TRUNCATE = GROUP + ".truncate";
/** Whether the group count should be included in the response. */
public static final String GROUP_TOTAL_COUNT = GROUP + ".ngroups";