You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jb...@apache.org on 2013/12/18 18:05:01 UTC
svn commit: r1552027 - in /lucene/dev/branches/branch_4x: ./ solr/
solr/core/ solr/core/src/java/org/apache/solr/search/
solr/core/src/test/org/apache/solr/search/
Author: jbernste
Date: Wed Dec 18 17:05:00 2013
New Revision: 1552027
URL: http://svn.apache.org/r1552027
Log:
SOLR-5416: CollapsingQParserPlugin bug with tagging
Added:
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/ScoreFilter.java
- copied unchanged from r1551999, lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/ScoreFilter.java
Modified:
lucene/dev/branches/branch_4x/ (props changed)
lucene/dev/branches/branch_4x/solr/ (props changed)
lucene/dev/branches/branch_4x/solr/core/ (props changed)
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/search/TestCollapseQParserPlugin.java
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java?rev=1552027&r1=1552026&r2=1552027&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java Wed Dec 18 17:05:00 2013
@@ -118,21 +118,15 @@ public class CollapsingQParserPlugin ext
}
}
- private class CollapsingPostFilter extends ExtendedQueryBase implements PostFilter {
+ public class CollapsingPostFilter extends ExtendedQueryBase implements PostFilter, ScoreFilter {
private Object cacheId;
private String field;
- private int leafCount;
- private SortedDocValues docValues;
- private int maxDoc;
private String max;
private String min;
- private FieldType fieldType;
+ private boolean needsScores = true;
private int nullPolicy;
- private SolrIndexSearcher searcher;
- private SolrParams solrParams;
private Map context;
- private IndexSchema schema;
public static final int NULL_POLICY_IGNORE = 0;
public static final int NULL_POLICY_COLLAPSE = 1;
public static final int NULL_POLICY_EXPAND = 2;
@@ -180,7 +174,13 @@ public class CollapsingQParserPlugin ext
public CollapsingPostFilter(SolrParams localParams, SolrParams params, SolrQueryRequest request) throws IOException {
this.cacheId = new Object();
this.field = localParams.get("field");
- this.solrParams = params;
+ this.max = localParams.get("max");
+ this.min = localParams.get("min");
+ this.context = request.getContext();
+ if(this.min != null || this.max != null) {
+ this.needsScores = needsScores(params);
+ }
+
String nPolicy = localParams.get("nullPolicy", NULL_IGNORE);
if(nPolicy.equals(NULL_IGNORE)) {
this.nullPolicy = NULL_POLICY_IGNORE;
@@ -191,34 +191,12 @@ public class CollapsingQParserPlugin ext
} else {
throw new IOException("Invalid nullPolicy:"+nPolicy);
}
- this.searcher = request.getSearcher();
- this.leafCount = searcher.getTopReaderContext().leaves().size();
- this.maxDoc = searcher.maxDoc();
- this.schema = searcher.getSchema();
- SchemaField schemaField = schema.getField(this.field);
- if(schemaField.hasDocValues()) {
- this.docValues = searcher.getAtomicReader().getSortedDocValues(this.field);
- } else {
- this.docValues = FieldCache.DEFAULT.getTermsIndex(searcher.getAtomicReader(), this.field);
- }
-
- this.max = localParams.get("max");
- if(this.max != null) {
- this.fieldType = searcher.getSchema().getField(this.max).getType();
- }
-
- this.min = localParams.get("min");
- if(this.min != null) {
- this.fieldType = searcher.getSchema().getField(this.min).getType();
- }
-
- this.context = request.getContext();
}
- private IntOpenHashSet getBoostDocs(IndexSearcher indexSearcher, Set<String> boosted) throws IOException {
+ private IntOpenHashSet getBoostDocs(SolrIndexSearcher indexSearcher, Set<String> boosted) throws IOException {
IntOpenHashSet boostDocs = null;
if(boosted != null) {
- SchemaField idField = this.schema.getUniqueKeyField();
+ SchemaField idField = indexSearcher.getSchema().getUniqueKeyField();
String fieldName = idField.getName();
HashSet<BytesRef> localBoosts = new HashSet(boosted.size()*2);
Iterator<String> boostedIt = boosted.iterator();
@@ -258,22 +236,47 @@ public class CollapsingQParserPlugin ext
public DelegatingCollector getFilterCollector(IndexSearcher indexSearcher) {
try {
- IntOpenHashSet boostDocs = getBoostDocs(indexSearcher, (Set<String>) (this.context.get(QueryElevationComponent.BOOSTED)));
+
+ SolrIndexSearcher searcher = (SolrIndexSearcher)indexSearcher;
+ IndexSchema schema = searcher.getSchema();
+ SchemaField schemaField = schema.getField(this.field);
+
+ SortedDocValues docValues = null;
+
+ if(schemaField.hasDocValues()) {
+ docValues = searcher.getAtomicReader().getSortedDocValues(this.field);
+ } else {
+ docValues = FieldCache.DEFAULT.getTermsIndex(searcher.getAtomicReader(), this.field);
+ }
+
+ FieldType fieldType = null;
+
+ if(this.max != null) {
+ fieldType = searcher.getSchema().getField(this.max).getType();
+ }
+
+ if(this.min != null) {
+ fieldType = searcher.getSchema().getField(this.min).getType();
+ }
+
+ int maxDoc = searcher.maxDoc();
+ int leafCount = searcher.getTopReaderContext().leaves().size();
+
+ IntOpenHashSet boostDocs = getBoostDocs(searcher, (Set<String>) (this.context.get(QueryElevationComponent.BOOSTED)));
if(this.min != null || this.max != null) {
- return new CollapsingFieldValueCollector(this.maxDoc,
- this.leafCount,
- this.docValues,
- this.searcher,
- this.nullPolicy,
- max != null ? this.max : this.min,
- max != null,
- needsScores(this.solrParams),
- this.fieldType,
- boostDocs);
+ return new CollapsingFieldValueCollector(maxDoc,
+ leafCount,
+ docValues,
+ this.nullPolicy,
+ max != null ? this.max : this.min,
+ max != null,
+ this.needsScores,
+ fieldType,
+ boostDocs);
} else {
- return new CollapsingScoreCollector(this.maxDoc, this.leafCount, this.docValues, this.nullPolicy, boostDocs);
+ return new CollapsingScoreCollector(maxDoc, leafCount, docValues, this.nullPolicy, boostDocs);
}
} catch (Exception e) {
throw new RuntimeException(e);
@@ -283,7 +286,7 @@ public class CollapsingQParserPlugin ext
private boolean needsScores(SolrParams params) {
String sortSpec = params.get("sort");
- if(sortSpec != null) {
+ if(sortSpec != null && sortSpec.length()!=0) {
String[] sorts = sortSpec.split(",");
for(String s: sorts) {
String parts[] = s.split(" ");
@@ -500,7 +503,6 @@ public class CollapsingQParserPlugin ext
public CollapsingFieldValueCollector(int maxDoc,
int segments,
SortedDocValues values,
- SolrIndexSearcher searcher,
int nullPolicy,
String field,
boolean max,
@@ -516,11 +518,11 @@ public class CollapsingQParserPlugin ext
this.needsScores = needsScores;
this.boostDocs = boostDocs;
if(fieldType instanceof TrieIntField) {
- this.fieldValueCollapse = new IntValueCollapse(searcher, field, nullPolicy, new int[valueCount], max, this.needsScores, boostDocs);
+ this.fieldValueCollapse = new IntValueCollapse(maxDoc, field, nullPolicy, new int[valueCount], max, this.needsScores, boostDocs);
} else if(fieldType instanceof TrieLongField) {
- this.fieldValueCollapse = new LongValueCollapse(searcher, field, nullPolicy, new int[valueCount], max, this.needsScores, boostDocs);
+ this.fieldValueCollapse = new LongValueCollapse(maxDoc, field, nullPolicy, new int[valueCount], max, this.needsScores, boostDocs);
} else if(fieldType instanceof TrieFloatField) {
- this.fieldValueCollapse = new FloatValueCollapse(searcher, field, nullPolicy, new int[valueCount], max, this.needsScores, boostDocs);
+ this.fieldValueCollapse = new FloatValueCollapse(maxDoc, field, nullPolicy, new int[valueCount], max, this.needsScores, boostDocs);
} else {
throw new IOException("min/max must be either TrieInt, TrieLong or TrieFloat.");
}
@@ -616,7 +618,7 @@ public class CollapsingQParserPlugin ext
public abstract void collapse(int ord, int contextDoc, int globalDoc) throws IOException;
public abstract void setNextReader(AtomicReaderContext context) throws IOException;
- public FieldValueCollapse(SolrIndexSearcher searcher,
+ public FieldValueCollapse(int maxDoc,
String field,
int nullPolicy,
boolean max,
@@ -626,7 +628,7 @@ public class CollapsingQParserPlugin ext
this.nullPolicy = nullPolicy;
this.max = max;
this.needsScores = needsScores;
- this.collapsedSet = new OpenBitSet(searcher.maxDoc());
+ this.collapsedSet = new OpenBitSet(maxDoc);
this.boostDocs = boostDocs;
if(this.boostDocs != null) {
Iterator<IntCursor> it = boostDocs.iterator();
@@ -676,14 +678,14 @@ public class CollapsingQParserPlugin ext
private int nullVal;
private int[] ordVals;
- public IntValueCollapse(SolrIndexSearcher searcher,
+ public IntValueCollapse(int maxDoc,
String field,
int nullPolicy,
int[] ords,
boolean max,
boolean needsScores,
IntOpenHashSet boostDocs) throws IOException {
- super(searcher, field, nullPolicy, max, needsScores, boostDocs);
+ super(maxDoc, field, nullPolicy, max, needsScores, boostDocs);
this.ords = ords;
this.ordVals = new int[ords.length];
Arrays.fill(ords, -1);
@@ -745,14 +747,13 @@ public class CollapsingQParserPlugin ext
private long nullVal;
private long[] ordVals;
- public LongValueCollapse(SolrIndexSearcher searcher,
- String field,
+ public LongValueCollapse(int maxDoc, String field,
int nullPolicy,
int[] ords,
boolean max,
boolean needsScores,
IntOpenHashSet boostDocs) throws IOException {
- super(searcher, field, nullPolicy, max, needsScores, boostDocs);
+ super(maxDoc, field, nullPolicy, max, needsScores, boostDocs);
this.ords = ords;
this.ordVals = new long[ords.length];
Arrays.fill(ords, -1);
@@ -814,14 +815,14 @@ public class CollapsingQParserPlugin ext
private float nullVal;
private float[] ordVals;
- public FloatValueCollapse(SolrIndexSearcher searcher,
+ public FloatValueCollapse(int maxDoc,
String field,
int nullPolicy,
int[] ords,
boolean max,
boolean needsScores,
IntOpenHashSet boostDocs) throws IOException {
- super(searcher, field, nullPolicy, max, needsScores, boostDocs);
+ super(maxDoc, field, nullPolicy, max, needsScores, boostDocs);
this.ords = ords;
this.ordVals = new float[ords.length];
Arrays.fill(ords, -1);
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java?rev=1552027&r1=1552026&r2=1552027&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java Wed Dec 18 17:05:00 2013
@@ -862,6 +862,25 @@ public class SolrIndexSearcher extends I
}
};
+ private DocSet getDocSetScore(List<Query> queries) throws IOException {
+ Query main = queries.remove(0);
+ ProcessedFilter pf = getProcessedFilter(null, queries);
+ DocSetCollector setCollector = new DocSetCollector(maxDoc()>>6, maxDoc());
+ Collector collector = setCollector;
+ if (pf.postFilter != null) {
+ pf.postFilter.setLastDelegate(collector);
+ collector = pf.postFilter;
+ }
+
+ search(main, pf.filter, collector);
+
+ if(collector instanceof DelegatingCollector) {
+ ((DelegatingCollector) collector).finish();
+ }
+
+ DocSet docSet = setCollector.getDocSet();
+ return docSet;
+ }
/**
* Returns the set of document ids matching all queries.
@@ -872,6 +891,15 @@ public class SolrIndexSearcher extends I
* The DocSet returned should <b>not</b> be modified.
*/
public DocSet getDocSet(List<Query> queries) throws IOException {
+
+ if(queries != null) {
+ for(Query q : queries) {
+ if(q instanceof ScoreFilter) {
+ return getDocSetScore(queries);
+ }
+ }
+ }
+
ProcessedFilter pf = getProcessedFilter(null, queries);
if (pf.answer != null) return pf.answer;
Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/search/TestCollapseQParserPlugin.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/search/TestCollapseQParserPlugin.java?rev=1552027&r1=1552026&r2=1552027&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/search/TestCollapseQParserPlugin.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/search/TestCollapseQParserPlugin.java Wed Dec 18 17:05:00 2013
@@ -47,24 +47,54 @@ public class TestCollapseQParserPlugin e
public void testCollapseQueries() throws Exception {
String[] doc = {"id","1", "term_s", "YYYY", "group_s", "group1", "test_ti", "5", "test_tl", "10", "test_tf", "2000"};
assertU(adoc(doc));
+ assertU(commit());
String[] doc1 = {"id","2", "term_s","YYYY", "group_s", "group1", "test_ti", "50", "test_tl", "100", "test_tf", "200"};
assertU(adoc(doc1));
+
+
String[] doc2 = {"id","3", "term_s", "YYYY", "test_ti", "5000", "test_tl", "100", "test_tf", "200"};
assertU(adoc(doc2));
-
+ assertU(commit());
String[] doc3 = {"id","4", "term_s", "YYYY", "test_ti", "500", "test_tl", "1000", "test_tf", "2000"};
assertU(adoc(doc3));
+
+ String[] doc4 = {"id","5", "term_s", "YYYY", "group_s", "group2", "test_ti", "4", "test_tl", "10", "test_tf", "2000"};
+ assertU(adoc(doc4));
assertU(commit());
+ String[] doc5 = {"id","6", "term_s","YYYY", "group_s", "group2", "test_ti", "10", "test_tl", "100", "test_tf", "200"};
+ assertU(adoc(doc5));
+ assertU(commit());
+
- //Test collapse by score
+
+ //Test collapse by score and following sort by score
ModifiableSolrParams params = new ModifiableSolrParams();
params.add("q", "*:*");
params.add("fq", "{!collapse field=group_s}");
params.add("defType", "edismax");
params.add("bf", "field(test_ti)");
- assertQ(req(params), "*[count(//doc)=1]", "//doc[./int[@name='test_ti']='50']");
+ assertQ(req(params), "*[count(//doc)=2]",
+ "//result/doc[1]/float[@name='id'][.='2.0']",
+ "//result/doc[2]/float[@name='id'][.='6.0']"
+ );
+
+
+ // SOLR-5544 test ordering with empty sort param
+ params = new ModifiableSolrParams();
+ params.add("q", "*:*");
+ params.add("fq", "{!collapse field=group_s nullPolicy=expand min=test_tf}");
+ params.add("defType", "edismax");
+ params.add("bf", "field(test_ti)");
+ params.add("sort","");
+ assertQ(req(params), "*[count(//doc)=4]",
+ "//result/doc[1]/float[@name='id'][.='3.0']",
+ "//result/doc[2]/float[@name='id'][.='4.0']",
+ "//result/doc[3]/float[@name='id'][.='2.0']",
+ "//result/doc[4]/float[@name='id'][.='6.0']"
+ );
+
//Test collapse by score with elevation
@@ -75,54 +105,152 @@ public class TestCollapseQParserPlugin e
params.add("bf", "field(test_ti)");
params.add("qf", "term_s");
params.add("qt", "/elevate");
- assertQ(req(params), "*[count(//doc)=3]", "//doc[./int[1][@name='test_ti']='5']");
+ assertQ(req(params), "*[count(//doc)=4]",
+ "//result/doc[1]/float[@name='id'][.='1.0']");
+
+
+
+
+ //Test collapse by min int field and sort
+ params = new ModifiableSolrParams();
+ params.add("q", "*:*");
+ params.add("fq", "{!collapse field=group_s min=test_ti}");
+ params.add("sort", "id desc");
+ assertQ(req(params), "*[count(//doc)=2]",
+ "//result/doc[1]/float[@name='id'][.='5.0']",
+ "//result/doc[2]/float[@name='id'][.='1.0']");
+
+ params = new ModifiableSolrParams();
+ params.add("q", "*:*");
+ params.add("fq", "{!collapse field=group_s min=test_ti}");
+ params.add("sort", "id asc");
+ assertQ(req(params), "*[count(//doc)=2]",
+ "//result/doc[1]/float[@name='id'][.='1.0']",
+ "//result/doc[2]/float[@name='id'][.='5.0']");
+
+ params = new ModifiableSolrParams();
+ params.add("q", "*:*");
+ params.add("fq", "{!collapse field=group_s min=test_ti}");
+ params.add("sort", "test_tl asc,id desc");
+ assertQ(req(params), "*[count(//doc)=2]",
+ "//result/doc[1]/float[@name='id'][.='5.0']",
+ "//result/doc[2]/float[@name='id'][.='1.0']");
- //Test collapse by min int field
params = new ModifiableSolrParams();
params.add("q", "*:*");
params.add("fq", "{!collapse field=group_s min=test_ti}");
- assertQ(req(params), "*[count(//doc)=1]", "//doc[./int[@name='test_ti']='5']");
+ params.add("sort", "score desc,id asc");
+ params.add("defType", "edismax");
+ params.add("bf", "field(id)");
+ assertQ(req(params), "*[count(//doc)=2]",
+ "//result/doc[1]/float[@name='id'][.='5.0']",
+ "//result/doc[2]/float[@name='id'][.='1.0']");
+
+
+
//Test collapse by max int field
params = new ModifiableSolrParams();
params.add("q", "*:*");
params.add("fq", "{!collapse field=group_s max=test_ti}");
- assertQ(req(params), "*[count(//doc)=1]", "//doc[./int[@name='test_ti']='50']");
+ params.add("sort", "test_ti asc");
+ assertQ(req(params), "*[count(//doc)=2]",
+ "//result/doc[1]/float[@name='id'][.='6.0']",
+ "//result/doc[2]/float[@name='id'][.='2.0']"
+ );
+
+
//Test collapse by min long field
params = new ModifiableSolrParams();
params.add("q", "*:*");
params.add("fq", "{!collapse field=group_s min=test_tl}");
- assertQ(req(params), "*[count(//doc)=1]", "//doc[./int[@name='test_ti']='5']");
+ params.add("sort", "test_ti desc");
+ assertQ(req(params), "*[count(//doc)=2]",
+ "//result/doc[1]/float[@name='id'][.='1.0']",
+ "//result/doc[2]/float[@name='id'][.='5.0']");
+
//Test collapse by max long field
params = new ModifiableSolrParams();
params.add("q", "*:*");
params.add("fq", "{!collapse field=group_s max=test_tl}");
- assertQ(req(params), "*[count(//doc)=1]", "//doc[./int[@name='test_ti']='50']");
+ params.add("sort", "test_ti desc");
+ assertQ(req(params), "*[count(//doc)=2]",
+ "//result/doc[1]/float[@name='id'][.='2.0']",
+ "//result/doc[2]/float[@name='id'][.='6.0']");
+
//Test collapse by min float field
params = new ModifiableSolrParams();
params.add("q", "*:*");
params.add("fq", "{!collapse field=group_s min=test_tf}");
- assertQ(req(params), "*[count(//doc)=1]", "//doc[./int[@name='test_ti']='50']");
+ params.add("sort", "test_ti desc");
+ assertQ(req(params), "*[count(//doc)=2]",
+ "//result/doc[1]/float[@name='id'][.='2.0']",
+ "//result/doc[2]/float[@name='id'][.='6.0']");
+
+
+
//Test collapse by min float field
params = new ModifiableSolrParams();
params.add("q", "*:*");
params.add("fq", "{!collapse field=group_s max=test_tf}");
- assertQ(req(params), "*[count(//doc)=1]", "//doc[./int[@name='test_ti']='5']");
+ params.add("sort", "test_ti asc");
+ assertQ(req(params), "*[count(//doc)=2]",
+ "//result/doc[1]/float[@name='id'][.='5.0']",
+ "//result/doc[2]/float[@name='id'][.='1.0']");
+
+ //Test collapse by min float field sort by score
+ params = new ModifiableSolrParams();
+ params.add("q", "*:*");
+ params.add("fq", "{!collapse field=group_s max=test_tf}");
+ params.add("defType", "edismax");
+ params.add("bf", "field(id)");
+ params.add("fl", "score, id");
+ params.add("facet","true");
+ params.add("fq", "{!tag=test}term_s:YYYY");
+ params.add("facet.field", "{!ex=test}term_s");
+
+ assertQ(req(params), "*[count(//doc)=2]",
+ "//result/doc[1]/float[@name='id'][.='5.0']",
+ "//result/doc[2]/float[@name='id'][.='1.0']");
+
//Test nullPolicy expand
params = new ModifiableSolrParams();
params.add("q", "*:*");
params.add("fq", "{!collapse field=group_s max=test_tf nullPolicy=expand}");
- assertQ(req(params), "*[count(//doc)=3]");
+ params.add("sort", "id desc");
+ assertQ(req(params), "*[count(//doc)=4]",
+ "//result/doc[1]/float[@name='id'][.='5.0']",
+ "//result/doc[2]/float[@name='id'][.='4.0']",
+ "//result/doc[3]/float[@name='id'][.='3.0']",
+ "//result/doc[4]/float[@name='id'][.='1.0']");
//Test nullPolicy collapse
+
params = new ModifiableSolrParams();
- params.add("q", "test_ti:(500 5000)");
+ params.add("q", "*:*");
params.add("fq", "{!collapse field=group_s max=test_tf nullPolicy=collapse}");
- assertQ(req(params), "*[count(//doc)=1]", "//doc[./int[@name='test_ti']='500']");
+ params.add("sort", "id desc");
+ assertQ(req(params), "*[count(//doc)=3]",
+ "//result/doc[1]/float[@name='id'][.='5.0']",
+ "//result/doc[2]/float[@name='id'][.='4.0']",
+ "//result/doc[3]/float[@name='id'][.='1.0']");
+
+
+ params = new ModifiableSolrParams();
+ params.add("q", "*:*");
+ params.add("fq", "{!collapse field=group_s}");
+ params.add("defType", "edismax");
+ params.add("bf", "field(test_ti)");
+ params.add("fq","{!tag=test_ti}id:5");
+ params.add("facet","true");
+ params.add("facet.field","{!ex=test_ti}test_ti");
+ params.add("facet.mincount", "1");
+ assertQ(req(params), "*[count(//doc)=1]", "*[count(//lst[@name='facet_fields']/lst[@name='test_ti']/int)=2]");
+
}
}