You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by is...@apache.org on 2019/05/02 21:15:32 UTC
[lucene-solr] branch master updated: SOLR-13348: Speed up
collapsing by avoiding scoring of ineligible documents
This is an automated email from the ASF dual-hosted git repository.
ishan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/master by this push:
new 29a9882 SOLR-13348: Speed up collapsing by avoiding scoring of ineligible documents
29a9882 is described below
commit 29a98825d0b70166c6c759d022f2c1426b145fe1
Author: Ishan Chattopadhyaya <is...@apache.org>
AuthorDate: Fri May 3 02:45:15 2019 +0530
SOLR-13348: Speed up collapsing by avoiding scoring of ineligible documents
---
solr/CHANGES.txt | 2 +
.../solr/search/CollapsingQParserPlugin.java | 124 ++++++++++++++++-----
2 files changed, 100 insertions(+), 26 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 3abcfa1..1c5e89b 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -285,6 +285,8 @@ Improvements
* SOLR-12833: Avoid unnecessary memory cost when DistributedUpdateProcessor timed-out lock is not used.
(jefferyyuan, ab)
+* SOLR-13348: Speed up collapsing by avoiding scoring of ineligible documents (Andrzej Wislowski via
+ Ishan Chattopadhyaya)
Other Changes
----------------------
diff --git a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java
index ccbb71f..c0e9223 100644
--- a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java
+++ b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java
@@ -216,6 +216,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
private final SortSpec sortSpec; // may be null, parsed at most once from groupHeadSelector
public String hint;
private boolean needsScores = true;
+ private boolean needsScores4Collapsing = false;
private int nullPolicy;
private Set<BytesRef> boosted; // ordered by "priority"
public static final int NULL_POLICY_IGNORE = 0;
@@ -319,15 +320,15 @@ public class CollapsingQParserPlugin extends QParserPlugin {
// may be null in some esoteric corner usages
final ResponseBuilder rb = info.getResponseBuilder();
final SortSpec topSort = null == rb ? null : rb.getSortSpec();
-
- this.needsScores =
+
+ this.needsScores4Collapsing = GroupHeadSelectorType.SCORE.equals(groupHeadSelector.type) ||
+ (GroupHeadSelectorType.SORT.equals(groupHeadSelector.type)
+ && this.sortSpec.includesScore()) ||
+ (GroupHeadSelectorType.MIN_MAX.contains(groupHeadSelector.type)
+ && CollapseScore.wantsCScore(groupHeadSelector.selectorText));
+ this.needsScores = needsScores4Collapsing ||
(info.getRsp().getReturnFields().wantsScore() ||
(null != topSort && topSort.includesScore()) ||
- GroupHeadSelectorType.SCORE.equals(groupHeadSelector.type) ||
- (GroupHeadSelectorType.SORT.equals(groupHeadSelector.type)
- && this.sortSpec.includesScore()) ||
- (GroupHeadSelectorType.MIN_MAX.contains(groupHeadSelector.type)
- && CollapseScore.wantsCScore(groupHeadSelector.selectorText)) ||
(this.boosted != null));
if (this.needsScores && null != rb) {
@@ -378,6 +379,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
this.sortSpec,
this.nullPolicy,
this.hint,
+ this.needsScores4Collapsing,
this.needsScores,
this.size,
boostDocsMap,
@@ -930,6 +932,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
private int nullPolicy;
private OrdFieldValueStrategy collapseStrategy;
+ private boolean needsScores4Collapsing;
private boolean needsScores;
public OrdFieldValueCollector(int maxDoc,
@@ -938,6 +941,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
int nullPolicy,
GroupHeadSelector groupHeadSelector,
SortSpec sortSpec,
+ boolean needsScores4Collapsing,
boolean needsScores,
FieldType fieldType,
IntIntHashMap boostDocs,
@@ -956,11 +960,12 @@ public class CollapsingQParserPlugin extends QParserPlugin {
int valueCount = collapseValues.getValueCount();
this.nullPolicy = nullPolicy;
+ this.needsScores4Collapsing = needsScores4Collapsing;
this.needsScores = needsScores;
if (null != sortSpec) {
- this.collapseStrategy = new OrdSortSpecStrategy(maxDoc, nullPolicy, new int[valueCount], groupHeadSelector, this.needsScores, boostDocs, sortSpec, searcher, collapseValues);
+ this.collapseStrategy = new OrdSortSpecStrategy(maxDoc, nullPolicy, new int[valueCount], groupHeadSelector, this.needsScores4Collapsing, this.needsScores, boostDocs, sortSpec, searcher, collapseValues);
} else if (funcQuery != null) {
- this.collapseStrategy = new OrdValueSourceStrategy(maxDoc, nullPolicy, new int[valueCount], groupHeadSelector, this.needsScores, boostDocs, funcQuery, searcher, collapseValues);
+ this.collapseStrategy = new OrdValueSourceStrategy(maxDoc, nullPolicy, new int[valueCount], groupHeadSelector, this.needsScores4Collapsing, this.needsScores, boostDocs, funcQuery, searcher, collapseValues);
} else {
NumberType numType = fieldType.getNumberType();
if (null == numType) {
@@ -1116,6 +1121,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
private int nullPolicy;
private IntFieldValueStrategy collapseStrategy;
+ private boolean needsScores4Collapsing;
private boolean needsScores;
private String collapseField;
@@ -1127,6 +1133,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
String collapseField,
GroupHeadSelector groupHeadSelector,
SortSpec sortSpec,
+ boolean needsScores4Collapsing,
boolean needsScores,
FieldType fieldType,
IntIntHashMap boostDocsMap,
@@ -1140,11 +1147,12 @@ public class CollapsingQParserPlugin extends QParserPlugin {
this.collapseField = collapseField;
this.nullValue = nullValue;
this.nullPolicy = nullPolicy;
+ this.needsScores4Collapsing = needsScores4Collapsing;
this.needsScores = needsScores;
if (null != sortSpec) {
- this.collapseStrategy = new IntSortSpecStrategy(maxDoc, size, collapseField, nullValue, nullPolicy, groupHeadSelector, this.needsScores, boostDocsMap, sortSpec, searcher);
+ this.collapseStrategy = new IntSortSpecStrategy(maxDoc, size, collapseField, nullValue, nullPolicy, groupHeadSelector, this.needsScores4Collapsing, this.needsScores, boostDocsMap, sortSpec, searcher);
} else if (funcQuery != null) {
- this.collapseStrategy = new IntValueSourceStrategy(maxDoc, size, collapseField, nullValue, nullPolicy, groupHeadSelector, this.needsScores, boostDocsMap, funcQuery, searcher);
+ this.collapseStrategy = new IntValueSourceStrategy(maxDoc, size, collapseField, nullValue, nullPolicy, groupHeadSelector, this.needsScores4Collapsing, this.needsScores, boostDocsMap, funcQuery, searcher);
} else {
NumberType numType = fieldType.getNumberType();
assert null != numType; // shouldn't make it here for non-numeric types
@@ -1269,6 +1277,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
SortSpec sortSpec,
int nullPolicy,
String hint,
+ boolean needsScores4Collapsing,
boolean needsScores,
int size,
IntIntHashMap boostDocs,
@@ -1367,6 +1376,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
nullPolicy,
groupHeadSelector,
sortSpec,
+ needsScores4Collapsing,
needsScores,
minMaxFieldType,
boostDocs,
@@ -1398,6 +1408,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
collapseField,
groupHeadSelector,
sortSpec,
+ needsScores4Collapsing,
needsScores,
minMaxFieldType,
boostDocs,
@@ -1823,18 +1834,20 @@ public class CollapsingQParserPlugin extends QParserPlugin {
private float[] ordVals;
private Map rcontext;
private final CollapseScore collapseScore = new CollapseScore();
- private float score;
+ private boolean needsScores4Collapsing;
public OrdValueSourceStrategy(int maxDoc,
int nullPolicy,
int[] ords,
GroupHeadSelector groupHeadSelector,
+ boolean needsScores4Collapsing,
boolean needsScores,
IntIntHashMap boostDocs,
FunctionQuery funcQuery,
IndexSearcher searcher,
SortedDocValues values) throws IOException {
super(maxDoc, ords, nullPolicy, needsScores, boostDocs, values);
+ this.needsScores4Collapsing = needsScores4Collapsing;
this.valueSource = funcQuery.getValueSource();
this.rcontext = ValueSource.newContext(searcher);
this.ordVals = new float[ords.length];
@@ -1865,14 +1878,15 @@ public class CollapsingQParserPlugin extends QParserPlugin {
}
public void collapse(int ord, int contextDoc, int globalDoc) throws IOException {
+ float score = 0;
if(boosted && mergeBoost.boost(globalDoc)) {
this.boostOrds.add(ord);
this.boostDocs.add(globalDoc);
}
- if (needsScores) {
- this.score = scorer.score();
+ if (needsScores4Collapsing) {
+ score = scorer.score();
this.collapseScore.score = score;
}
@@ -1883,6 +1897,9 @@ public class CollapsingQParserPlugin extends QParserPlugin {
ords[ord] = globalDoc;
ordVals[ord] = currentVal;
if(needsScores) {
+ if (!needsScores4Collapsing) {
+ score = scorer.score();
+ }
scores[ord] = score;
}
}
@@ -1891,12 +1908,18 @@ public class CollapsingQParserPlugin extends QParserPlugin {
nullVal = currentVal;
nullDoc = globalDoc;
if(needsScores) {
+ if (!needsScores4Collapsing) {
+ score = scorer.score();
+ }
nullScore = score;
}
}
} else if(this.nullPolicy == CollapsingPostFilter.NULL_POLICY_EXPAND) {
this.collapsedSet.set(globalDoc);
if(needsScores) {
+ if (!needsScores4Collapsing) {
+ score = scorer.score();
+ }
nullScores.add(score);
}
}
@@ -1910,25 +1933,26 @@ public class CollapsingQParserPlugin extends QParserPlugin {
private static class OrdSortSpecStrategy extends OrdFieldValueStrategy {
private final SortFieldsCompare compareState;
- private final SortSpec sortSpec;
private final Sort sort;
private float score;
+ private boolean needsScores4Collapsing;
public OrdSortSpecStrategy(int maxDoc,
int nullPolicy,
int[] ords,
GroupHeadSelector groupHeadSelector,
+ boolean needsScores4Collapsing,
boolean needsScores,
IntIntHashMap boostDocs,
SortSpec sortSpec,
IndexSearcher searcher,
SortedDocValues values) throws IOException {
super(maxDoc, ords, nullPolicy, needsScores, boostDocs, values);
+ this.needsScores4Collapsing = needsScores4Collapsing;
assert GroupHeadSelectorType.SORT.equals(groupHeadSelector.type);
- this.sortSpec = sortSpec;
this.sort = rewriteSort(sortSpec, searcher);
this.compareState = new SortFieldsCompare(sort.getSort(), ords.length);
@@ -1960,7 +1984,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
this.boostDocs.add(globalDoc);
}
- if (needsScores) {
+ if (needsScores4Collapsing) {
this.score = scorer.score();
}
@@ -1970,13 +1994,19 @@ public class CollapsingQParserPlugin extends QParserPlugin {
compareState.setGroupValues(ord, contextDoc);
ords[ord] = globalDoc;
if (needsScores) {
+ if (!needsScores4Collapsing) {
+ this.score = scorer.score();
+ }
scores[ord] = score;
}
} else {
// test this ord to see if it's a new group leader
- if (compareState.testAndSetGroupValues(ord, contextDoc)) {
+ if (compareState.testAndSetGroupValues(ord, contextDoc)) {//TODO X
ords[ord] = globalDoc;
if (needsScores) {
+ if (!needsScores4Collapsing) {
+ this.score = scorer.score();
+ }
scores[ord] = score;
}
}
@@ -1987,6 +2017,9 @@ public class CollapsingQParserPlugin extends QParserPlugin {
compareState.setNullGroupValues(contextDoc);
nullDoc = globalDoc;
if (needsScores) {
+ if (!needsScores4Collapsing) {
+ this.score = scorer.score();
+ }
nullScore = score;
}
} else {
@@ -1994,6 +2027,9 @@ public class CollapsingQParserPlugin extends QParserPlugin {
if (compareState.testAndSetNullGroupValues(contextDoc)) {
nullDoc = globalDoc;
if (needsScores) {
+ if (!needsScores4Collapsing) {
+ this.score = scorer.score();
+ }
nullScore = score;
}
}
@@ -2001,6 +2037,9 @@ public class CollapsingQParserPlugin extends QParserPlugin {
} else if(this.nullPolicy == CollapsingPostFilter.NULL_POLICY_EXPAND) {
this.collapsedSet.set(globalDoc);
if (needsScores) {
+ if (!needsScores4Collapsing) {
+ this.score = scorer.score();
+ }
nullScores.add(score);
}
}
@@ -2355,8 +2394,8 @@ public class CollapsingQParserPlugin extends QParserPlugin {
private FunctionValues functionValues;
private Map rcontext;
private final CollapseScore collapseScore = new CollapseScore();
- private float score;
private int index=-1;
+ private boolean needsScores4Collapsing;
public IntValueSourceStrategy(int maxDoc,
int size,
@@ -2364,6 +2403,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
int nullValue,
int nullPolicy,
GroupHeadSelector groupHeadSelector,
+ boolean needsScores4Collapsing,
boolean needsScores,
IntIntHashMap boostDocs,
FunctionQuery funcQuery,
@@ -2371,6 +2411,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
super(maxDoc, size, collapseField, nullValue, nullPolicy, needsScores, boostDocs);
+ this.needsScores4Collapsing = needsScores4Collapsing;
this.testValues = new float[size];
this.valueSource = funcQuery.getValueSource();
@@ -2401,6 +2442,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
}
public void collapse(int collapseKey, int contextDoc, int globalDoc) throws IOException {
+ float score = 0;
// Check to see if we have documents boosted by the QueryElevationComponent
if(boosts && mergeBoost.boost(globalDoc)) {
@@ -2409,8 +2451,8 @@ public class CollapsingQParserPlugin extends QParserPlugin {
return;
}
- if (needsScores) {
- this.score = scorer.score();
+ if (needsScores4Collapsing) {
+ score = scorer.score();
this.collapseScore.score = score;
}
@@ -2424,6 +2466,9 @@ public class CollapsingQParserPlugin extends QParserPlugin {
testValues[pointer] = currentVal;
docs[pointer] = globalDoc;
if(needsScores){
+ if (!needsScores4Collapsing) {
+ score = scorer.score();
+ }
scores[pointer] = score;
}
}
@@ -2440,6 +2485,9 @@ public class CollapsingQParserPlugin extends QParserPlugin {
docs[index] = globalDoc;
testValues[index] = currentVal;
if(needsScores) {
+ if (!needsScores4Collapsing) {
+ score = scorer.score();
+ }
scores[index] = score;
}
}
@@ -2448,13 +2496,19 @@ public class CollapsingQParserPlugin extends QParserPlugin {
nullCompVal = currentVal;
nullDoc = globalDoc;
if(needsScores) {
- nullScore = scorer.score();
+ if (!needsScores4Collapsing) {
+ score = scorer.score();
+ }
+ nullScore = score;
}
}
} else if(this.nullPolicy == CollapsingPostFilter.NULL_POLICY_EXPAND) {
this.collapsedSet.set(globalDoc);
if(needsScores) {
- nullScores.add(scorer.score());
+ if (!needsScores4Collapsing) {
+ score = scorer.score();
+ }
+ nullScores.add(score);
}
}
}
@@ -2471,8 +2525,8 @@ public class CollapsingQParserPlugin extends QParserPlugin {
private final SortSpec sortSpec;
private final Sort sort;
- private float score;
private int index=-1;
+ private boolean needsScores4Collapsing;
public IntSortSpecStrategy(int maxDoc,
int size,
@@ -2480,12 +2534,14 @@ public class CollapsingQParserPlugin extends QParserPlugin {
int nullValue,
int nullPolicy,
GroupHeadSelector groupHeadSelector,
+ boolean needsScores4Collapsing,
boolean needsScores,
IntIntHashMap boostDocs,
SortSpec sortSpec,
IndexSearcher searcher) throws IOException {
super(maxDoc, size, collapseField, nullValue, nullPolicy, needsScores, boostDocs);
+ this.needsScores4Collapsing = needsScores4Collapsing;
assert GroupHeadSelectorType.SORT.equals(groupHeadSelector.type);
@@ -2513,6 +2569,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
}
public void collapse(int collapseKey, int contextDoc, int globalDoc) throws IOException {
+ float score = 0;
// Check to see if we have documents boosted by the QueryElevationComponent
if(boosts && mergeBoost.boost(globalDoc)) {
@@ -2521,8 +2578,8 @@ public class CollapsingQParserPlugin extends QParserPlugin {
return;
}
- if (needsScores) {
- this.score = scorer.score();
+ if (needsScores4Collapsing) {
+ score = scorer.score();
}
if (collapseKey != nullValue) {
@@ -2533,6 +2590,9 @@ public class CollapsingQParserPlugin extends QParserPlugin {
if (compareState.testAndSetGroupValues(pointer, contextDoc)) {
docs[pointer] = globalDoc;
if (needsScores) {
+ if (!needsScores4Collapsing) {
+ score = scorer.score();
+ }
scores[pointer] = score;
}
}
@@ -2550,6 +2610,9 @@ public class CollapsingQParserPlugin extends QParserPlugin {
docs[index] = globalDoc;
compareState.setGroupValues(index, contextDoc);
if(needsScores) {
+ if (!needsScores4Collapsing) {
+ score = scorer.score();
+ }
scores[index] = score;
}
}
@@ -2559,6 +2622,9 @@ public class CollapsingQParserPlugin extends QParserPlugin {
compareState.setNullGroupValues(contextDoc);
nullDoc = globalDoc;
if (needsScores) {
+ if (!needsScores4Collapsing) {
+ score = scorer.score();
+ }
nullScore = score;
}
} else {
@@ -2566,6 +2632,9 @@ public class CollapsingQParserPlugin extends QParserPlugin {
if (compareState.testAndSetNullGroupValues(contextDoc)) {
nullDoc = globalDoc;
if (needsScores) {
+ if (!needsScores4Collapsing) {
+ score = scorer.score();
+ }
nullScore = score;
}
}
@@ -2573,6 +2642,9 @@ public class CollapsingQParserPlugin extends QParserPlugin {
} else if(this.nullPolicy == CollapsingPostFilter.NULL_POLICY_EXPAND) {
this.collapsedSet.set(globalDoc);
if (needsScores) {
+ if (!needsScores4Collapsing) {
+ score = scorer.score();
+ }
nullScores.add(score);
}
}