You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by da...@apache.org on 2020/04/10 08:24:20 UTC
[lucene-solr] branch master updated: SOLR-14365: CollapsingQParser
- Avoiding always allocate int[] and float[] with size equals to number of
unique values (WIP) (#1395)
This is an automated email from the ASF dual-hosted git repository.
datcm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/master by this push:
new adbd714 SOLR-14365: CollapsingQParser - Avoiding always allocate int[] and float[] with size equals to number of unique values (WIP) (#1395)
adbd714 is described below
commit adbd714b37d794e9aa7615e61c431e42162c1d3c
Author: Cao Manh Dat <da...@apache.org>
AuthorDate: Fri Apr 10 15:24:10 2020 +0700
SOLR-14365: CollapsingQParser - Avoiding always allocate int[] and float[] with size equals to number of unique values (WIP) (#1395)
---
solr/CHANGES.txt | 3 +
.../solr/search/CollapsingQParserPlugin.java | 469 ++++++++-------------
.../src/java/org/apache/solr/util/DynamicMap.java | 55 +++
.../java/org/apache/solr/util/FloatConsumer.java | 39 ++
.../org/apache/solr/util/IntFloatDynamicMap.java | 118 ++++++
.../org/apache/solr/util/IntIntDynamicMap.java | 120 ++++++
.../org/apache/solr/util/IntLongDynamicMap.java | 120 ++++++
.../test/org/apache/solr/util/DynamicMapsTest.java | 90 ++++
8 files changed, 732 insertions(+), 282 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index cb21f71..9dda7df 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -89,6 +89,9 @@ Optimizations
* SOLR-14340: Remove unnecessary configset verification checks when SolrCloud merely wants to know which configset a
collection is using. Improves CLUSTERSTATUS times for massive clusters. (Mathieu Marie, David Smiley)
+* SOLR-14365: CollapsingQParser - Avoiding always allocate int[] and float[] with size equals to number of unique values.
+ (Cao Manh Dat)
+
* SOLR-14376: Optimize filter queries that match all docs. (David Smiley)
Bug Fixes
diff --git a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java
index 690f806..61e4bed 100644
--- a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java
+++ b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java
@@ -78,6 +78,9 @@ import org.apache.solr.schema.NumberType;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.schema.StrField;
import org.apache.solr.uninverting.UninvertingReader;
+import org.apache.solr.util.IntFloatDynamicMap;
+import org.apache.solr.util.IntIntDynamicMap;
+import org.apache.solr.util.IntLongDynamicMap;
import static org.apache.solr.common.params.CommonParams.SORT;
@@ -149,15 +152,15 @@ public class CollapsingQParserPlugin extends QParserPlugin {
MIN, MAX, SORT, SCORE;
public static EnumSet<GroupHeadSelectorType> MIN_MAX = EnumSet.of(MIN, MAX);
}
-
+
/**
* Models all the information about how group head documents should be selected
*/
public static final class GroupHeadSelector {
-
- /**
- * The param value for this selector whose meaning depends on type.
- * (ie: a field or valuesource for MIN/MAX, a sort string for SORT, "score" for SCORE).
+
+ /**
+ * The param value for this selector whose meaning depends on type.
+ * (ie: a field or valuesource for MIN/MAX, a sort string for SORT, "score" for SCORE).
* Will never be null.
*/
public final String selectorText;
@@ -166,7 +169,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
private GroupHeadSelector(String s, GroupHeadSelectorType type) {
assert null != s;
assert null != type;
-
+
this.selectorText = s;
this.type = type;
}
@@ -179,12 +182,12 @@ public class CollapsingQParserPlugin extends QParserPlugin {
}
return false;
}
-
+
@Override
public int hashCode() {
return 17 * (31 + selectorText.hashCode()) * (31 + type.hashCode());
}
-
+
/**
* returns a new GroupHeadSelector based on the specified local params
*/
@@ -197,7 +200,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
"At most one localparam for selecting documents (min, max, sort) may be specified: " + localParams.toString());
}
-
+
if (null != sortString) {
return new GroupHeadSelector(sortString, GroupHeadSelectorType.SORT);
} else if (null != min) {
@@ -302,7 +305,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
}
this.groupHeadSelector = GroupHeadSelector.build(localParams);
-
+
if (groupHeadSelector.type.equals(GroupHeadSelectorType.SORT) &&
CollapseScore.wantsCScore(groupHeadSelector.selectorText)) {
// we can't support Sorts that wrap functions that include "cscore()" because
@@ -314,9 +317,9 @@ public class CollapsingQParserPlugin extends QParserPlugin {
"param of the collapse parser is not supported");
}
-
-
- this.sortSpec = GroupHeadSelectorType.SORT.equals(groupHeadSelector.type)
+
+
+ this.sortSpec = GroupHeadSelectorType.SORT.equals(groupHeadSelector.type)
? SortSpecParsing.parseSortSpec(groupHeadSelector.selectorText, request)
: null;
@@ -326,7 +329,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
{
final SolrRequestInfo info = SolrRequestInfo.getRequestInfo();
assert null != info;
-
+
// may be null in some esoteric corner usages
final ResponseBuilder rb = info.getResponseBuilder();
final SortSpec topSort = null == rb ? null : rb.getSortSpec();
@@ -346,11 +349,11 @@ public class CollapsingQParserPlugin extends QParserPlugin {
// for the "real" docs. (ie: maybe we need them because we were
// asked to compute them for the collapsed docs, maybe we need them because in
// order to find the groupHead we need them computed for us.
-
+
rb.setFieldFlags( rb.getFieldFlags() | SolrIndexSearcher.GET_SCORES);
}
}
-
+
String nPolicy = localParams.get("nullPolicy", NULL_IGNORE);
if(nPolicy.equals(NULL_IGNORE)) {
this.nullPolicy = NULL_POLICY_IGNORE;
@@ -411,12 +414,12 @@ public class CollapsingQParserPlugin extends QParserPlugin {
final SchemaField f = searcher.getSchema().getFieldOrNull(collapseField);
assert null != f; // should already be enforced higher up
assert !f.multiValued(); // should already be enforced higher up
-
+
assert f.getType() instanceof StrField; // this method shouldn't be called otherwise
if (f.indexed() && f.isUninvertible()) {
type = UninvertingReader.Type.SORTED;
}
-
+
return UninvertingReader.wrap(
new ReaderWrapper(searcher.getSlowAtomicReader(), collapseField),
Collections.singletonMap(collapseField, type)::get);
@@ -510,8 +513,8 @@ public class CollapsingQParserPlugin extends QParserPlugin {
private SortedDocValues segmentValues;
private LongValues segmentOrdinalMap;
private MultiDocValues.MultiSortedDocValues multiSortedDocValues;
- private int[] ords;
- private float[] scores;
+ private IntIntDynamicMap ords;
+ private IntFloatDynamicMap scores;
private int maxDoc;
private int nullPolicy;
private float nullScore = -Float.MAX_VALUE;
@@ -538,16 +541,14 @@ public class CollapsingQParserPlugin extends QParserPlugin {
this.collapsedSet = new FixedBitSet(maxDoc);
this.collapseValuesProducer = collapseValuesProducer;
this.collapseValues = collapseValuesProducer.getSorted(null);
-
+
int valueCount = collapseValues.getValueCount();
if(collapseValues instanceof MultiDocValues.MultiSortedDocValues) {
this.multiSortedDocValues = (MultiDocValues.MultiSortedDocValues)collapseValues;
this.ordinalMap = multiSortedDocValues.mapping;
}
- this.ords = new int[valueCount];
- Arrays.fill(this.ords, -1);
- this.scores = new float[valueCount];
- Arrays.fill(this.scores, -Float.MAX_VALUE);
+ this.ords = new IntIntDynamicMap(valueCount, -1);
+ this.scores = new IntFloatDynamicMap(valueCount, -Float.MAX_VALUE);
this.nullPolicy = nullPolicy;
if(nullPolicy == CollapsingPostFilter.NULL_POLICY_EXPAND) {
nullScores = new FloatArrayList();
@@ -613,9 +614,9 @@ public class CollapsingQParserPlugin extends QParserPlugin {
if(ord > -1) {
float score = scorer.score();
- if(score > scores[ord]) {
- ords[ord] = globalDoc;
- scores[ord] = score;
+ if(score > scores.get(ord)) {
+ ords.put(ord, globalDoc);
+ scores.put(ord, score);
}
} else if(nullPolicy == CollapsingPostFilter.NULL_POLICY_COLLAPSE) {
float score = scorer.score();
@@ -646,7 +647,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
int ord = this.boostOrds.get(i);
if(ord > -1) {
//Remove any group heads that are in the same groups as boosted documents.
- ords[ord] = -1;
+ ords.remove(ord);
}
//Add the boosted docs to the collapsedSet
this.collapsedSet.set(boostDocs.get(i));
@@ -655,18 +656,13 @@ public class CollapsingQParserPlugin extends QParserPlugin {
}
//Build the sorted DocSet of group heads.
- for(int i=0; i<ords.length; i++) {
- int doc = ords[i];
- if(doc > -1) {
- collapsedSet.set(doc);
- }
- }
+ ords.forEachValue(doc -> collapsedSet.set(doc));
int currentContext = 0;
int currentDocBase = 0;
collapseValues = collapseValuesProducer.getSorted(null);
-
+
if(collapseValues instanceof MultiDocValues.MultiSortedDocValues) {
this.multiSortedDocValues = (MultiDocValues.MultiSortedDocValues)collapseValues;
this.ordinalMap = multiSortedDocValues.mapping;
@@ -715,7 +711,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
}
if(ord > -1) {
- dummy.score = scores[ord];
+ dummy.score = scores.get(ord);
} else if(boosts && mergeBoost.boost(docId)) {
//Ignore so it doesn't mess up the null scoring.
} else if(this.nullPolicy == CollapsingPostFilter.NULL_POLICY_COLLAPSE) {
@@ -934,7 +930,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
}
}
}
-
+
/**
* Collapse on Ordinal value field.
*/
@@ -968,7 +964,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
FunctionQuery funcQuery, IndexSearcher searcher) throws IOException{
assert ! GroupHeadSelectorType.SCORE.equals(groupHeadSelector.type);
-
+
this.maxDoc = maxDoc;
this.contexts = new LeafReaderContext[segments];
List<LeafReaderContext> con = searcher.getTopReaderContext().leaves();
@@ -987,9 +983,9 @@ public class CollapsingQParserPlugin extends QParserPlugin {
this.needsScores4Collapsing = needsScores4Collapsing;
this.needsScores = needsScores;
if (null != sortSpec) {
- this.collapseStrategy = new OrdSortSpecStrategy(maxDoc, nullPolicy, new int[valueCount], groupHeadSelector, this.needsScores4Collapsing, this.needsScores, boostDocs, sortSpec, searcher, collapseValues);
+ this.collapseStrategy = new OrdSortSpecStrategy(maxDoc, nullPolicy, valueCount, groupHeadSelector, this.needsScores4Collapsing, this.needsScores, boostDocs, sortSpec, searcher, collapseValues);
} else if (funcQuery != null) {
- this.collapseStrategy = new OrdValueSourceStrategy(maxDoc, nullPolicy, new int[valueCount], groupHeadSelector, this.needsScores4Collapsing, this.needsScores, boostDocs, funcQuery, searcher, collapseValues);
+ this.collapseStrategy = new OrdValueSourceStrategy(maxDoc, nullPolicy, valueCount, groupHeadSelector, this.needsScores4Collapsing, this.needsScores, boostDocs, funcQuery, searcher, collapseValues);
} else {
NumberType numType = fieldType.getNumberType();
if (null == numType) {
@@ -997,15 +993,15 @@ public class CollapsingQParserPlugin extends QParserPlugin {
}
switch (numType) {
case INTEGER: {
- this.collapseStrategy = new OrdIntStrategy(maxDoc, nullPolicy, new int[valueCount], groupHeadSelector, this.needsScores, boostDocs, collapseValues);
+ this.collapseStrategy = new OrdIntStrategy(maxDoc, nullPolicy, valueCount, groupHeadSelector, this.needsScores, boostDocs, collapseValues);
break;
}
case FLOAT: {
- this.collapseStrategy = new OrdFloatStrategy(maxDoc, nullPolicy, new int[valueCount], groupHeadSelector, this.needsScores, boostDocs, collapseValues);
+ this.collapseStrategy = new OrdFloatStrategy(maxDoc, nullPolicy, valueCount, groupHeadSelector, this.needsScores, boostDocs, collapseValues);
break;
}
case LONG: {
- this.collapseStrategy = new OrdLongStrategy(maxDoc, nullPolicy, new int[valueCount], groupHeadSelector, this.needsScores, boostDocs, collapseValues);
+ this.collapseStrategy = new OrdLongStrategy(maxDoc, nullPolicy, valueCount, groupHeadSelector, this.needsScores, boostDocs, collapseValues);
break;
}
default: {
@@ -1075,7 +1071,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
DocIdSetIterator it = new BitSetIterator(collapseStrategy.getCollapsedSet(), 0); // cost is not useful here
int globalDoc = -1;
int nullScoreIndex = 0;
- float[] scores = collapseStrategy.getScores();
+ IntFloatDynamicMap scores = collapseStrategy.getScores();
FloatArrayList nullScores = collapseStrategy.getNullScores();
float nullScore = collapseStrategy.getNullScore();
@@ -1112,7 +1108,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
}
if(ord > -1) {
- dummy.score = scores[ord];
+ dummy.score = scores.get(ord);
} else if (mergeBoost != null && mergeBoost.boost(globalDoc)) {
//It's an elevated doc so no score is needed
dummy.score = 0F;
@@ -1165,7 +1161,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
IndexSearcher searcher) throws IOException{
assert ! GroupHeadSelectorType.SCORE.equals(groupHeadSelector.type);
-
+
this.maxDoc = maxDoc;
this.contexts = new LeafReaderContext[segments];
List<LeafReaderContext> con = searcher.getTopReaderContext().leaves();
@@ -1222,7 +1218,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
} else {
collapseKey = 0;
}
-
+
int globalDoc = contextDoc+this.docBase;
collapseStrategy.collapse(collapseKey, contextDoc, globalDoc);
}
@@ -1243,8 +1239,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
int globalDoc = -1;
int nullScoreIndex = 0;
IntIntHashMap cmap = collapseStrategy.getCollapseMap();
- int[] docs = collapseStrategy.getDocs();
- float[] scores = collapseStrategy.getScores();
+ IntFloatDynamicMap scores = collapseStrategy.getScores();
FloatArrayList nullScores = collapseStrategy.getNullScores();
MergeBoost mergeBoost = collapseStrategy.getMergeBoost();
float nullScore = collapseStrategy.getNullScore();
@@ -1272,7 +1267,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
if(collapseValue != nullValue) {
int pointer = cmap.get(collapseValue);
- dummy.score = scores[pointer];
+ dummy.score = scores.get(pointer);
} else if (mergeBoost != null && mergeBoost.boost(globalDoc)) {
//Its an elevated doc so no score is needed
dummy.score = 0F;
@@ -1364,7 +1359,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
int leafCount = searcher.getTopReaderContext().leaves().size();
if (GroupHeadSelectorType.SCORE.equals(groupHeadSelector.type)) {
-
+
if (collapseFieldType instanceof StrField) {
return new OrdScoreCollector(maxDoc, leafCount, docValuesProducer, nullPolicy, boostDocs, searcher);
@@ -1392,7 +1387,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
"Collapsing field should be of either String, Int or Float type");
}
-
+
} else { // min, max, sort, etc.. something other then just "score"
if (collapseFieldType instanceof StrField) {
@@ -1445,15 +1440,15 @@ public class CollapsingQParserPlugin extends QParserPlugin {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
"Collapsing field should be of either String, Int or Float type");
}
-
+
}
}
}
public static final class CollapseScore {
- /**
+ /**
* Inspects the GroupHeadSelector to determine if this CollapseScore is needed.
- * If it is, then "this" will be added to the readerContext
+ * If it is, then "this" will be added to the readerContext
* using the "CSCORE" key, and true will be returned. If not returns false.
*/
public boolean setupIfNeeded(final GroupHeadSelector groupHeadSelector,
@@ -1467,18 +1462,18 @@ public class CollapsingQParserPlugin extends QParserPlugin {
return false;
}
- /**
- * Huge HACK, but not really any better options until/unless we can recursively
- * ask value sources if they depend on score
+ /**
+ * Huge HACK, but not really any better options until/unless we can recursively
+ * ask value sources if they depend on score
*/
public static boolean wantsCScore(final String text) {
return (0 <= text.indexOf("cscore()"));
}
-
+
private CollapseScore() {
// No-Op
}
-
+
public float score;
}
@@ -1494,11 +1489,11 @@ public class CollapsingQParserPlugin extends QParserPlugin {
*/
private static abstract class OrdFieldValueStrategy {
protected int nullPolicy;
- protected int[] ords;
+ protected IntIntDynamicMap ords;
protected Scorable scorer;
protected FloatArrayList nullScores;
protected float nullScore;
- protected float[] scores;
+ protected IntFloatDynamicMap scores;
protected FixedBitSet collapsedSet;
protected int nullDoc = -1;
protected boolean needsScores;
@@ -1512,13 +1507,12 @@ public class CollapsingQParserPlugin extends QParserPlugin {
public abstract void setNextReader(LeafReaderContext context) throws IOException;
public OrdFieldValueStrategy(int maxDoc,
- int[] ords,
+ int valueCount,
int nullPolicy,
boolean needsScores,
IntIntHashMap boostDocsMap,
SortedDocValues values) {
- this.ords = ords;
- Arrays.fill(ords, -1);
+ this.ords = new IntIntDynamicMap(valueCount, -1);
this.nullPolicy = nullPolicy;
this.needsScores = needsScores;
this.collapsedSet = new FixedBitSet(maxDoc);
@@ -1538,6 +1532,13 @@ public class CollapsingQParserPlugin extends QParserPlugin {
this.mergeBoost = new MergeBoost(bd);
this.boosted = true;
}
+
+ if (this.needsScores) {
+ this.scores = new IntFloatDynamicMap(valueCount, 0.0f);
+ if(nullPolicy == CollapsingPostFilter.NULL_POLICY_EXPAND) {
+ nullScores = new FloatArrayList();
+ }
+ }
}
public MergeBoost getMergeBoost() {
@@ -1554,7 +1555,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
for(int i=0; i<s; i++) {
int ord = boostOrds.get(i);
if(ord > -1) {
- ords[ord] = -1;
+ ords.remove(ord);
}
collapsedSet.set(boostDocs.get(i));
}
@@ -1562,12 +1563,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
mergeBoost.reset();
}
- for(int i=0; i<ords.length; i++) {
- int doc = ords[i];
- if(doc > -1) {
- collapsedSet.set(doc);
- }
- }
+ ords.forEachValue(doc -> collapsedSet.set(doc));
return collapsedSet;
}
@@ -1584,7 +1580,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
return this.nullScore;
}
- public float[] getScores() {
+ public IntFloatDynamicMap getScores() {
return scores;
}
}
@@ -1598,36 +1594,28 @@ public class CollapsingQParserPlugin extends QParserPlugin {
private NumericDocValues minMaxValues;
private IntCompare comp;
private int nullVal;
- private int[] ordVals;
+ private IntIntDynamicMap ordVals;
public OrdIntStrategy(int maxDoc,
int nullPolicy,
- int[] ords,
+ int valueCount,
GroupHeadSelector groupHeadSelector,
boolean needsScores,
IntIntHashMap boostDocs,
SortedDocValues values) throws IOException {
- super(maxDoc, ords, nullPolicy, needsScores, boostDocs, values);
+ super(maxDoc, valueCount, nullPolicy, needsScores, boostDocs, values);
this.field = groupHeadSelector.selectorText;
- this.ordVals = new int[ords.length];
assert GroupHeadSelectorType.MIN_MAX.contains(groupHeadSelector.type);
-
+
if (GroupHeadSelectorType.MAX.equals(groupHeadSelector.type)) {
comp = new MaxIntComp();
- Arrays.fill(ordVals, Integer.MIN_VALUE);
+ this.ordVals = new IntIntDynamicMap(valueCount, Integer.MIN_VALUE);
} else {
comp = new MinIntComp();
- Arrays.fill(ordVals, Integer.MAX_VALUE);
+ this.ordVals = new IntIntDynamicMap(valueCount, Integer.MAX_VALUE);
this.nullVal = Integer.MAX_VALUE;
}
-
- if(needsScores) {
- this.scores = new float[ords.length];
- if(nullPolicy == CollapsingPostFilter.NULL_POLICY_EXPAND) {
- nullScores = new FloatArrayList();
- }
- }
}
public void setNextReader(LeafReaderContext context) throws IOException {
@@ -1648,13 +1636,13 @@ public class CollapsingQParserPlugin extends QParserPlugin {
} else {
currentVal = 0;
}
-
+
if(ord > -1) {
- if(comp.test(currentVal, ordVals[ord])) {
- ords[ord] = globalDoc;
- ordVals[ord] = currentVal;
+ if(comp.test(currentVal, ordVals.get(ord))) {
+ ords.put(ord, globalDoc);
+ ordVals.put(ord, currentVal);
if(needsScores) {
- scores[ord] = scorer.score();
+ scores.put(ord, scorer.score());
}
}
} else if(this.nullPolicy == CollapsingPostFilter.NULL_POLICY_COLLAPSE) {
@@ -1684,37 +1672,29 @@ public class CollapsingQParserPlugin extends QParserPlugin {
private NumericDocValues minMaxValues;
private FloatCompare comp;
private float nullVal;
- private float[] ordVals;
+ private IntFloatDynamicMap ordVals;
public OrdFloatStrategy(int maxDoc,
int nullPolicy,
- int[] ords,
+ int valueCount,
GroupHeadSelector groupHeadSelector,
boolean needsScores,
IntIntHashMap boostDocs,
SortedDocValues values) throws IOException {
- super(maxDoc, ords, nullPolicy, needsScores, boostDocs, values);
+ super(maxDoc, valueCount, nullPolicy, needsScores, boostDocs, values);
this.field = groupHeadSelector.selectorText;
- this.ordVals = new float[ords.length];
-
+
assert GroupHeadSelectorType.MIN_MAX.contains(groupHeadSelector.type);
if (GroupHeadSelectorType.MAX.equals(groupHeadSelector.type)) {
comp = new MaxFloatComp();
- Arrays.fill(ordVals, -Float.MAX_VALUE);
+ this.ordVals = new IntFloatDynamicMap(valueCount, -Float.MAX_VALUE);
this.nullVal = -Float.MAX_VALUE;
} else {
comp = new MinFloatComp();
- Arrays.fill(ordVals, Float.MAX_VALUE);
+ this.ordVals = new IntFloatDynamicMap(valueCount, Float.MAX_VALUE);
this.nullVal = Float.MAX_VALUE;
}
-
- if(needsScores) {
- this.scores = new float[ords.length];
- if(nullPolicy == CollapsingPostFilter.NULL_POLICY_EXPAND) {
- nullScores = new FloatArrayList();
- }
- }
}
public void setNextReader(LeafReaderContext context) throws IOException {
@@ -1739,11 +1719,11 @@ public class CollapsingQParserPlugin extends QParserPlugin {
float currentVal = Float.intBitsToFloat(currentMinMax);
if(ord > -1) {
- if(comp.test(currentVal, ordVals[ord])) {
- ords[ord] = globalDoc;
- ordVals[ord] = currentVal;
+ if(comp.test(currentVal, ordVals.get(ord))) {
+ ords.put(ord, globalDoc);
+ ordVals.put(ord, currentVal);
if(needsScores) {
- scores[ord] = scorer.score();
+ scores.put(ord, scorer.score());
}
}
} else if(this.nullPolicy == CollapsingPostFilter.NULL_POLICY_COLLAPSE) {
@@ -1774,34 +1754,26 @@ public class CollapsingQParserPlugin extends QParserPlugin {
private NumericDocValues minMaxVals;
private LongCompare comp;
private long nullVal;
- private long[] ordVals;
+ private IntLongDynamicMap ordVals;
public OrdLongStrategy(int maxDoc,
int nullPolicy,
- int[] ords,
+ int valueCount,
GroupHeadSelector groupHeadSelector,
boolean needsScores,
IntIntHashMap boostDocs, SortedDocValues values) throws IOException {
- super(maxDoc, ords, nullPolicy, needsScores, boostDocs, values);
+ super(maxDoc, valueCount, nullPolicy, needsScores, boostDocs, values);
this.field = groupHeadSelector.selectorText;
- this.ordVals = new long[ords.length];
assert GroupHeadSelectorType.MIN_MAX.contains(groupHeadSelector.type);
-
+
if (GroupHeadSelectorType.MAX.equals(groupHeadSelector.type)) {
comp = new MaxLongComp();
- Arrays.fill(ordVals, Long.MIN_VALUE);
+ this.ordVals = new IntLongDynamicMap(valueCount, Long.MIN_VALUE);
} else {
this.nullVal = Long.MAX_VALUE;
comp = new MinLongComp();
- Arrays.fill(ordVals, Long.MAX_VALUE);
- }
-
- if(needsScores) {
- this.scores = new float[ords.length];
- if(nullPolicy == CollapsingPostFilter.NULL_POLICY_EXPAND) {
- nullScores = new FloatArrayList();
- }
+ this.ordVals = new IntLongDynamicMap(valueCount, Long.MAX_VALUE);
}
}
@@ -1825,11 +1797,11 @@ public class CollapsingQParserPlugin extends QParserPlugin {
}
if(ord > -1) {
- if(comp.test(currentVal, ordVals[ord])) {
- ords[ord] = globalDoc;
- ordVals[ord] = currentVal;
+ if(comp.test(currentVal, ordVals.get(ord))) {
+ ords.put(ord, globalDoc);
+ ordVals.put(ord, currentVal);
if(needsScores) {
- scores[ord] = scorer.score();
+ scores.put(ord, scorer.score());
}
}
} else if(this.nullPolicy == CollapsingPostFilter.NULL_POLICY_COLLAPSE) {
@@ -1859,14 +1831,14 @@ public class CollapsingQParserPlugin extends QParserPlugin {
private float nullVal;
private ValueSource valueSource;
private FunctionValues functionValues;
- private float[] ordVals;
+ private IntFloatDynamicMap ordVals;
private Map rcontext;
private final CollapseScore collapseScore = new CollapseScore();
private boolean needsScores4Collapsing;
public OrdValueSourceStrategy(int maxDoc,
int nullPolicy,
- int[] ords,
+ int valueCount,
GroupHeadSelector groupHeadSelector,
boolean needsScores4Collapsing,
boolean needsScores,
@@ -1874,31 +1846,23 @@ public class CollapsingQParserPlugin extends QParserPlugin {
FunctionQuery funcQuery,
IndexSearcher searcher,
SortedDocValues values) throws IOException {
- super(maxDoc, ords, nullPolicy, needsScores, boostDocs, values);
+ super(maxDoc, valueCount, nullPolicy, needsScores, boostDocs, values);
this.needsScores4Collapsing = needsScores4Collapsing;
this.valueSource = funcQuery.getValueSource();
this.rcontext = ValueSource.newContext(searcher);
- this.ordVals = new float[ords.length];
assert GroupHeadSelectorType.MIN_MAX.contains(groupHeadSelector.type);
-
+
if (GroupHeadSelectorType.MAX.equals(groupHeadSelector.type)) {
comp = new MaxFloatComp();
- Arrays.fill(ordVals, -Float.MAX_VALUE );
+ this.ordVals = new IntFloatDynamicMap(valueCount, -Float.MAX_VALUE);
} else {
this.nullVal = Float.MAX_VALUE;
comp = new MinFloatComp();
- Arrays.fill(ordVals, Float.MAX_VALUE);
+ this.ordVals = new IntFloatDynamicMap(valueCount, Float.MAX_VALUE);
}
collapseScore.setupIfNeeded(groupHeadSelector, rcontext);
-
- if(this.needsScores) {
- this.scores = new float[ords.length];
- if(nullPolicy == CollapsingPostFilter.NULL_POLICY_EXPAND) {
- nullScores = new FloatArrayList();
- }
- }
}
public void setNextReader(LeafReaderContext context) throws IOException {
@@ -1921,14 +1885,14 @@ public class CollapsingQParserPlugin extends QParserPlugin {
float currentVal = functionValues.floatVal(contextDoc);
if(ord > -1) {
- if(comp.test(currentVal, ordVals[ord])) {
- ords[ord] = globalDoc;
- ordVals[ord] = currentVal;
+ if(comp.test(currentVal, ordVals.get(ord))) {
+ ords.put(ord, globalDoc);
+ ordVals.put(ord, currentVal);
if(needsScores) {
if (!needsScores4Collapsing) {
score = scorer.score();
}
- scores[ord] = score;
+ scores.put(ord, score);
}
}
} else if(this.nullPolicy == CollapsingPostFilter.NULL_POLICY_COLLAPSE) {
@@ -1968,7 +1932,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
public OrdSortSpecStrategy(int maxDoc,
int nullPolicy,
- int[] ords,
+ int valueCount,
GroupHeadSelector groupHeadSelector,
boolean needsScores4Collapsing,
boolean needsScores,
@@ -1976,21 +1940,14 @@ public class CollapsingQParserPlugin extends QParserPlugin {
SortSpec sortSpec,
IndexSearcher searcher,
SortedDocValues values) throws IOException {
- super(maxDoc, ords, nullPolicy, needsScores, boostDocs, values);
+ super(maxDoc, valueCount, nullPolicy, needsScores, boostDocs, values);
this.needsScores4Collapsing = needsScores4Collapsing;
-
+
assert GroupHeadSelectorType.SORT.equals(groupHeadSelector.type);
-
+
this.sort = rewriteSort(sortSpec, searcher);
-
- this.compareState = new SortFieldsCompare(sort.getSort(), ords.length);
- if (this.needsScores) {
- this.scores = new float[ords.length];
- if(nullPolicy == CollapsingPostFilter.NULL_POLICY_EXPAND) {
- nullScores = new FloatArrayList();
- }
- }
+ this.compareState = new SortFieldsCompare(sort.getSort(), valueCount);
}
@Override
@@ -2003,10 +1960,10 @@ public class CollapsingQParserPlugin extends QParserPlugin {
super.setScorer(s);
this.compareState.setScorer(s);
}
-
+
@Override
public void collapse(int ord, int contextDoc, int globalDoc) throws IOException {
-
+
if(boosted && mergeBoost.boost(globalDoc)) {
this.boostOrds.add(ord);
this.boostDocs.add(globalDoc);
@@ -2017,25 +1974,25 @@ public class CollapsingQParserPlugin extends QParserPlugin {
}
if (ord > -1) { // real collapseKey
- if (-1 == ords[ord]) {
+ if (-1 == ords.get(ord)) {
// we've never seen this ord (aka: collapseKey) before, treat it as group head for now
compareState.setGroupValues(ord, contextDoc);
- ords[ord] = globalDoc;
+ ords.put(ord, globalDoc);
if (needsScores) {
if (!needsScores4Collapsing) {
this.score = scorer.score();
}
- scores[ord] = score;
+ scores.put(ord, score);
}
} else {
// test this ord to see if it's a new group leader
if (compareState.testAndSetGroupValues(ord, contextDoc)) {//TODO X
- ords[ord] = globalDoc;
+ ords.put(ord, globalDoc);
if (needsScores) {
if (!needsScores4Collapsing) {
this.score = scorer.score();
}
- scores[ord] = score;
+ scores.put(ord, score);
}
}
}
@@ -2085,12 +2042,12 @@ public class CollapsingQParserPlugin extends QParserPlugin {
protected Scorable scorer;
protected FloatArrayList nullScores;
protected float nullScore;
- protected float[] scores;
+ protected IntFloatDynamicMap scores;
protected FixedBitSet collapsedSet;
protected int nullDoc = -1;
protected boolean needsScores;
protected String collapseField;
- protected int[] docs;
+ protected IntIntDynamicMap docs;
protected int nullValue;
protected IntArrayList boostDocs;
protected IntArrayList boostKeys;
@@ -2113,7 +2070,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
this.needsScores = needsScores;
this.collapsedSet = new FixedBitSet(maxDoc);
this.cmap = new IntIntHashMap(size);
- this.docs = new int[size];
+ this.docs = new IntIntDynamicMap(size, 0);
if(boostDocsMap != null) {
this.boosts = true;
this.boostDocs = new IntArrayList();
@@ -2129,6 +2086,13 @@ public class CollapsingQParserPlugin extends QParserPlugin {
Arrays.sort(bd);
this.mergeBoost = new MergeBoost(bd);
}
+
+ if(needsScores) {
+ this.scores = new IntFloatDynamicMap(size, 0.0f);
+ if(nullPolicy == CollapsingPostFilter.NULL_POLICY_EXPAND) {
+ nullScores = new FloatArrayList();
+ }
+ }
}
public FixedBitSet getCollapsedSet() {
@@ -2156,7 +2120,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
while(it1.hasNext()) {
IntIntCursor cursor = it1.next();
int pointer = cursor.value;
- collapsedSet.set(docs[pointer]);
+ collapsedSet.set(docs.get(pointer));
}
return collapsedSet;
@@ -2178,11 +2142,11 @@ public class CollapsingQParserPlugin extends QParserPlugin {
return this.nullScore;
}
- public float[] getScores() {
+ public IntFloatDynamicMap getScores() {
return scores;
}
- public int[] getDocs() { return docs;}
+ public IntIntDynamicMap getDocs() { return docs;}
public MergeBoost getMergeBoost() {
return this.mergeBoost;
@@ -2197,7 +2161,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
private final String field;
private NumericDocValues minMaxVals;
- private int[] testValues;
+ private IntIntDynamicMap testValues;
private IntCompare comp;
private int nullCompVal;
@@ -2214,10 +2178,10 @@ public class CollapsingQParserPlugin extends QParserPlugin {
super(maxDoc, size, collapseField, nullValue, nullPolicy, needsScores, boostDocs);
this.field = groupHeadSelector.selectorText;
- this.testValues = new int[size];
+ this.testValues = new IntIntDynamicMap(size, 0);
assert GroupHeadSelectorType.MIN_MAX.contains(groupHeadSelector.type);
-
+
if (GroupHeadSelectorType.MAX.equals(groupHeadSelector.type)) {
comp = new MaxIntComp();
this.nullCompVal = Integer.MIN_VALUE;
@@ -2225,13 +2189,6 @@ public class CollapsingQParserPlugin extends QParserPlugin {
comp = new MinIntComp();
this.nullCompVal = Integer.MAX_VALUE;
}
-
- if(needsScores) {
- this.scores = new float[size];
- if(nullPolicy == CollapsingPostFilter.NULL_POLICY_EXPAND) {
- nullScores = new FloatArrayList();
- }
- }
}
public void setNextReader(LeafReaderContext context) throws IOException {
@@ -2258,29 +2215,20 @@ public class CollapsingQParserPlugin extends QParserPlugin {
final int idx;
if((idx = cmap.indexOf(collapseKey)) >= 0) {
int pointer = cmap.indexGet(idx);
- if(comp.test(currentVal, testValues[pointer])) {
- testValues[pointer]= currentVal;
- docs[pointer] = globalDoc;
+ if(comp.test(currentVal, testValues.get(pointer))) {
+ testValues.put(pointer, currentVal);
+ docs.put(pointer, globalDoc);
if(needsScores) {
- scores[pointer] = scorer.score();
+ scores.put(pointer, scorer.score());
}
}
} else {
++index;
cmap.put(collapseKey, index);
- if(index == testValues.length) {
- testValues = ArrayUtil.grow(testValues);
- docs = ArrayUtil.grow(docs);
- if(needsScores) {
- scores = ArrayUtil.grow(scores);
- }
- }
-
- testValues[index] = currentVal;
- docs[index] = (globalDoc);
-
+ testValues.put(index, currentVal);
+ docs.put(index, globalDoc);
if(needsScores) {
- scores[index] = scorer.score();
+ scores.put(index, scorer.score());
}
}
} else if(this.nullPolicy == CollapsingPostFilter.NULL_POLICY_COLLAPSE) {
@@ -2304,7 +2252,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
private final String field;
private NumericDocValues minMaxVals;
- private float[] testValues;
+ private IntFloatDynamicMap testValues;
private FloatCompare comp;
private float nullCompVal;
@@ -2321,10 +2269,10 @@ public class CollapsingQParserPlugin extends QParserPlugin {
super(maxDoc, size, collapseField, nullValue, nullPolicy, needsScores, boostDocs);
this.field = groupHeadSelector.selectorText;
- this.testValues = new float[size];
+ this.testValues = new IntFloatDynamicMap(size, 0.0f);
assert GroupHeadSelectorType.MIN_MAX.contains(groupHeadSelector.type);
-
+
if (GroupHeadSelectorType.MAX.equals(groupHeadSelector.type)) {
comp = new MaxFloatComp();
this.nullCompVal = -Float.MAX_VALUE;
@@ -2332,13 +2280,6 @@ public class CollapsingQParserPlugin extends QParserPlugin {
comp = new MinFloatComp();
this.nullCompVal = Float.MAX_VALUE;
}
-
- if(needsScores) {
- this.scores = new float[size];
- if(nullPolicy == CollapsingPostFilter.NULL_POLICY_EXPAND) {
- nullScores = new FloatArrayList();
- }
- }
}
public void setNextReader(LeafReaderContext context) throws IOException {
@@ -2367,28 +2308,20 @@ public class CollapsingQParserPlugin extends QParserPlugin {
final int idx;
if((idx = cmap.indexOf(collapseKey)) >= 0) {
int pointer = cmap.indexGet(idx);
- if(comp.test(currentVal, testValues[pointer])) {
- testValues[pointer] = currentVal;
- docs[pointer] = globalDoc;
+ if(comp.test(currentVal, testValues.get(pointer))) {
+ testValues.put(pointer, currentVal);
+ docs.put(pointer, globalDoc);
if(needsScores) {
- scores[pointer] = scorer.score();
+ scores.put(pointer, scorer.score());
}
}
} else {
++index;
cmap.put(collapseKey, index);
- if(index == testValues.length) {
- testValues = ArrayUtil.grow(testValues);
- docs = ArrayUtil.grow(docs);
- if(needsScores) {
- scores = ArrayUtil.grow(scores);
- }
- }
-
- testValues[index] = currentVal;
- docs[index] = globalDoc;
+ testValues.put(index, currentVal);
+ docs.put(index, globalDoc);
if(needsScores) {
- scores[index] = scorer.score();
+ scores.put(index, scorer.score());
}
}
} else if(this.nullPolicy == CollapsingPostFilter.NULL_POLICY_COLLAPSE) {
@@ -2415,7 +2348,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
private static class IntValueSourceStrategy extends IntFieldValueStrategy {
private FloatCompare comp;
- private float[] testValues;
+ private IntFloatDynamicMap testValues;
private float nullCompVal;
private ValueSource valueSource;
@@ -2440,13 +2373,13 @@ public class CollapsingQParserPlugin extends QParserPlugin {
super(maxDoc, size, collapseField, nullValue, nullPolicy, needsScores, boostDocs);
this.needsScores4Collapsing = needsScores4Collapsing;
- this.testValues = new float[size];
+ this.testValues = new IntFloatDynamicMap(size, 0.0f);
this.valueSource = funcQuery.getValueSource();
this.rcontext = ValueSource.newContext(searcher);
assert GroupHeadSelectorType.MIN_MAX.contains(groupHeadSelector.type);
-
+
if (GroupHeadSelectorType.MAX.equals(groupHeadSelector.type)) {
this.nullCompVal = -Float.MAX_VALUE;
comp = new MaxFloatComp();
@@ -2456,13 +2389,6 @@ public class CollapsingQParserPlugin extends QParserPlugin {
}
collapseScore.setupIfNeeded(groupHeadSelector, rcontext);
-
- if(needsScores) {
- this.scores = new float[size];
- if(nullPolicy == CollapsingPostFilter.NULL_POLICY_EXPAND) {
- nullScores = new FloatArrayList();
- }
- }
}
public void setNextReader(LeafReaderContext context) throws IOException {
@@ -2490,33 +2416,26 @@ public class CollapsingQParserPlugin extends QParserPlugin {
final int idx;
if((idx = cmap.indexOf(collapseKey)) >= 0) {
int pointer = cmap.indexGet(idx);
- if(comp.test(currentVal, testValues[pointer])) {
- testValues[pointer] = currentVal;
- docs[pointer] = globalDoc;
+ if(comp.test(currentVal, testValues.get(pointer))) {
+ testValues.put(pointer, currentVal);
+ docs.put(pointer, globalDoc);
if(needsScores){
if (!needsScores4Collapsing) {
score = scorer.score();
}
- scores[pointer] = score;
+ scores.put(pointer, score);
}
}
} else {
++index;
cmap.put(collapseKey, index);
- if(index == testValues.length) {
- testValues = ArrayUtil.grow(testValues);
- docs = ArrayUtil.grow(docs);
- if(needsScores) {
- scores = ArrayUtil.grow(scores);
- }
- }
- docs[index] = globalDoc;
- testValues[index] = currentVal;
+ docs.put(index, globalDoc);
+ testValues.put(index, currentVal);
if(needsScores) {
if (!needsScores4Collapsing) {
score = scorer.score();
}
- scores[index] = score;
+ scores.put(index, score);
}
}
} else if(this.nullPolicy == CollapsingPostFilter.NULL_POLICY_COLLAPSE) {
@@ -2542,9 +2461,9 @@ public class CollapsingQParserPlugin extends QParserPlugin {
}
}
-
+
/*
- * Strategy for collapsing on a 32 bit numeric field and using the first document according
+ * Strategy for collapsing on a 32 bit numeric field and using the first document according
* to a complex sort as the group head
*/
private static class IntSortSpecStrategy extends IntFieldValueStrategy {
@@ -2552,7 +2471,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
private final SortFieldsCompare compareState;
private final SortSpec sortSpec;
private final Sort sort;
-
+
private int index=-1;
private boolean needsScores4Collapsing;
@@ -2567,7 +2486,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
IntIntHashMap boostDocs,
SortSpec sortSpec,
IndexSearcher searcher) throws IOException {
-
+
super(maxDoc, size, collapseField, nullValue, nullPolicy, needsScores, boostDocs);
this.needsScores4Collapsing = needsScores4Collapsing;
@@ -2576,13 +2495,6 @@ public class CollapsingQParserPlugin extends QParserPlugin {
this.sortSpec = sortSpec;
this.sort = rewriteSort(sortSpec, searcher);
this.compareState = new SortFieldsCompare(sort.getSort(), size);
-
- if(needsScores) {
- this.scores = new float[size];
- if(nullPolicy == CollapsingPostFilter.NULL_POLICY_EXPAND) {
- nullScores = new FloatArrayList();
- }
- }
}
@Override
@@ -2616,32 +2528,25 @@ public class CollapsingQParserPlugin extends QParserPlugin {
// we've seen this collapseKey before, test to see if it's a new group leader
int pointer = cmap.indexGet(idx);
if (compareState.testAndSetGroupValues(pointer, contextDoc)) {
- docs[pointer] = globalDoc;
+ docs.put(pointer, globalDoc);
if (needsScores) {
if (!needsScores4Collapsing) {
score = scorer.score();
}
- scores[pointer] = score;
+ scores.put(pointer, score);
}
}
} else {
// we've never seen this collapseKey before, treat it as group head for now
++index;
cmap.put(collapseKey, index);
- if (index == docs.length) {
- docs = ArrayUtil.grow(docs);
- compareState.grow(docs.length);
- if(needsScores) {
- scores = ArrayUtil.grow(scores);
- }
- }
- docs[index] = globalDoc;
+ docs.put(index, globalDoc);
compareState.setGroupValues(index, contextDoc);
if(needsScores) {
if (!needsScores4Collapsing) {
score = scorer.score();
}
- scores[index] = score;
+ scores.put(index, score);
}
}
} else if(this.nullPolicy == CollapsingPostFilter.NULL_POLICY_COLLAPSE) {
@@ -2716,11 +2621,11 @@ public class CollapsingQParserPlugin extends QParserPlugin {
}
}
- /**
- * Class for comparing documents according to a list of SortField clauses and
- * tracking the groupHeadLeaders and their sort values. groups will be identified
- * by int "contextKey values, which may either be (encoded) 32bit numeric values, or
- * ordinal values for Strings -- this class doesn't care, and doesn't assume any special
+ /**
+ * Class for comparing documents according to a list of SortField clauses and
+ * tracking the groupHeadLeaders and their sort values. groups will be identified
+ * by int "contextKey values, which may either be (encoded) 32bit numeric values, or
+ * ordinal values for Strings -- this class doesn't care, and doesn't assume any special
* meaning.
*/
private static class SortFieldsCompare {
@@ -2732,9 +2637,9 @@ public class CollapsingQParserPlugin extends QParserPlugin {
private Object[][] groupHeadValues; // growable
final private Object[] nullGroupValues;
-
+
/**
- * Constructs an instance based on the the (raw, un-rewritten) SortFields to be used,
+ * Constructs an instance based on the the (raw, un-rewritten) SortFields to be used,
* and an initial number of expected groups (will grow as needed).
*/
public SortFieldsCompare(SortField[] sorts, int initNumGroups) {
@@ -2763,7 +2668,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
leafFieldComparators[clause].setScorer(s);
}
}
-
+
// LUCENE-6808 workaround
private static Object cloneIfBytesRef(Object val) {
if (val instanceof BytesRef) {
@@ -2773,8 +2678,8 @@ public class CollapsingQParserPlugin extends QParserPlugin {
}
/**
- * Returns the current SortField values for the specified collapseKey.
- * If this collapseKey has never been seen before, then an array of null values is inited
+ * Returns the current SortField values for the specified collapseKey.
+ * If this collapseKey has never been seen before, then an array of null values is inited
* and tracked so that the caller may update it if needed.
*/
private Object[] getOrInitGroupHeadValues(int collapseKey) {
@@ -2785,10 +2690,10 @@ public class CollapsingQParserPlugin extends QParserPlugin {
}
return values;
}
-
+
/**
- * Records the SortField values for the specified contextDoc as the "best" values
- * for the group identified by the specified collapseKey.
+ * Records the SortField values for the specified contextDoc as the "best" values
+ * for the group identified by the specified collapseKey.
*
* Should be called the first time a contextKey is encountered.
*/
diff --git a/solr/core/src/java/org/apache/solr/util/DynamicMap.java b/solr/core/src/java/org/apache/solr/util/DynamicMap.java
new file mode 100644
index 0000000..b9e4c87
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/util/DynamicMap.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.util;
+
+/**
+ * An efficient map for storing keys as integer in range from 0..n with n can be estimated up-front.
+ * By automatically switching from a hashMap (which is memory efficient) to an array (which is faster)
+ * on increasing number of keys.
+ * So it SHOULD not be used for other cases where key can be any arbitrary integer.
+ */
+public interface DynamicMap {
+
+ default boolean useArrayBased(int expectedKeyMax) {
+ boolean assertsEnabled = false;
+ assert assertsEnabled = true; // Intentional side-effect!
+ if (assertsEnabled) {
+ // avoid using array based up-front on testing
+ return false;
+ }
+
+ // for small size, prefer using array based
+ return expectedKeyMax < (1 << 12);
+ }
+
+ /**
+ * Compute threshold for switching from hashMap based to array
+ */
+ default int threshold(int expectedKeyMax) {
+ return expectedKeyMax >>> 6;
+ }
+
+ /**
+ * Compute expected elements for hppc maps, so resizing won't happen if we store less elements than {@code threshold}
+ */
+ default int mapExpectedElements(int expectedKeyMax) {
+ // hppc's expectedElements <= first hppc's resizeAt.
+ // +2 let's us not to worry about which comparison operator to choose
+ return threshold(expectedKeyMax) + 2;
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/util/FloatConsumer.java b/solr/core/src/java/org/apache/solr/util/FloatConsumer.java
new file mode 100644
index 0000000..d9ad851
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/util/FloatConsumer.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.util;
+
+import java.util.function.Consumer;
+
+/**
+ * Represents an operation that accepts a single {@code float}-valued argument and
+ * returns no result. This is the primitive type specialization of
+ * {@link Consumer} for {@code float}. Unlike most other functional interfaces,
+ * {@code IntConsumer} is expected to operate via side-effects.
+ *
+ * @see Consumer
+ */
+@FunctionalInterface
+public interface FloatConsumer {
+
+ /**
+ * Performs this operation on the given argument.
+ *
+ * @param value the input argument
+ */
+ void accept(float value);
+}
diff --git a/solr/core/src/java/org/apache/solr/util/IntFloatDynamicMap.java b/solr/core/src/java/org/apache/solr/util/IntFloatDynamicMap.java
new file mode 100644
index 0000000..11a2e09
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/util/IntFloatDynamicMap.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.util;
+
+import java.util.Arrays;
+
+import com.carrotsearch.hppc.IntFloatHashMap;
+import com.carrotsearch.hppc.cursors.FloatCursor;
+import com.carrotsearch.hppc.procedures.IntFloatProcedure;
+import org.apache.lucene.util.ArrayUtil;
+
+public class IntFloatDynamicMap implements DynamicMap {
+ private int maxSize;
+ private IntFloatHashMap hashMap;
+ private float[] keyValues;
+ private float emptyValue;
+ private int threshold;
+
+ /**
+ * Create map with expected max value of key.
+ * Although the map will automatically do resizing to be able to hold key {@code >= expectedKeyMax}.
+ * But putting key much larger than {@code expectedKeyMax} is discourage since it can leads to use LOT OF memory.
+ */
+ public IntFloatDynamicMap(int expectedKeyMax, float emptyValue) {
+ this.threshold = threshold(expectedKeyMax);
+ this.maxSize = expectedKeyMax;
+ this.emptyValue = emptyValue;
+ if (useArrayBased(expectedKeyMax)) {
+ upgradeToArray();
+ } else {
+ this.hashMap = new IntFloatHashMap(mapExpectedElements(expectedKeyMax));
+ }
+ }
+
+ private void upgradeToArray() {
+ keyValues = new float[maxSize];
+ if (emptyValue != 0.0f) {
+ Arrays.fill(keyValues, emptyValue);
+ }
+ if (hashMap != null) {
+ hashMap.forEach((IntFloatProcedure) (key, value) -> keyValues[key] = value);
+ hashMap = null;
+ }
+ }
+
+ private void growBuffer(int minSize) {
+ assert keyValues != null;
+ int size = keyValues.length;
+ keyValues = ArrayUtil.grow(keyValues, minSize);
+ if (emptyValue != 0.0f) {
+ for (int i = size; i < keyValues.length; i++) {
+ keyValues[i] = emptyValue;
+ }
+ }
+ }
+
+ public void put(int key, float value) {
+ if (keyValues != null) {
+ if (key >= keyValues.length) {
+ growBuffer(key + 1);
+ }
+ keyValues[key] = value;
+ } else {
+ this.hashMap.put(key, value);
+ this.maxSize = Math.max(key+1, maxSize);
+ if (this.hashMap.size() >= threshold) {
+ upgradeToArray();
+ }
+ }
+ }
+
+ public float get(int key) {
+ if (keyValues != null) {
+ if (key >= keyValues.length) {
+ return emptyValue;
+ }
+ return keyValues[key];
+ } else {
+ return this.hashMap.getOrDefault(key, emptyValue);
+ }
+ }
+
+ public void forEachValue(FloatConsumer consumer) {
+ if (keyValues != null) {
+ for (float val : keyValues) {
+ if (val != emptyValue) consumer.accept(val);
+ }
+ } else {
+ for (FloatCursor ord : hashMap.values()) {
+ consumer.accept(ord.value);
+ }
+ }
+ }
+
+ public void remove(int key) {
+ if (keyValues != null) {
+ if (key < keyValues.length)
+ keyValues[key] = emptyValue;
+ } else {
+ hashMap.remove(key);
+ }
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/util/IntIntDynamicMap.java b/solr/core/src/java/org/apache/solr/util/IntIntDynamicMap.java
new file mode 100644
index 0000000..c63fb8b
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/util/IntIntDynamicMap.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.util;
+
+import java.util.Arrays;
+import java.util.function.IntConsumer;
+
+import com.carrotsearch.hppc.IntIntHashMap;
+import com.carrotsearch.hppc.cursors.IntCursor;
+import com.carrotsearch.hppc.procedures.IntIntProcedure;
+import org.apache.lucene.util.ArrayUtil;
+
+public class IntIntDynamicMap implements DynamicMap {
+ private int maxSize;
+ private IntIntHashMap hashMap;
+ private int[] keyValues;
+ private int emptyValue;
+ private int threshold;
+
+ /**
+ * Create map with expected max value of key.
+ * Although the map will automatically do resizing to be able to hold key {@code >= expectedKeyMax}.
+ * But putting key much larger than {@code expectedKeyMax} is discourage since it can leads to use LOT OF memory.
+ */
+ public IntIntDynamicMap(int expectedKeyMax, int emptyValue) {
+ this.threshold = threshold(expectedKeyMax);
+ this.maxSize = expectedKeyMax;
+ this.emptyValue = emptyValue;
+ if (useArrayBased(expectedKeyMax)) {
+ upgradeToArray();
+ } else {
+ this.hashMap = new IntIntHashMap(mapExpectedElements(expectedKeyMax));
+ }
+ }
+
+ private void upgradeToArray() {
+ keyValues = new int[maxSize];
+ if (emptyValue != 0) {
+ Arrays.fill(keyValues, emptyValue);
+ }
+ if (hashMap != null) {
+ hashMap.forEach((IntIntProcedure) (key, value) -> keyValues[key] = value);
+ hashMap = null;
+ }
+ }
+
+ private void growBuffer(int minSize) {
+ assert keyValues != null;
+ int size = keyValues.length;
+ keyValues = ArrayUtil.grow(keyValues, minSize);
+ if (emptyValue != 0) {
+ for (int i = size; i < keyValues.length; i++) {
+ keyValues[i] = emptyValue;
+ }
+ }
+ }
+
+ public void put(int key, int value) {
+ if (keyValues != null) {
+ if (key >= keyValues.length) {
+ growBuffer(key + 1);
+ }
+ keyValues[key] = value;
+ } else {
+ this.maxSize = Math.max(key + 1, maxSize);
+ this.hashMap.put(key, value);
+ if (this.hashMap.size() >= threshold) {
+ upgradeToArray();
+ }
+ }
+ }
+
+ public int get(int key) {
+ if (keyValues != null) {
+ if (key >= keyValues.length) {
+ return emptyValue;
+ }
+ return keyValues[key];
+ } else {
+ return this.hashMap.getOrDefault(key, emptyValue);
+ }
+ }
+
+ public void forEachValue(IntConsumer consumer) {
+ if (keyValues != null) {
+ for (int val : keyValues) {
+ if (val != emptyValue) consumer.accept(val);
+ }
+ } else {
+ for (IntCursor ord : hashMap.values()) {
+ consumer.accept(ord.value);
+ }
+ }
+ }
+
+ public void remove(int key) {
+ if (keyValues != null) {
+ if (key < keyValues.length)
+ keyValues[key] = emptyValue;
+ } else {
+ hashMap.remove(key);
+ }
+ }
+
+}
diff --git a/solr/core/src/java/org/apache/solr/util/IntLongDynamicMap.java b/solr/core/src/java/org/apache/solr/util/IntLongDynamicMap.java
new file mode 100644
index 0000000..1769a87
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/util/IntLongDynamicMap.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.util;
+
+import java.util.Arrays;
+import java.util.function.LongConsumer;
+
+import com.carrotsearch.hppc.IntLongHashMap;
+import com.carrotsearch.hppc.cursors.LongCursor;
+import com.carrotsearch.hppc.procedures.IntLongProcedure;
+import org.apache.lucene.util.ArrayUtil;
+
+public class IntLongDynamicMap implements DynamicMap {
+ private int maxSize;
+ private IntLongHashMap hashMap;
+ private long[] keyValues;
+ private long emptyValue;
+ private int threshold;
+
+ /**
+ * Create map with expected max value of key.
+ * Although the map will automatically do resizing to be able to hold key {@code >=g expectedKeyMax}.
+ * But putting key much larger than {@code expectedKeyMax} is discourage since it can leads to use LOT OF memory.
+ */
+ public IntLongDynamicMap(int expectedKeyMax, long emptyValue) {
+ this.threshold = threshold(expectedKeyMax);
+ this.maxSize = expectedKeyMax;
+ this.emptyValue = emptyValue;
+ if (useArrayBased(expectedKeyMax)) {
+ upgradeToArray();
+ } else {
+ this.hashMap = new IntLongHashMap(mapExpectedElements(expectedKeyMax));
+ }
+ }
+
+ private void upgradeToArray() {
+ keyValues = new long[maxSize];
+ if (emptyValue != 0) {
+ Arrays.fill(keyValues, emptyValue);
+ }
+ if (hashMap != null) {
+ hashMap.forEach((IntLongProcedure) (key, value) -> keyValues[key] = value);
+ hashMap = null;
+ }
+ }
+
+ private void growBuffer(int minSize) {
+ assert keyValues != null;
+ int size = keyValues.length;
+ keyValues = ArrayUtil.grow(keyValues, minSize);
+ if (emptyValue != 0) {
+ for (int i = size; i < keyValues.length; i++) {
+ keyValues[i] = emptyValue;
+ }
+ }
+ }
+
+ public void put(int key, long value) {
+ if (keyValues != null) {
+ if (key >= keyValues.length) {
+ growBuffer(key + 1);
+ }
+ keyValues[key] = value;
+ } else {
+ this.maxSize = Math.max(key + 1, maxSize);
+ this.hashMap.put(key, value);
+ if (this.hashMap.size() >= threshold) {
+ upgradeToArray();
+ }
+ }
+ }
+
+ public long get(int key) {
+ if (keyValues != null) {
+ if (key >= keyValues.length) {
+ return emptyValue;
+ }
+ return keyValues[key];
+ } else {
+ return this.hashMap.getOrDefault(key, emptyValue);
+ }
+ }
+
+ public void forEachValue(LongConsumer consumer) {
+ if (keyValues != null) {
+ for (long val : keyValues) {
+ if (val != emptyValue) consumer.accept(val);
+ }
+ } else {
+ for (LongCursor ord : hashMap.values()) {
+ consumer.accept(ord.value);
+ }
+ }
+ }
+
+ public void remove(int key) {
+ if (keyValues != null) {
+ if (key < keyValues.length)
+ keyValues[key] = emptyValue;
+ } else {
+ hashMap.remove(key);
+ }
+ }
+
+}
diff --git a/solr/core/src/test/org/apache/solr/util/DynamicMapsTest.java b/solr/core/src/test/org/apache/solr/util/DynamicMapsTest.java
new file mode 100644
index 0000000..cc20b99
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/util/DynamicMapsTest.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.util;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.solr.SolrTestCase;
+
+public class DynamicMapsTest extends SolrTestCase {
+
+ public void testIntFloatMap() {
+ IntFloatDynamicMap map = new IntFloatDynamicMap(10, -1.1f);
+ Map<Integer, Float> standard = new HashMap<>();
+ for (int i = 0; i < 100; i++) {
+ int key = random().nextInt(100) + 1;
+ float val = random().nextFloat();
+ standard.put(key, val);
+ map.put(key, val);
+ }
+
+ for (Map.Entry<Integer, Float> entry: standard.entrySet()) {
+ assertEquals(entry.getValue(), map.get(entry.getKey()), 0.0001);
+ }
+ AtomicInteger size = new AtomicInteger(0);
+ map.forEachValue(i -> size.incrementAndGet());
+ assertEquals(standard.size(), size.get());
+
+ assertEquals(-1.1f, map.get(101), 0.0001);
+ assertEquals(-1.1f, map.get(0), 0.0001);
+ }
+
+ public void testIntLongMap() {
+ IntLongDynamicMap map = new IntLongDynamicMap(10, -1);
+ Map<Integer, Long> standard = new HashMap<>();
+ for (int i = 0; i < 100; i++) {
+ int key = random().nextInt(100) + 1;
+ long val = random().nextLong();
+ standard.put(key, val);
+ map.put(key, val);
+ }
+
+ for (Map.Entry<Integer, Long> entry: standard.entrySet()) {
+ assertEquals((long)entry.getValue(), map.get(entry.getKey()));
+ }
+ AtomicInteger size = new AtomicInteger(0);
+ map.forEachValue(i -> size.incrementAndGet());
+ assertEquals(standard.size(), size.get());
+
+ assertEquals(-1, map.get(101));
+ assertEquals(-1, map.get(0));
+ }
+
+ public void testIntIntMap() {
+ IntIntDynamicMap map = new IntIntDynamicMap(10, -1);
+ Map<Integer, Integer> standard = new HashMap<>();
+ for (int i = 0; i < 100; i++) {
+ int key = random().nextInt(100) + 1;
+ int val = random().nextInt();
+ standard.put(key, val);
+ map.put(key, val);
+ }
+
+ for (Map.Entry<Integer, Integer> entry: standard.entrySet()) {
+ assertEquals((int)entry.getValue(), map.get(entry.getKey()));
+ }
+ AtomicInteger size = new AtomicInteger(0);
+ map.forEachValue(i -> size.incrementAndGet());
+ assertEquals(standard.size(), size.get());
+
+ assertEquals(-1, map.get(101));
+ assertEquals(-1, map.get(0));
+ }
+}