You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by yo...@apache.org on 2015/05/09 22:53:18 UTC
svn commit: r1678523 - in /lucene/dev/branches/branch_5x: ./ solr/
solr/core/ solr/core/src/java/org/apache/solr/search/facet/
Author: yonik
Date: Sat May 9 20:53:18 2015
New Revision: 1678523
URL: http://svn.apache.org/r1678523
Log:
SOLR-7519: SlotAcc resize ability with Resizer
Modified:
lucene/dev/branches/branch_5x/ (props changed)
lucene/dev/branches/branch_5x/solr/ (props changed)
lucene/dev/branches/branch_5x/solr/core/ (props changed)
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/facet/CountAgg.java
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/facet/FacetRequest.java
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/facet/PercentileAgg.java
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/facet/SlotAcc.java
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/facet/UnInvertedField.java
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/facet/UniqueAgg.java
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/facet/CountAgg.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/facet/CountAgg.java?rev=1678523&r1=1678522&r2=1678523&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/facet/CountAgg.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/facet/CountAgg.java Sat May 9 20:53:18 2015
@@ -26,7 +26,7 @@ public class CountAgg extends SimpleAggV
@Override
public SlotAcc createSlotAcc(FacetContext fcontext, int numDocs, int numSlots) throws IOException {
- return new CountSlotAcc(fcontext, numSlots);
+ return new CountSlotArrAcc(fcontext, numSlots);
}
@Override
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/facet/FacetRequest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/facet/FacetRequest.java?rev=1678523&r1=1678522&r2=1678523&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/facet/FacetRequest.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/facet/FacetRequest.java Sat May 9 20:53:18 2015
@@ -206,7 +206,7 @@ class FacetProcessor<FacetRequestT exten
protected void createAccs(int docCount, int slotCount) throws IOException {
accMap = new LinkedHashMap<String,SlotAcc>();
- countAcc = new CountSlotAcc(fcontext, slotCount);
+ countAcc = new CountSlotArrAcc(fcontext, slotCount);
countAcc.key = "count";
for (Map.Entry<String,AggValueSource> entry : freq.getFacetStats().entrySet()) {
SlotAcc acc = entry.getValue().createSlotAcc(fcontext, docCount, slotCount);
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/facet/PercentileAgg.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/facet/PercentileAgg.java?rev=1678523&r1=1678522&r2=1678523&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/facet/PercentileAgg.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/facet/PercentileAgg.java Sat May 9 20:53:18 2015
@@ -178,6 +178,11 @@ public class PercentileAgg extends Simpl
digests = new AVLTreeDigest[digests.length];
sortvals = null;
}
+
+ @Override
+ public void resize(Resizer resizer) {
+ digests = resizer.resize(digests, null);
+ }
}
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/facet/SlotAcc.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/facet/SlotAcc.java?rev=1678523&r1=1678522&r2=1678523&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/facet/SlotAcc.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/facet/SlotAcc.java Sat May 9 20:53:18 2015
@@ -33,7 +33,9 @@ import org.apache.solr.search.SolrIndexS
import java.io.Closeable;
import java.io.IOException;
+import java.lang.reflect.Array;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
@@ -61,11 +63,71 @@ public abstract class SlotAcc implements
public abstract void reset();
+ public abstract void resize(Resizer resizer);
+
@Override
public void close() throws IOException {
}
-}
+ public static abstract class Resizer {
+ public abstract int getNewSize();
+
+ public abstract int getNewSlot(int oldSlot);
+
+ public double[] resize(double[] old, double defaultValue) {
+ double[] values = new double[getNewSize()];
+ if (defaultValue != 0) {
+ Arrays.fill(values, 0, values.length, defaultValue);
+ }
+ for (int i = 0; i < old.length; i++) {
+ double val = old[i];
+ if (val != defaultValue) {
+ int newSlot = getNewSlot(i);
+ if (newSlot >= 0) {
+ values[newSlot] = val;
+ }
+ }
+ }
+ return values;
+ }
+
+ public int[] resize(int[] old, int defaultValue) {
+ int[] values = new int[getNewSize()];
+ if (defaultValue != 0) {
+ Arrays.fill(values, 0, values.length, defaultValue);
+ }
+ for (int i = 0; i < old.length; i++) {
+ int val = old[i];
+ if (val != defaultValue) {
+ int newSlot = getNewSlot(i);
+ if (newSlot >= 0) {
+ values[newSlot] = val;
+ }
+ }
+ }
+ return values;
+ }
+
+ public <T> T[] resize(T[] old, T defaultValue) {
+ T[] values = (T[]) Array.newInstance(old.getClass().getComponentType(), getNewSize());
+ if (defaultValue != null) {
+ Arrays.fill(values, 0, values.length, defaultValue);
+ }
+ for (int i = 0; i < old.length; i++) {
+ T val = old[i];
+ if (val != defaultValue) {
+ int newSlot = getNewSlot(i);
+ if (newSlot >= 0) {
+ values[newSlot] = val;
+ }
+ }
+ }
+ return values;
+ }
+
+ } // end class Resizer
+
+}
// TODO: we should really have a decoupled value provider...
// This would enhance reuse and also prevent multiple lookups of same value across diff stats
@@ -125,6 +187,11 @@ abstract class DoubleFuncSlotAcc extends
result[i] = initialValue;
}
}
+
+ @Override
+ public void resize(Resizer resizer) {
+ result = resizer.resize(result, initialValue);
+ }
}
abstract class IntSlotAcc extends SlotAcc {
@@ -156,6 +223,11 @@ abstract class IntSlotAcc extends SlotAc
result[i] = initialValue;
}
}
+
+ @Override
+ public void resize(Resizer resizer) {
+ result = resizer.resize(result, initialValue);
+ }
}
@@ -270,18 +342,44 @@ class AvgSlotAcc extends DoubleFuncSlotA
}
}
+ @Override
+ public void resize(Resizer resizer) {
+ super.resize(resizer);
+ counts = resizer.resize(counts, 0);
+ }
}
+abstract class CountSlotAcc extends SlotAcc {
+ public CountSlotAcc(FacetContext fcontext) {
+ super(fcontext);
+ }
+
+ public abstract void incrementCount(int slot, int count);
+ public abstract int getCount(int slot);
+}
-class CountSlotAcc extends IntSlotAcc {
- public CountSlotAcc(FacetContext fcontext, int numSlots) {
- super(fcontext, numSlots, 0);
+
+class CountSlotArrAcc extends CountSlotAcc {
+ int[] result;
+ public CountSlotArrAcc(FacetContext fcontext, int numSlots) {
+ super(fcontext);
+ result = new int[numSlots];
}
@Override
public void collect(int doc, int slotNum) { // TODO: count arrays can use fewer bytes based on the number of docs in the base set (that's the upper bound for single valued) - look at ttf?
- result[slotNum] = result[slotNum] + 1;
+ result[slotNum]++;
+ }
+
+ @Override
+ public int compare(int slotA, int slotB) {
+ return Integer.compare( result[slotA], result[slotB] );
+ }
+
+ @Override
+ public Object getValue(int slotNum) throws IOException {
+ return result[slotNum];
}
public void incrementCount(int slot, int count) {
@@ -299,7 +397,12 @@ class CountSlotAcc extends IntSlotAcc {
@Override
public void reset() {
- super.reset();
+ Arrays.fill(result, 0);
+ }
+
+ @Override
+ public void resize(Resizer resizer) {
+ resizer.resize(result, 0);
}
}
@@ -327,6 +430,12 @@ class SortSlotAcc extends SlotAcc {
public void reset() {
// no-op
}
+
+ @Override
+ public void resize(Resizer resizer) {
+ // sort slot only works with direct-mapped accumulators
+ throw new UnsupportedOperationException();
+ }
}
@@ -427,6 +536,10 @@ abstract class UniqueSlotAcc extends Slo
return counts[slotA] - counts[slotB];
}
+ @Override
+ public void resize(Resizer resizer) {
+ arr = resizer.resize(arr, null);
+ }
}
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/facet/UnInvertedField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/facet/UnInvertedField.java?rev=1678523&r1=1678522&r2=1678523&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/facet/UnInvertedField.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/facet/UnInvertedField.java Sat May 9 20:53:18 2015
@@ -304,7 +304,7 @@ public class UnInvertedField extends Doc
- private void getCountsInArray(FacetFieldProcessorUIF processor, int[] counts) throws IOException {
+ private void getCounts(FacetFieldProcessorUIF processor, CountSlotAcc counts) throws IOException {
DocSet docs = processor.fcontext.base;
int baseSize = docs.size();
int maxDoc = searcher.maxDoc();
@@ -330,7 +330,7 @@ public class UnInvertedField extends Doc
// For the biggest terms, do straight set intersections
for (TopTerm tt : bigTerms.values()) {
// TODO: counts could be deferred if sorting by index order
- counts[tt.termNum] = searcher.numDocs(tt.termQuery, docs);
+ counts.incrementCount(tt.termNum, searcher.numDocs(tt.termQuery, docs));
}
// TODO: we could short-circuit counting altogether for sorted faceting
@@ -356,7 +356,7 @@ public class UnInvertedField extends Doc
}
if (delta == 0) break;
tnum += delta - TNUM_OFFSET;
- counts[tnum]++;
+ counts.incrementCount(tnum,1);
}
} else {
int tnum = 0;
@@ -366,7 +366,7 @@ public class UnInvertedField extends Doc
if ((code & 0x80) == 0) {
if (delta == 0) break;
tnum += delta - TNUM_OFFSET;
- counts[tnum]++;
+ counts.incrementCount(tnum,1);
delta = 0;
}
code >>>= 8;
@@ -377,16 +377,17 @@ public class UnInvertedField extends Doc
if (doNegative) {
for (int i=0; i<numTermsInField; i++) {
- counts[i] = maxTermCounts[i] - counts[i];
+ // counts[i] = maxTermCounts[i] - counts[i];
+ counts.incrementCount(i, maxTermCounts[i] - counts.getCount(i)*2);
}
}
if (processor.allBucketsSlot >= 0) {
int all = 0; // overflow potential
for (int i=0; i<numTermsInField; i++) {
- all += counts[i];
+ all += counts.getCount(i);
}
- counts[processor.allBucketsSlot] = all;
+ counts.incrementCount(processor.allBucketsSlot, all);
}
}
@@ -395,8 +396,7 @@ public class UnInvertedField extends Doc
public void collectDocs(FacetFieldProcessorUIF processor) throws IOException {
if (processor.accs.length == 0 && processor.startTermIndex == 0 && processor.endTermIndex >= numTermsInField)
{
- int[] arr = processor.countAcc.getCountArray();
- getCountsInArray(processor, arr);
+ getCounts(processor, processor.countAcc);
/*** debugging
int sz = processor.countAcc.getCountArray().length;
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/facet/UniqueAgg.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/facet/UniqueAgg.java?rev=1678523&r1=1678522&r2=1678523&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/facet/UniqueAgg.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/facet/UniqueAgg.java Sat May 9 20:53:18 2015
@@ -210,6 +210,11 @@ public class UniqueAgg extends StrAggVal
}
@Override
+ public void resize(Resizer resizer) {
+ resizer.resize(sets, null);
+ }
+
+ @Override
public void setNextReader(LeafReaderContext readerContext) throws IOException {
values = DocValues.getNumeric(readerContext.reader(), sf.getName());
exists = DocValues.getDocsWithField(readerContext.reader(), sf.getName());