You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mu...@apache.org on 2019/12/07 06:08:04 UTC
[lucene-solr] branch master updated: SOLR-11706: minor refactors
This is an automated email from the ASF dual-hosted git repository.
munendrasn pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/master by this push:
new a1e51cd SOLR-11706: minor refactors
a1e51cd is described below
commit a1e51cd7778f2de93327772bdcf0996bd012f371
Author: Munendra S N <mu...@apache.org>
AuthorDate: Sat Dec 7 11:16:25 2019 +0530
SOLR-11706: minor refactors
* rename stdDev, variance methods to reflect the functionality
* add util functions to compute corrected stdDev and variance
* use DocValuesIterator#advanceExact to check if values exists for the doc
---
.../java/org/apache/solr/search/facet/AggUtil.java | 24 +++++++++++----
.../org/apache/solr/search/facet/DocValuesAcc.java | 34 ++++++++++------------
.../java/org/apache/solr/search/facet/HLLAgg.java | 12 ++++++--
.../org/apache/solr/search/facet/MinMaxAgg.java | 5 ++--
.../apache/solr/search/facet/PercentileAgg.java | 9 +++---
.../java/org/apache/solr/search/facet/SlotAcc.java | 4 +--
.../org/apache/solr/search/facet/StddevAgg.java | 8 ++---
.../org/apache/solr/search/facet/UniqueAgg.java | 12 ++++++--
.../org/apache/solr/search/facet/VarianceAgg.java | 8 ++---
9 files changed, 69 insertions(+), 47 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/search/facet/AggUtil.java b/solr/core/src/java/org/apache/solr/search/facet/AggUtil.java
index 3370fd9..dfd9994 100644
--- a/solr/core/src/java/org/apache/solr/search/facet/AggUtil.java
+++ b/solr/core/src/java/org/apache/solr/search/facet/AggUtil.java
@@ -34,20 +34,32 @@ public class AggUtil {
/**
* Computes and returns uncorrected standard deviation for given values
*/
+ public static double uncorrectedStdDev(double sumSq, double sum, long count) {
+ // todo: should we return NAN when count==0?
+ return count == 0 ? 0 : Math.sqrt((sumSq / count) - Math.pow(sum / count, 2));
+ }
+
+ /**
+ * Computes and returns corrected standard deviation for given values
+ */
public static double stdDev(double sumSq, double sum, long count) {
- // todo: switch to corrected stddev SOLR-11725
// todo: should we return NAN when count==0?
- double val = count == 0 ? 0 : Math.sqrt((sumSq / count) - Math.pow(sum / count, 2));
- return val;
+ return count == 0 ? 0 : Math.sqrt(((count * sumSq) - (sum * sum)) / (count * (count - 1.0D)));
}
/**
* Computes and returns uncorrected variance for given values
*/
+ public static double uncorrectedVariance(double sumSq, double sum, long count) {
+ // todo: should we return NAN when count==0?
+ return count == 0 ? 0 : (sumSq / count) - Math.pow(sum / count, 2);
+ }
+
+ /**
+ * Computes and returns corrected variance for given values
+ */
public static double variance(double sumSq, double sum, long count) {
- // todo: switch to corrected variance SOLR-11725
// todo: should we return NAN when count==0?
- double val = count == 0 ? 0 : (sumSq / count) - Math.pow(sum / count, 2);
- return val;
+ return count == 0 ? 0 : ((count * sumSq) - (sum * sum)) / (count * (count - 1.0D));
}
}
diff --git a/solr/core/src/java/org/apache/solr/search/facet/DocValuesAcc.java b/solr/core/src/java/org/apache/solr/search/facet/DocValuesAcc.java
index 2bd07a4..bd52c42 100644
--- a/solr/core/src/java/org/apache/solr/search/facet/DocValuesAcc.java
+++ b/solr/core/src/java/org/apache/solr/search/facet/DocValuesAcc.java
@@ -29,7 +29,6 @@ import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedNumericDocValues;
import org.apache.lucene.index.SortedSetDocValues;
-import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.NumericUtils;
import org.apache.solr.schema.SchemaField;
@@ -47,21 +46,18 @@ public abstract class DocValuesAcc extends SlotAcc {
@Override
public void collect(int doc, int slot, IntFunction<SlotContext> slotContext) throws IOException {
- int valuesDocID = docIdSetIterator().docID();
- if (valuesDocID < doc) {
- valuesDocID = docIdSetIterator().advance(doc);
+ if (advanceExact(doc)) {
+ collectValues(doc, slot);
}
- if (valuesDocID > doc) {
- // missing
- return;
- }
- assert valuesDocID == doc;
- collectValues(doc, slot);
}
protected abstract void collectValues(int doc, int slot) throws IOException;
- protected abstract DocIdSetIterator docIdSetIterator();
+ /**
+ * Wrapper to {@code org.apache.lucene.index.DocValuesIterator#advanceExact(int)}
+ * returns whether or not given {@code doc} has value
+ */
+ protected abstract boolean advanceExact(int doc) throws IOException;
}
/**
@@ -81,8 +77,8 @@ abstract class NumericDVAcc extends DocValuesAcc {
}
@Override
- protected DocIdSetIterator docIdSetIterator() {
- return values;
+ protected boolean advanceExact(int doc) throws IOException {
+ return values.advanceExact(doc);
}
}
@@ -103,8 +99,8 @@ abstract class SortedNumericDVAcc extends DocValuesAcc {
}
@Override
- protected DocIdSetIterator docIdSetIterator() {
- return values;
+ protected boolean advanceExact(int doc) throws IOException {
+ return values.advanceExact(doc);
}
}
@@ -272,8 +268,8 @@ abstract class SortedDVAcc extends DocValuesAcc {
}
@Override
- protected DocIdSetIterator docIdSetIterator() {
- return values;
+ protected boolean advanceExact(int doc) throws IOException {
+ return values.advanceExact(doc);
}
}
@@ -294,8 +290,8 @@ abstract class SortedSetDVAcc extends DocValuesAcc {
}
@Override
- protected DocIdSetIterator docIdSetIterator() {
- return values;
+ protected boolean advanceExact(int doc) throws IOException {
+ return values.advanceExact(doc);
}
}
diff --git a/solr/core/src/java/org/apache/solr/search/facet/HLLAgg.java b/solr/core/src/java/org/apache/solr/search/facet/HLLAgg.java
index 61c7cde..d336932 100644
--- a/solr/core/src/java/org/apache/solr/search/facet/HLLAgg.java
+++ b/solr/core/src/java/org/apache/solr/search/facet/HLLAgg.java
@@ -190,12 +190,16 @@ public class HLLAgg extends StrAggValueSource {
values = DocValues.getNumeric(readerContext.reader(), sf.getName());
}
- @Override
protected DocIdSetIterator docIdSetIterator() {
return values;
}
@Override
+ protected boolean advanceExact(int doc) throws IOException {
+ return values.advanceExact(doc);
+ }
+
+ @Override
protected void collectValues(int doc, HLL hll) throws IOException {
long val = values.longValue();
long hash = Hash.fmix64(val);
@@ -216,12 +220,16 @@ public class HLLAgg extends StrAggValueSource {
values = DocValues.getSortedNumeric(readerContext.reader(), sf.getName());
}
- @Override
protected DocIdSetIterator docIdSetIterator() {
return values;
}
@Override
+ protected boolean advanceExact(int doc) throws IOException {
+ return values.advanceExact(doc);
+ }
+
+ @Override
protected void collectValues(int doc, HLL hll) throws IOException {
for (int i = 0, count = values.docValueCount(); i < count; i++) {
// duplicates may be produced for a single doc, but won't matter here.
diff --git a/solr/core/src/java/org/apache/solr/search/facet/MinMaxAgg.java b/solr/core/src/java/org/apache/solr/search/facet/MinMaxAgg.java
index 0d7d863..a62503a 100644
--- a/solr/core/src/java/org/apache/solr/search/facet/MinMaxAgg.java
+++ b/solr/core/src/java/org/apache/solr/search/facet/MinMaxAgg.java
@@ -27,7 +27,6 @@ import org.apache.lucene.index.OrdinalMap;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.queries.function.ValueSource;
-import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.LongValues;
@@ -522,8 +521,8 @@ public class MinMaxAgg extends SimpleAggValueSource {
}
@Override
- protected DocIdSetIterator docIdSetIterator() {
- return subDv;
+ protected boolean advanceExact(int doc) throws IOException {
+ return subDv.advanceExact(doc);
}
}
}
diff --git a/solr/core/src/java/org/apache/solr/search/facet/PercentileAgg.java b/solr/core/src/java/org/apache/solr/search/facet/PercentileAgg.java
index c298fd1..24e3273 100644
--- a/solr/core/src/java/org/apache/solr/search/facet/PercentileAgg.java
+++ b/solr/core/src/java/org/apache/solr/search/facet/PercentileAgg.java
@@ -31,7 +31,6 @@ import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.SortedNumericDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.queries.function.ValueSource;
-import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.NumericUtils;
import org.apache.solr.common.SolrException;
@@ -313,8 +312,8 @@ public class PercentileAgg extends SimpleAggValueSource {
}
@Override
- protected DocIdSetIterator docIdSetIterator() {
- return values;
+ protected boolean advanceExact(int doc) throws IOException {
+ return values.advanceExact(doc);
}
/**
@@ -366,8 +365,8 @@ public class PercentileAgg extends SimpleAggValueSource {
}
@Override
- protected DocIdSetIterator docIdSetIterator() {
- return values;
+ protected boolean advanceExact(int doc) throws IOException {
+ return values.advanceExact(doc);
}
}
diff --git a/solr/core/src/java/org/apache/solr/search/facet/SlotAcc.java b/solr/core/src/java/org/apache/solr/search/facet/SlotAcc.java
index 2a92348..ffdbe79 100644
--- a/solr/core/src/java/org/apache/solr/search/facet/SlotAcc.java
+++ b/solr/core/src/java/org/apache/solr/search/facet/SlotAcc.java
@@ -484,7 +484,7 @@ class VarianceSlotAcc extends DoubleFuncSlotAcc {
}
private double variance(int slot) {
- return AggUtil.variance(result[slot], sum[slot], counts[slot]); // calc once and cache in result?
+ return AggUtil.uncorrectedVariance(result[slot], sum[slot], counts[slot]); // calc once and cache in result?
}
@Override
@@ -541,7 +541,7 @@ class StddevSlotAcc extends DoubleFuncSlotAcc {
}
private double stdDev(int slot) {
- return AggUtil.stdDev(result[slot], sum[slot], counts[slot]); // calc once and cache in result?
+ return AggUtil.uncorrectedStdDev(result[slot], sum[slot], counts[slot]); // calc once and cache in result?
}
@Override
diff --git a/solr/core/src/java/org/apache/solr/search/facet/StddevAgg.java b/solr/core/src/java/org/apache/solr/search/facet/StddevAgg.java
index d7237d3..8481973 100644
--- a/solr/core/src/java/org/apache/solr/search/facet/StddevAgg.java
+++ b/solr/core/src/java/org/apache/solr/search/facet/StddevAgg.java
@@ -86,7 +86,7 @@ public class StddevAgg extends SimpleAggValueSource {
@Override
protected double getDouble() {
- return AggUtil.stdDev(sumSq, sum, count);
+ return AggUtil.uncorrectedStdDev(sumSq, sum, count);
}
}
@@ -98,7 +98,7 @@ public class StddevAgg extends SimpleAggValueSource {
@Override
protected double computeVal(int slot) {
- return AggUtil.stdDev(result[slot], sum[slot], counts[slot]); // calc once and cache in result?
+ return AggUtil.uncorrectedStdDev(result[slot], sum[slot], counts[slot]); // calc once and cache in result?
}
}
@@ -110,7 +110,7 @@ public class StddevAgg extends SimpleAggValueSource {
@Override
protected double computeVal(int slot) {
- return AggUtil.stdDev(result[slot], sum[slot], counts[slot]); // calc once and cache in result?
+ return AggUtil.uncorrectedStdDev(result[slot], sum[slot], counts[slot]); // calc once and cache in result?
}
}
@@ -122,7 +122,7 @@ public class StddevAgg extends SimpleAggValueSource {
@Override
protected double computeVal(int slot) {
- return AggUtil.stdDev(result[slot], sum[slot], counts[slot]); // calc once and cache in result?
+ return AggUtil.uncorrectedStdDev(result[slot], sum[slot], counts[slot]); // calc once and cache in result?
}
}
}
diff --git a/solr/core/src/java/org/apache/solr/search/facet/UniqueAgg.java b/solr/core/src/java/org/apache/solr/search/facet/UniqueAgg.java
index aeb7197..743d6f6 100644
--- a/solr/core/src/java/org/apache/solr/search/facet/UniqueAgg.java
+++ b/solr/core/src/java/org/apache/solr/search/facet/UniqueAgg.java
@@ -205,12 +205,16 @@ public class UniqueAgg extends StrAggValueSource {
super(fcontext, field, numSlots);
}
- @Override
protected DocIdSetIterator docIdSetIterator() {
return values;
}
@Override
+ protected boolean advanceExact(int doc) throws IOException {
+ return values.advanceExact(doc);
+ }
+
+ @Override
public void setNextReader(LeafReaderContext readerContext) throws IOException {
values = DocValues.getNumeric(readerContext.reader(), sf.getName());
}
@@ -228,12 +232,16 @@ public class UniqueAgg extends StrAggValueSource {
super(fcontext, field, numSlots);
}
- @Override
protected DocIdSetIterator docIdSetIterator() {
return values;
}
@Override
+ protected boolean advanceExact(int doc) throws IOException {
+ return values.advanceExact(doc);
+ }
+
+ @Override
public void setNextReader(LeafReaderContext readerContext) throws IOException {
values = DocValues.getSortedNumeric(readerContext.reader(), sf.getName());
}
diff --git a/solr/core/src/java/org/apache/solr/search/facet/VarianceAgg.java b/solr/core/src/java/org/apache/solr/search/facet/VarianceAgg.java
index f04a073..6a014ca 100644
--- a/solr/core/src/java/org/apache/solr/search/facet/VarianceAgg.java
+++ b/solr/core/src/java/org/apache/solr/search/facet/VarianceAgg.java
@@ -85,7 +85,7 @@ public class VarianceAgg extends SimpleAggValueSource {
@Override
protected double getDouble() {
- return AggUtil.variance(sumSq, sum, count);
+ return AggUtil.uncorrectedVariance(sumSq, sum, count);
}
}
@@ -97,7 +97,7 @@ public class VarianceAgg extends SimpleAggValueSource {
@Override
protected double computeVal(int slot) {
- return AggUtil.variance(result[slot], sum[slot], counts[slot]); // calc once and cache in result?
+ return AggUtil.uncorrectedVariance(result[slot], sum[slot], counts[slot]); // calc once and cache in result?
}
}
@@ -109,7 +109,7 @@ public class VarianceAgg extends SimpleAggValueSource {
@Override
protected double computeVal(int slot) {
- return AggUtil.variance(result[slot], sum[slot], counts[slot]); // calc once and cache in result?
+ return AggUtil.uncorrectedVariance(result[slot], sum[slot], counts[slot]); // calc once and cache in result?
}
}
@@ -121,7 +121,7 @@ public class VarianceAgg extends SimpleAggValueSource {
@Override
protected double computeVal(int slot) {
- return AggUtil.variance(result[slot], sum[slot], counts[slot]); // calc once and cache in result?
+ return AggUtil.uncorrectedVariance(result[slot], sum[slot], counts[slot]); // calc once and cache in result?
}
}
}