You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mu...@apache.org on 2020/03/19 13:52:16 UTC
[lucene-solr] branch master updated (126e4a6 -> 78e670f)
This is an automated email from the ASF dual-hosted git repository.
munendrasn pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git.
from 126e4a6 LUCENE-9283: Exclude DelimitedBoostTokenFilter from TestRandomChains
new e9d6c24 SOLR-11725: use corrected sample formula to calc stdDev in JSON facets
new 78e670f SOLR-14012: return long from unique and hll even for standalone
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
solr/CHANGES.txt | 5 +
.../java/org/apache/solr/search/facet/AggUtil.java | 8 +-
.../java/org/apache/solr/search/facet/HLLAgg.java | 17 +---
.../java/org/apache/solr/search/facet/SlotAcc.java | 4 +-
.../org/apache/solr/search/facet/StddevAgg.java | 8 +-
.../org/apache/solr/search/facet/UniqueAgg.java | 37 +++----
.../apache/solr/search/facet/UniqueBlockAgg.java | 7 +-
.../apache/solr/search/facet/UniqueSlotAcc.java | 22 +++--
.../org/apache/solr/search/facet/VarianceAgg.java | 8 +-
.../solr/handler/component/StatsComponentTest.java | 110 +++++++++++++--------
...stributedFacetSimpleRefinementLongTailTest.java | 24 ++---
.../apache/solr/search/facet/TestJsonFacets.java | 20 +++-
.../src/major-changes-in-solr-9.adoc | 6 ++
13 files changed, 163 insertions(+), 113 deletions(-)
[lucene-solr] 02/02: SOLR-14012: return long from unique and hll
even for standalone
Posted by mu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
munendrasn pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 78e670f19ec253fb5e9dc3980986feb501762995
Author: Munendra S N <mu...@apache.org>
AuthorDate: Thu Mar 19 19:07:47 2020 +0530
SOLR-14012: return long from unique and hll even for standalone
---
solr/CHANGES.txt | 2 +
.../java/org/apache/solr/search/facet/HLLAgg.java | 17 +---
.../org/apache/solr/search/facet/UniqueAgg.java | 37 ++++----
.../apache/solr/search/facet/UniqueBlockAgg.java | 7 +-
.../apache/solr/search/facet/UniqueSlotAcc.java | 22 +++--
.../solr/handler/component/StatsComponentTest.java | 101 +++++++++++++--------
.../src/major-changes-in-solr-9.adoc | 3 +
7 files changed, 113 insertions(+), 76 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 1f4a5f4..b18d9af 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -45,6 +45,8 @@ Other Changes
* SOLR-14197: SolrResourceLoader: remove deprecated methods and do other improvements. (David Smiley)
+* SOLR-14012: Return long value for unique and hll aggregations irrespective of shard count (Munendra S N, hossman)
+
================== 8.6.0 ==================
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.
diff --git a/solr/core/src/java/org/apache/solr/search/facet/HLLAgg.java b/solr/core/src/java/org/apache/solr/search/facet/HLLAgg.java
index d336932..32c1b14 100644
--- a/solr/core/src/java/org/apache/solr/search/facet/HLLAgg.java
+++ b/solr/core/src/java/org/apache/solr/search/facet/HLLAgg.java
@@ -22,7 +22,6 @@ import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.SortedNumericDocValues;
-import org.apache.lucene.search.DocIdSetIterator;
import org.apache.solr.common.util.Hash;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.schema.SchemaField;
@@ -30,7 +29,7 @@ import org.apache.solr.util.hll.HLL;
import org.apache.solr.util.hll.HLLType;
public class HLLAgg extends StrAggValueSource {
- public static Integer NO_VALUES = 0;
+ public static Long NO_VALUES = 0L;
protected HLLFactory factory;
@@ -156,9 +155,9 @@ public class HLLAgg extends StrAggValueSource {
return getCardinality(slot);
}
- private int getCardinality(int slot) {
+ private long getCardinality(int slot) {
HLL set = sets[slot];
- return set==null ? 0 : (int)set.cardinality();
+ return set == null ? 0 : set.cardinality();
}
public Object getShardValue(int slot) throws IOException {
@@ -172,7 +171,7 @@ public class HLLAgg extends StrAggValueSource {
@Override
public int compare(int slotA, int slotB) {
- return getCardinality(slotA) - getCardinality(slotB);
+ return Long.compare(getCardinality(slotA), getCardinality(slotB));
}
}
@@ -190,10 +189,6 @@ public class HLLAgg extends StrAggValueSource {
values = DocValues.getNumeric(readerContext.reader(), sf.getName());
}
- protected DocIdSetIterator docIdSetIterator() {
- return values;
- }
-
@Override
protected boolean advanceExact(int doc) throws IOException {
return values.advanceExact(doc);
@@ -220,10 +215,6 @@ public class HLLAgg extends StrAggValueSource {
values = DocValues.getSortedNumeric(readerContext.reader(), sf.getName());
}
- protected DocIdSetIterator docIdSetIterator() {
- return values;
- }
-
@Override
protected boolean advanceExact(int doc) throws IOException {
return values.advanceExact(doc);
diff --git a/solr/core/src/java/org/apache/solr/search/facet/UniqueAgg.java b/solr/core/src/java/org/apache/solr/search/facet/UniqueAgg.java
index 743d6f6..b31da09 100644
--- a/solr/core/src/java/org/apache/solr/search/facet/UniqueAgg.java
+++ b/solr/core/src/java/org/apache/solr/search/facet/UniqueAgg.java
@@ -26,17 +26,16 @@ import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.SortedNumericDocValues;
-import org.apache.lucene.search.DocIdSetIterator;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.util.LongIterator;
import org.apache.solr.util.LongSet;
public class UniqueAgg extends StrAggValueSource {
- public static String UNIQUE = "unique";
+ public static final String UNIQUE = "unique";
// internal constants used for aggregating values from multiple shards
- static String VALS = "vals";
+ static final String VALS = "vals";
public UniqueAgg(String field) {
super(UNIQUE, field);
@@ -78,11 +77,11 @@ public class UniqueAgg extends StrAggValueSource {
@Override
public void merge(Object facetResult, Context mcontext) {
SimpleOrderedMap map = (SimpleOrderedMap)facetResult;
- long unique = ((Number)map.get("unique")).longValue();
+ long unique = ((Number)map.get(UNIQUE)).longValue();
sumUnique += unique;
int valsListed = 0;
- List vals = (List) map.get("vals");
+ List vals = (List) map.get(VALS);
if (vals != null) {
if (values == null) {
values = new HashSet<>(vals.size()*4);
@@ -158,12 +157,24 @@ public class UniqueAgg extends StrAggValueSource {
if (fcontext.isShard()) {
return getShardValue(slot);
}
- return getCardinality(slot);
+ return getNonShardValue(slot);
}
+ /**
+ * Returns the current slot value as long
+ * This is used to get non-sharded value
+ */
+ private long getNonShardValue(int slot) {
+ return (long) getCardinality(slot);
+ }
+
+ /**
+ * Returns the size of the {@code LongSet} for given slot
+ * If value doesn't exist for slot then returns 0
+ */
private int getCardinality(int slot) {
LongSet set = sets[slot];
- return set==null ? 0 : set.cardinality();
+ return set == null ? 0 : set.cardinality();
}
public Object getShardValue(int slot) throws IOException {
@@ -171,7 +182,7 @@ public class UniqueAgg extends StrAggValueSource {
int unique = getCardinality(slot);
SimpleOrderedMap map = new SimpleOrderedMap();
- map.add("unique", unique);
+ map.add(UNIQUE, unique);
int maxExplicit=100;
// TODO: make configurable
@@ -184,7 +195,7 @@ public class UniqueAgg extends StrAggValueSource {
lst.add( iter.next() );
}
}
- map.add("vals", lst);
+ map.add(VALS, lst);
}
return map;
@@ -205,10 +216,6 @@ public class UniqueAgg extends StrAggValueSource {
super(fcontext, field, numSlots);
}
- protected DocIdSetIterator docIdSetIterator() {
- return values;
- }
-
@Override
protected boolean advanceExact(int doc) throws IOException {
return values.advanceExact(doc);
@@ -232,10 +239,6 @@ public class UniqueAgg extends StrAggValueSource {
super(fcontext, field, numSlots);
}
- protected DocIdSetIterator docIdSetIterator() {
- return values;
- }
-
@Override
protected boolean advanceExact(int doc) throws IOException {
return values.advanceExact(doc);
diff --git a/solr/core/src/java/org/apache/solr/search/facet/UniqueBlockAgg.java b/solr/core/src/java/org/apache/solr/search/facet/UniqueBlockAgg.java
index 77dfa64..165fb8d 100644
--- a/solr/core/src/java/org/apache/solr/search/facet/UniqueBlockAgg.java
+++ b/solr/core/src/java/org/apache/solr/search/facet/UniqueBlockAgg.java
@@ -62,7 +62,12 @@ public abstract class UniqueBlockAgg extends UniqueAgg {
@Override
public Object getValue(int slot) throws IOException {
- return counts[slot];
+ return getNonShardValue(slot);
+ }
+
+ @Override
+ public long getNonShardValue(int slot) {
+ return (long)counts[slot];
}
}
diff --git a/solr/core/src/java/org/apache/solr/search/facet/UniqueSlotAcc.java b/solr/core/src/java/org/apache/solr/search/facet/UniqueSlotAcc.java
index 5e85fd1..2987312 100644
--- a/solr/core/src/java/org/apache/solr/search/facet/UniqueSlotAcc.java
+++ b/solr/core/src/java/org/apache/solr/search/facet/UniqueSlotAcc.java
@@ -56,12 +56,22 @@ abstract class UniqueSlotAcc extends SlotAcc {
if (fcontext.isShard()) {
return getShardValue(slot);
}
+ return getNonShardValue(slot);
+ }
+
+ /**
+ * Returns the current slot value as long
+ * This is used to get non-sharded value
+ */
+ public long getNonShardValue(int slot) {
+ long res;
if (counts != null) { // will only be pre-populated if this was used for sorting.
- return counts[slot];
+ res = counts[slot];
+ } else {
+ FixedBitSet bs = arr[slot];
+ res = bs == null ? 0 : bs.cardinality();
}
-
- FixedBitSet bs = arr[slot];
- return bs==null ? 0 : bs.cardinality();
+ return res;
}
private Object getShardHLL(int slot) throws IOException {
@@ -108,8 +118,8 @@ abstract class UniqueSlotAcc extends SlotAcc {
List lst = new ArrayList( Math.min(unique, maxExplicit) );
- long maxOrd = ords.length();
- if (ords != null && ords.length() > 0) {
+ int maxOrd = ords.length();
+ if (maxOrd > 0) {
for (int ord=0; lst.size() < maxExplicit;) {
ord = ords.nextSetBit(ord);
if (ord == DocIdSetIterator.NO_MORE_DOCS) break;
diff --git a/solr/core/src/test/org/apache/solr/handler/component/StatsComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/StatsComponentTest.java
index a2fa1d2..2a445d4 100644
--- a/solr/core/src/test/org/apache/solr/handler/component/StatsComponentTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/component/StatsComponentTest.java
@@ -310,7 +310,7 @@ public class StatsComponentTest extends SolrTestCaseJ4 {
, "//double[@name='sum'][.='9.0']"
, "//long[@name='count'][.='8']"
, "//long[@name='missing'][.='3']"
- , json ? "//int[@name='countDistinct'][.='8']": "//long[@name='countDistinct'][.='8']" // SOLR-11775
+ , "//long[@name='countDistinct'][.='8']"
, json ? "//*" : "count(//arr[@name='distinctValues']/*)=8"
, "//double[@name='sumOfSquares'][.='53101.0']"
, "//double[@name='mean'][.='1.125']"
@@ -325,7 +325,7 @@ public class StatsComponentTest extends SolrTestCaseJ4 {
, "//double[@name='sum'][.='119.0']"
, "//long[@name='count'][.='6']"
, "//long[@name='missing'][.='3']"
- , json? "//int[@name='countDistinct'][.='6']" :"//long[@name='countDistinct'][.='6']" // SOLR-11775
+ , "//long[@name='countDistinct'][.='6']"
, json ? "//*" : "count(//arr[@name='distinctValues']/*)=6"
, "//double[@name='sumOfSquares'][.='43001.0']"
, "//double[@name='mean'][.='19.833333333333332']"
@@ -377,59 +377,82 @@ public class StatsComponentTest extends SolrTestCaseJ4 {
}
}
- assertQ("cardinality"
- , req("q", "*:*", "rows", "0", "stats", "true", "stats.field", "{!cardinality=true}" + f)
- , "//long[@name='cardinality'][.='8']"
- );
- assertQ("json cardinality"
- , req("q", "*:*", "rows", "0", "json.facet", "{cardinality:'hll("+f+")'}")
- , "//int[@name='cardinality'][.='8']" // SOLR-11775
- );
+ // cardinality
+ for (SolrParams baseParams : new SolrParams[] {
+ params("stats.field", "{!cardinality=true}"+f, "stats", "true"),
+ params("json.facet", "{cardinality:'hll("+f+")'}")
+ }) {
+ assertQ("test cardinality",
+ req(baseParams, "q", "*:*", "rows", "0"),
+ "//long[@name='cardinality'][.='8']"
+ );
+ }
}
public void testFieldStatisticsResultsStringField() throws Exception {
+ String f = "active_s";
+
SolrCore core = h.getCore();
- assertU(adoc("id", "1", "active_s", "string1"));
- assertU(adoc("id", "2", "active_s", "string2"));
- assertU(adoc("id", "3", "active_s", "string3"));
+ assertU(adoc("id", "1", f, "string1"));
+ assertU(adoc("id", "2", f, "string2"));
+ assertU(adoc("id", "3", f, "string3"));
assertU(adoc("id", "4"));
assertU(commit());
Map<String, String> args = new HashMap<>();
args.put(CommonParams.Q, "*:*");
args.put(StatsParams.STATS, "true");
- args.put(StatsParams.STATS_FIELD, "active_s");
- args.put("f.active_s.stats.calcdistinct","true");
+ args.put(StatsParams.STATS_FIELD, f);
+ args.put("f." + f +".stats.calcdistinct","true");
args.put("indent", "true");
SolrQueryRequest req = new LocalSolrQueryRequest(core, new MapSolrParams(args));
- assertQ("test string statistics values", req,
- "//str[@name='min'][.='string1']",
- "//str[@name='max'][.='string3']",
- "//long[@name='count'][.='3']",
- "//long[@name='missing'][.='1']",
- "//long[@name='countDistinct'][.='3']",
- "count(//arr[@name='distinctValues']/str)=3");
- assertQ("test string cardinality"
- , req("q", "*:*",
- "rows", "0",
- "stats","true",
- "stats.field","{!cardinality=true}active_s")
- , "//long[@name='cardinality'][.='3']");
+ for (SolrParams baseParams : new SolrParams[] {
+ params("stats.field", f, "stats", "true", "f." + f +".stats.calcdistinct","true"),
+ params("json.facet", // note: no distinctValues support
+ "{min:'min("+f+")',count:'countvals("+f+")',missing:'missing("+f+")',max:'max("+f+")', " +
+ " countDistinct:'unique("+f+")'}")
+ }) {
+ final boolean json = (null != baseParams.get("json.facet"));
+ assertQ("test string statistics values", req(baseParams, "q", "*:*", "rows", "0"),
+ "//str[@name='min'][.='string1']",
+ "//str[@name='max'][.='string3']",
+ "//long[@name='count'][.='3']",
+ "//long[@name='missing'][.='1']",
+ "//long[@name='countDistinct'][.='3']",
+ json ? "//*": "count(//arr[@name='distinctValues']/str)=3"); // SOLR-14011
+ }
+
+ // string field cardinality
+ for (SolrParams baseParams : new SolrParams[] {
+ params("stats.field", "{!cardinality=true}"+f, "stats", "true"),
+ params("json.facet", "{cardinality:'hll("+f+")'}")
+ }) {
+ assertQ("test string cardinality",
+ req(baseParams, "q", "*:*", "rows", "0"),
+ "//long[@name='cardinality'][.='3']"
+ );
+ }
+ String strFunc = "strdist(\"string22\","+ f +",edit)";
// stats over a string function
- assertQ("strdist func stats",
- req("q", "*:*",
- "stats","true",
- "stats.field","{!func}strdist('string22',active_s,edit)")
- , "//double[@name='min'][.='0.75']"
- , "//double[@name='max'][.='0.875']"
- , "//double[@name='sum'][.='2.375']"
- , "//long[@name='count'][.='3']"
- ,"//long[@name='missing'][.='1']"
- );
-
+ for (SolrParams baseParams : new SolrParams[] {
+ params("stats.field", "{!func}"+strFunc, "stats", "true"),
+ params("json.facet", // note: no function support for unique
+ "{min:'min("+strFunc+")',count:'countvals("+strFunc+")',missing:'missing("+strFunc+")'," +
+ "sum:'sum("+ strFunc +")', max:'max("+strFunc+")'}")
+ }) {
+ final boolean json = (null != baseParams.get("json.facet"));
+ assertQ("strdist func stats",
+ req(baseParams, "q", "*:*", "rows", "0")
+ , "//double[@name='min'][.='0.75']"
+ , "//double[@name='max'][.='0.875']"
+ , "//double[@name='sum'][.='2.375']"
+ , json? "//*": "//long[@name='count'][.='3']" // SOLR-14010
+ ,"//long[@name='missing'][.='1']"
+ );
+ }
}
public void testFieldStatisticsResultsDateField() throws Exception {
diff --git a/solr/solr-ref-guide/src/major-changes-in-solr-9.adoc b/solr/solr-ref-guide/src/major-changes-in-solr-9.adoc
index f94763f..d1b2e8e 100644
--- a/solr/solr-ref-guide/src/major-changes-in-solr-9.adoc
+++ b/solr/solr-ref-guide/src/major-changes-in-solr-9.adoc
@@ -93,6 +93,9 @@ _(raw; not yet edited)_
* SOLR-11725: JSON aggregations uses corrected sample formula to compute standard deviation and variance.
The computation of stdDev and variance in JSON aggregation is same as StatsComponent. (hossman, Munendra S N, yonik)
+
+* SOLR-14012: unique and hll aggregations always returns long value irrespective of standalone or solcloud
+ (Munendra S N, hossman)
=== Upgrade Prerequisites in Solr 9
[lucene-solr] 01/02: SOLR-11725: use corrected sample formula to
calc stdDev in JSON facets
Posted by mu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
munendrasn pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit e9d6c24fb786291fe1e0750acb371774bd09cc3d
Author: Munendra S N <mu...@apache.org>
AuthorDate: Thu Mar 19 18:57:56 2020 +0530
SOLR-11725: use corrected sample formula to calc stdDev in JSON facets
* Both stdDev and variance uses corrected sample formula to compute
the values. This is similar to StatsComponent
---
solr/CHANGES.txt | 3 +++
.../java/org/apache/solr/search/facet/AggUtil.java | 8 ++++----
.../java/org/apache/solr/search/facet/SlotAcc.java | 4 ++--
.../org/apache/solr/search/facet/StddevAgg.java | 8 ++++----
.../org/apache/solr/search/facet/VarianceAgg.java | 8 ++++----
.../solr/handler/component/StatsComponentTest.java | 9 ++++++--
...stributedFacetSimpleRefinementLongTailTest.java | 24 ++++++++--------------
.../apache/solr/search/facet/TestJsonFacets.java | 20 +++++++++++++-----
.../src/major-changes-in-solr-9.adoc | 3 +++
9 files changed, 50 insertions(+), 37 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 8b4ff38..1f4a5f4 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -18,6 +18,9 @@ Improvements
* SOLR-14223: PKI Auth can bootstrap from existing key files instead of creating new keys on startup (Mike Drob)
+* SOLR-11725: Use corrected sample formula for computing stdDev and variance in JSON aggregations
+ (hossman, Munendra S N, yonik)
+
Other Changes
----------------------
* SOLR-10288: Remove non-minified JavaScript from the webapp. (Erik Hatcher, marcussorealheis)
diff --git a/solr/core/src/java/org/apache/solr/search/facet/AggUtil.java b/solr/core/src/java/org/apache/solr/search/facet/AggUtil.java
index dfd9994..24391dd 100644
--- a/solr/core/src/java/org/apache/solr/search/facet/AggUtil.java
+++ b/solr/core/src/java/org/apache/solr/search/facet/AggUtil.java
@@ -43,8 +43,8 @@ public class AggUtil {
* Computes and returns corrected standard deviation for given values
*/
public static double stdDev(double sumSq, double sum, long count) {
- // todo: should we return NAN when count==0?
- return count == 0 ? 0 : Math.sqrt(((count * sumSq) - (sum * sum)) / (count * (count - 1.0D)));
+ // todo: should we return NAN when count==0 or count==1?
+ return count <= 1 ? 0.0d : Math.sqrt(((count * sumSq) - (sum * sum)) / (count * (count - 1.0D)));
}
/**
@@ -59,7 +59,7 @@ public class AggUtil {
* Computes and returns corrected variance for given values
*/
public static double variance(double sumSq, double sum, long count) {
- // todo: should we return NAN when count==0?
- return count == 0 ? 0 : ((count * sumSq) - (sum * sum)) / (count * (count - 1.0D));
+ // todo: should we return NAN when count==0 or count==1?
+ return count <= 1 ? 0.0d : ((count * sumSq) - (sum * sum)) / (count * (count - 1.0D));
}
}
diff --git a/solr/core/src/java/org/apache/solr/search/facet/SlotAcc.java b/solr/core/src/java/org/apache/solr/search/facet/SlotAcc.java
index ffdbe79..2a92348 100644
--- a/solr/core/src/java/org/apache/solr/search/facet/SlotAcc.java
+++ b/solr/core/src/java/org/apache/solr/search/facet/SlotAcc.java
@@ -484,7 +484,7 @@ class VarianceSlotAcc extends DoubleFuncSlotAcc {
}
private double variance(int slot) {
- return AggUtil.uncorrectedVariance(result[slot], sum[slot], counts[slot]); // calc once and cache in result?
+ return AggUtil.variance(result[slot], sum[slot], counts[slot]); // calc once and cache in result?
}
@Override
@@ -541,7 +541,7 @@ class StddevSlotAcc extends DoubleFuncSlotAcc {
}
private double stdDev(int slot) {
- return AggUtil.uncorrectedStdDev(result[slot], sum[slot], counts[slot]); // calc once and cache in result?
+ return AggUtil.stdDev(result[slot], sum[slot], counts[slot]); // calc once and cache in result?
}
@Override
diff --git a/solr/core/src/java/org/apache/solr/search/facet/StddevAgg.java b/solr/core/src/java/org/apache/solr/search/facet/StddevAgg.java
index 8481973..d7237d3 100644
--- a/solr/core/src/java/org/apache/solr/search/facet/StddevAgg.java
+++ b/solr/core/src/java/org/apache/solr/search/facet/StddevAgg.java
@@ -86,7 +86,7 @@ public class StddevAgg extends SimpleAggValueSource {
@Override
protected double getDouble() {
- return AggUtil.uncorrectedStdDev(sumSq, sum, count);
+ return AggUtil.stdDev(sumSq, sum, count);
}
}
@@ -98,7 +98,7 @@ public class StddevAgg extends SimpleAggValueSource {
@Override
protected double computeVal(int slot) {
- return AggUtil.uncorrectedStdDev(result[slot], sum[slot], counts[slot]); // calc once and cache in result?
+ return AggUtil.stdDev(result[slot], sum[slot], counts[slot]); // calc once and cache in result?
}
}
@@ -110,7 +110,7 @@ public class StddevAgg extends SimpleAggValueSource {
@Override
protected double computeVal(int slot) {
- return AggUtil.uncorrectedStdDev(result[slot], sum[slot], counts[slot]); // calc once and cache in result?
+ return AggUtil.stdDev(result[slot], sum[slot], counts[slot]); // calc once and cache in result?
}
}
@@ -122,7 +122,7 @@ public class StddevAgg extends SimpleAggValueSource {
@Override
protected double computeVal(int slot) {
- return AggUtil.uncorrectedStdDev(result[slot], sum[slot], counts[slot]); // calc once and cache in result?
+ return AggUtil.stdDev(result[slot], sum[slot], counts[slot]); // calc once and cache in result?
}
}
}
diff --git a/solr/core/src/java/org/apache/solr/search/facet/VarianceAgg.java b/solr/core/src/java/org/apache/solr/search/facet/VarianceAgg.java
index 6a014ca..f04a073 100644
--- a/solr/core/src/java/org/apache/solr/search/facet/VarianceAgg.java
+++ b/solr/core/src/java/org/apache/solr/search/facet/VarianceAgg.java
@@ -85,7 +85,7 @@ public class VarianceAgg extends SimpleAggValueSource {
@Override
protected double getDouble() {
- return AggUtil.uncorrectedVariance(sumSq, sum, count);
+ return AggUtil.variance(sumSq, sum, count);
}
}
@@ -97,7 +97,7 @@ public class VarianceAgg extends SimpleAggValueSource {
@Override
protected double computeVal(int slot) {
- return AggUtil.uncorrectedVariance(result[slot], sum[slot], counts[slot]); // calc once and cache in result?
+ return AggUtil.variance(result[slot], sum[slot], counts[slot]); // calc once and cache in result?
}
}
@@ -109,7 +109,7 @@ public class VarianceAgg extends SimpleAggValueSource {
@Override
protected double computeVal(int slot) {
- return AggUtil.uncorrectedVariance(result[slot], sum[slot], counts[slot]); // calc once and cache in result?
+ return AggUtil.variance(result[slot], sum[slot], counts[slot]); // calc once and cache in result?
}
}
@@ -121,7 +121,7 @@ public class VarianceAgg extends SimpleAggValueSource {
@Override
protected double computeVal(int slot) {
- return AggUtil.uncorrectedVariance(result[slot], sum[slot], counts[slot]); // calc once and cache in result?
+ return AggUtil.variance(result[slot], sum[slot], counts[slot]); // calc once and cache in result?
}
}
}
diff --git a/solr/core/src/test/org/apache/solr/handler/component/StatsComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/StatsComponentTest.java
index 4df839b..a2fa1d2 100644
--- a/solr/core/src/test/org/apache/solr/handler/component/StatsComponentTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/component/StatsComponentTest.java
@@ -314,7 +314,7 @@ public class StatsComponentTest extends SolrTestCaseJ4 {
, json ? "//*" : "count(//arr[@name='distinctValues']/*)=8"
, "//double[@name='sumOfSquares'][.='53101.0']"
, "//double[@name='mean'][.='1.125']"
- ,json ? "//*" : "//double[@name='stddev'][.='87.08852228787508']" // SOLR-11725
+ ,"//double[@name='stddev'][.='87.08852228787508']"
);
assertQ("test statistics values w/fq",
@@ -329,8 +329,13 @@ public class StatsComponentTest extends SolrTestCaseJ4 {
, json ? "//*" : "count(//arr[@name='distinctValues']/*)=6"
, "//double[@name='sumOfSquares'][.='43001.0']"
, "//double[@name='mean'][.='19.833333333333332']"
- , json ? "//*" : "//double[@name='stddev'][.='90.15634568163611']" // SOLR-11725
+ ,"//double[@name='stddev'][.='90.15634568163611']"
);
+
+ assertQ("test stdDev",
+ req(baseParams, "q", "id:5", "rows", "0")
+ ,"//double[@name='stddev'][.='0.0']"
+ );
if (!json) { // checking stats.facet makes no sense for json faceting
assertQ("test stats.facet (using boolean facet field)",
diff --git a/solr/core/src/test/org/apache/solr/search/facet/DistributedFacetSimpleRefinementLongTailTest.java b/solr/core/src/test/org/apache/solr/search/facet/DistributedFacetSimpleRefinementLongTailTest.java
index 0ed8659..b611165 100644
--- a/solr/core/src/test/org/apache/solr/search/facet/DistributedFacetSimpleRefinementLongTailTest.java
+++ b/solr/core/src/test/org/apache/solr/search/facet/DistributedFacetSimpleRefinementLongTailTest.java
@@ -235,10 +235,8 @@ public class DistributedFacetSimpleRefinementLongTailTest extends BaseDistribute
assertEquals(1.0D, bucket.get("percentile"));
assertEquals(0.475247524752475D, (double) bucket.get("avg"), 0.1E-7);
assertEquals(54.0D, (double) bucket.get("sumsq"), 0.1E-7);
- // assertEquals(0.55846323792D, (double) bucket.get("stddev"), 0.1E-7); // TODO: SOLR-11725
- // assertEquals(0.3118811881D, (double) bucket.get("variance"), 0.1E-7); // TODO: SOLR-11725
- assertEquals(0.55569169111D, (double) bucket.get("stddev"), 0.1E-7); // json.facet is using the "uncorrected stddev"
- assertEquals(0.3087932556D, (double) bucket.get("variance"), 0.1E-7); // json.facet is using the "uncorrected variance"
+ assertEquals(0.55846323792D, (double) bucket.get("stddev"), 0.1E-7);
+ assertEquals(0.3118811881D, (double) bucket.get("variance"), 0.1E-7);
assertEquals(3L, bucket.get("unique"));
assertEquals(3L, bucket.get("hll"));
}
@@ -399,10 +397,8 @@ public class DistributedFacetSimpleRefinementLongTailTest extends BaseDistribute
assertEquals(483.70000000000016D, (double)aaa0_Bucket.get("percentile"), 0.1E-7);
assertEquals(115.5D, (double) aaa0_Bucket.get("avg"), 0.1E-7);
assertEquals(1.674585E7D, (double) aaa0_Bucket.get("sumsq"), 0.1E-7);
- // assertEquals(206.4493184076D, (double) aaa0_Bucket.get("stddev"), 0.1E-7); // TODO: SOLR-11725
- // assertEquals(42621.32107023412D, (double) aaa0_Bucket.get("variance"), 0.1E-7); // TODO: SOLR-11725
- assertEquals(206.1049489944D, (double) aaa0_Bucket.get("stddev"), 0.1E-7); // json.facet is using the "uncorrected stddev"
- assertEquals(42479.25D, (double) aaa0_Bucket.get("variance"), 0.1E-7); // json.facet is using the "uncorrected variance"
+ assertEquals(206.4493184076D, (double) aaa0_Bucket.get("stddev"), 0.1E-7);
+ assertEquals(42621.32107023412D, (double) aaa0_Bucket.get("variance"), 0.1E-7);
assertEquals(284L, aaa0_Bucket.get("unique"));
assertEquals(284L, aaa0_Bucket.get("hll"));
@@ -418,10 +414,8 @@ public class DistributedFacetSimpleRefinementLongTailTest extends BaseDistribute
assertEquals(1980.0D, tail_Bucket.get("sum"));
assertEquals(22.0D, (double) tail_Bucket.get("avg"), 0.1E-7);
assertEquals(58740.0D, (double) tail_Bucket.get("sumsq"), 0.1E-7);
- // assertEquals(13.0599310011D, (double) tail_Bucket.get("stddev"), 0.1E-7); // TODO: SOLR-11725
- // assertEquals(170.5617977535D, (double) tail_Bucket.get("variance"), 0.1E-7); // TODO: SOLR-11725
- assertEquals(12.9871731592D, (double) tail_Bucket.get("stddev"), 0.1E-7); // json.facet is using the "uncorrected stddev"
- assertEquals(168.666666667D, (double) tail_Bucket.get("variance"), 0.1E-7); // json.facet is using the "uncorrected variance"
+ assertEquals(13.0599310011D, (double) tail_Bucket.get("stddev"), 0.1E-7);
+ assertEquals(170.5617977535D, (double) tail_Bucket.get("variance"), 0.1E-7);
assertEquals(45L, tail_Bucket.get("unique"));
assertEquals(45L, tail_Bucket.get("hll"));
@@ -439,10 +433,8 @@ public class DistributedFacetSimpleRefinementLongTailTest extends BaseDistribute
assertEquals(450.0D, tailB_Bucket.get("sum"));
assertEquals(37.5D, (double) tailB_Bucket.get("avg"), 0.1E-7);
assertEquals(16910.0D, (double) tailB_Bucket.get("sumsq"), 0.1E-7);
- // assertEquals(1.78376517D, (double) tailB_Bucket.get("stddev"), 0.1E-7); // TODO: SOLR-11725
- // assertEquals(3.1818181817D, (double) tailB_Bucket.get("variance"), 0.1E-7); // TODO: SOLR-11725
- assertEquals(1.70782513D, (double) tailB_Bucket.get("stddev"), 0.1E-7); // json.facet is using the "uncorrected stddev"
- assertEquals(2.9166666747D, (double) tailB_Bucket.get("variance"), 0.1E-7); // json.facet is using the "uncorrected variance"
+ assertEquals(1.78376517D, (double) tailB_Bucket.get("stddev"), 0.1E-7);
+ assertEquals(3.1818181817D, (double) tailB_Bucket.get("variance"), 0.1E-7);
assertEquals(6L, tailB_Bucket.get("unique"));
assertEquals(6L, tailB_Bucket.get("hll"));
diff --git a/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java b/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java
index dd1cf43..105faf9 100644
--- a/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java
+++ b/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java
@@ -1289,13 +1289,23 @@ public class TestJsonFacets extends SolrTestCaseHS {
", f2:{ 'buckets':[{ val:'B', count:3, x:11.0 }, { val:'A', count:2, x:4.0 }]} " +
", f3:{ 'buckets':[{ val:'A', count:2, x:2 }, { val:'B', count:3, x:2 }]} " +
", f4:{ 'buckets':[{ val:'A', count:2, x:2 }, { val:'B', count:3, x:2 }]} " +
- ", f5:{ 'buckets':[{ val:'B', count:3, x:74.6666666666666 }, { val:'A', count:2, x:1.0 }]} " +
+ ", f5:{ 'buckets':[{ val:'B', count:3, x:112.0 }, { val:'A', count:2, x:2.0 }]} " +
", f6:{ buckets:[{ val:-9.0, count:1, x:1 }]} " +
", f7:{ buckets:[{ val:B, count:3, x:3 },{ val:A, count:2, x:0 }]} " +
", f8:{ buckets:[{ val:A, count:2, x:2 },{ val:B, count:3, x:0 }]} " +
"}"
);
+ // test for stdDev and variance of size 1 and 0
+ client.testJQ(params(p, "q", "id:1", "json.facet", "{n1:'stddev(${num_d})', n2: 'variance(${num_d})'}")
+ , "facets=={ 'count':1, " +
+ " n1:0.0, n2:0.0 }"
+ );
+ client.testJQ(params(p, "q", "id:3", "json.facet", "{n1:'stddev(${num_d})', n2: 'variance(${num_d})'}")
+ , "facets=={ 'count':1, " +
+ " n1:0.0, n2:0.0 }"
+ );
+
// test sorting by stat with function
client.testJQ(params(p, "q", "*:*"
, "json.facet", "{f1:{terms:{${terms} field:'${cat_s}', sort:'n1 desc', facet:{n1:'avg(add(${num_d},${num_d}))'} }}" +
@@ -1808,7 +1818,7 @@ public class TestJsonFacets extends SolrTestCaseHS {
"sum1:3.0, sumsq1:247.0, avg1:0.6, avg2:0.5, mind:-9.0, maxd:11.0" +
", numwhere:2, unique_num_i:4, unique_num_d:5, unique_date:5" +
", where_hll:2, hll_num_i:4, hll_num_d:5, hll_date:5" +
- ", med:2.0, perc:[-9.0,2.0,11.0], variance:49.04, stddev:7.002856560004639" +
+ ", med:2.0, perc:[-9.0,2.0,11.0], variance:61.3, stddev:7.829431652425353" +
", mini:-5, maxi:7, missing:4, vals:2" +
"}"
);
@@ -1824,7 +1834,7 @@ public class TestJsonFacets extends SolrTestCaseHS {
, "facets=={ 'count':6, " +
"sum1:0.0, sumsq1:51.5, avg1:0.0, mind:-5.0, maxd:3.0" +
", mini:-5, maxi:3, mins:'a', maxs:'b'" +
- ", stddev:2.537222891273055, variance:6.4375, median:0.0, perc:[-5.0,2.25,3.0]" +
+ ", stddev:2.712405363721075, variance:7.3571428571, median:0.0, perc:[-5.0,2.25,3.0]" +
"}"
);
@@ -2314,8 +2324,8 @@ public class TestJsonFacets extends SolrTestCaseHS {
", sumd:'sum(${num_d})', avgd:'avg(${num_d})', variance:'variance(${num_d})', stddev:'stddev(${num_d})', missing:'missing(${multi_ss})', vals:'countvals(${multi_ss})'} }}"
)
, "facets=={ 'count':6, " +
- "'f1':{ buckets:[{val:B, count:3, h:2, u:2, mind:-9.0, maxd:11.0, mini:-5, maxi:7, sumd:-3.0, avgd:-1.0, variance:74.66666666666667, stddev:8.640987597877148, missing:0, vals:5}," +
- " {val:A, count:2, h:2, u:2, mind:2.0, maxd:4.0, mini:2, maxi:3, sumd:6.0, avgd:3.0, variance:1.0, stddev:1.0, missing:1, vals:1}] } } "
+ "'f1':{ buckets:[{val:B, count:3, h:2, u:2, mind:-9.0, maxd:11.0, mini:-5, maxi:7, sumd:-3.0, avgd:-1.0, variance:112.0, stddev:10.583005244258363, missing:0, vals:5}," +
+ " {val:A, count:2, h:2, u:2, mind:2.0, maxd:4.0, mini:2, maxi:3, sumd:6.0, avgd:3.0, variance:2.0, stddev:1.4142135623730951, missing:1, vals:1}] } } "
);
diff --git a/solr/solr-ref-guide/src/major-changes-in-solr-9.adoc b/solr/solr-ref-guide/src/major-changes-in-solr-9.adoc
index dab1ba6..f94763f 100644
--- a/solr/solr-ref-guide/src/major-changes-in-solr-9.adoc
+++ b/solr/solr-ref-guide/src/major-changes-in-solr-9.adoc
@@ -90,6 +90,9 @@ _(raw; not yet edited)_
This embedded zookeeper should not be used in production. If you rely
upon the previous behavior, then you can change the clientPortAddress
in solr/server/solr/zoo.cfg (Robert Muir)
+
+* SOLR-11725: JSON aggregations uses corrected sample formula to compute standard deviation and variance.
+ The computation of stdDev and variance in JSON aggregation is same as StatsComponent. (hossman, Munendra S N, yonik)
=== Upgrade Prerequisites in Solr 9