You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by so...@apache.org on 2019/05/03 22:46:33 UTC
[drill] 07/08: DRILL-7228: Upgrade to a newer version of t-digest
to address inaccuracies in histogram buckets. closes #1774
This is an automated email from the ASF dual-hosted git repository.
sorabh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit e191c54cab9ef969e290dc2bf95e100cb4f5b1eb
Author: Aman Sinha <as...@maprtech.com>
AuthorDate: Tue Apr 30 12:03:53 2019 -0700
DRILL-7228: Upgrade to a newer version of t-digest to address inaccuracies in histogram buckets.
closes #1774
---
exec/java-exec/pom.xml | 5 +
.../drill/exec/expr/fn/impl/TDigestFunctions.java | 140 ++++++++++-----------
.../impl/statistics/TDigestMergedStatistic.java | 12 +-
.../planner/common/NumericEquiDepthHistogram.java | 4 +-
4 files changed, 83 insertions(+), 78 deletions(-)
diff --git a/exec/java-exec/pom.xml b/exec/java-exec/pom.xml
index 2fcd935..c35e935 100644
--- a/exec/java-exec/pom.xml
+++ b/exec/java-exec/pom.xml
@@ -344,6 +344,11 @@
<version>2.7.0</version>
</dependency>
<dependency>
+ <groupId>com.tdunning</groupId>
+ <artifactId>t-digest</artifactId>
+ <version>3.2</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<exclusions>
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/TDigestFunctions.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/TDigestFunctions.java
index 041543b..bfcf78d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/TDigestFunctions.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/TDigestFunctions.java
@@ -67,13 +67,13 @@ public class TDigestFunctions {
public void setup() {
work = new ObjectHolder();
compression.value = (int) options.getLong(org.apache.drill.exec.ExecConstants.TDIGEST_COMPRESSION);
- work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value);
+ work.obj = new com.tdunning.math.stats.MergingDigest(compression.value);
}
@Override
public void add() {
if (work.obj != null) {
- com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj;
+ com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest) work.obj;
tdigest.add(in.value);
}
}
@@ -81,7 +81,7 @@ public class TDigestFunctions {
@Override
public void output() {
if (work.obj != null) {
- com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj;
+ com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest) work.obj;
try {
if (tdigest.size() > 0) {
int size = tdigest.smallByteSize();
@@ -105,7 +105,7 @@ public class TDigestFunctions {
@Override
public void reset() {
- work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value);
+ work.obj = new com.tdunning.math.stats.MergingDigest(compression.value);
}
}
@@ -122,13 +122,13 @@ public class TDigestFunctions {
public void setup() {
work = new ObjectHolder();
compression.value = (int) options.getLong(org.apache.drill.exec.ExecConstants.TDIGEST_COMPRESSION);
- work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value);
+ work.obj = new com.tdunning.math.stats.MergingDigest(compression.value);
}
@Override
public void add() {
if (work.obj != null) {
- com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj;
+ com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest) work.obj;
if (in.isSet == 1) {
tdigest.add(in.value);
} else {
@@ -140,7 +140,7 @@ public class TDigestFunctions {
@Override
public void output() {
if (work.obj != null) {
- com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj;
+ com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest) work.obj;
try {
if (tdigest.size() > 0) {
int size = tdigest.smallByteSize();
@@ -164,7 +164,7 @@ public class TDigestFunctions {
@Override
public void reset() {
- work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value);
+ work.obj = new com.tdunning.math.stats.MergingDigest(compression.value);
}
}
@@ -181,13 +181,13 @@ public class TDigestFunctions {
public void setup() {
work = new ObjectHolder();
compression.value = (int) options.getLong(org.apache.drill.exec.ExecConstants.TDIGEST_COMPRESSION);
- work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value);
+ work.obj = new com.tdunning.math.stats.MergingDigest(compression.value);
}
@Override
public void add() {
if (work.obj != null) {
- com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj;
+ com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest) work.obj;
tdigest.add(in.value);
}
}
@@ -195,7 +195,7 @@ public class TDigestFunctions {
@Override
public void output() {
if (work.obj != null) {
- com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj;
+ com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest) work.obj;
try {
if (tdigest.size() > 0) {
int size = tdigest.smallByteSize();
@@ -219,7 +219,7 @@ public class TDigestFunctions {
@Override
public void reset() {
- work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value);
+ work.obj = new com.tdunning.math.stats.MergingDigest(compression.value);
}
}
@@ -236,13 +236,13 @@ public class TDigestFunctions {
public void setup() {
work = new ObjectHolder();
compression.value = (int) options.getLong(org.apache.drill.exec.ExecConstants.TDIGEST_COMPRESSION);
- work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value);
+ work.obj = new com.tdunning.math.stats.MergingDigest(compression.value);
}
@Override
public void add() {
if (work.obj != null) {
- com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj;
+ com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest) work.obj;
if (in.isSet == 1) {
tdigest.add(in.value);
} else {
@@ -254,7 +254,7 @@ public class TDigestFunctions {
@Override
public void output() {
if (work.obj != null) {
- com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj;
+ com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest) work.obj;
try {
if (tdigest.size() > 0) {
int size = tdigest.smallByteSize();
@@ -278,7 +278,7 @@ public class TDigestFunctions {
@Override
public void reset() {
- work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value);
+ work.obj = new com.tdunning.math.stats.MergingDigest(compression.value);
}
}
@@ -295,13 +295,13 @@ public class TDigestFunctions {
public void setup() {
work = new ObjectHolder();
compression.value = (int) options.getLong(org.apache.drill.exec.ExecConstants.TDIGEST_COMPRESSION);
- work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value);
+ work.obj = new com.tdunning.math.stats.MergingDigest(compression.value);
}
@Override
public void add() {
if (work.obj != null) {
- com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj;
+ com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest) work.obj;
tdigest.add(in.value);
}
}
@@ -309,7 +309,7 @@ public class TDigestFunctions {
@Override
public void output() {
if (work.obj != null) {
- com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj;
+ com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest) work.obj;
try {
if (tdigest.size() > 0) {
int size = tdigest.smallByteSize();
@@ -333,7 +333,7 @@ public class TDigestFunctions {
@Override
public void reset() {
- work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value);
+ work.obj = new com.tdunning.math.stats.MergingDigest(compression.value);
}
}
@@ -350,13 +350,13 @@ public class TDigestFunctions {
public void setup() {
work = new ObjectHolder();
compression.value = (int) options.getLong(org.apache.drill.exec.ExecConstants.TDIGEST_COMPRESSION);
- work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value);
+ work.obj = new com.tdunning.math.stats.MergingDigest(compression.value);
}
@Override
public void add() {
if (work.obj != null) {
- com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj;
+ com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest) work.obj;
if (in.isSet == 1) {
tdigest.add(in.value);
} else {
@@ -368,7 +368,7 @@ public class TDigestFunctions {
@Override
public void output() {
if (work.obj != null) {
- com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj;
+ com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest) work.obj;
try {
if (tdigest.size() > 0) {
int size = tdigest.smallByteSize();
@@ -392,7 +392,7 @@ public class TDigestFunctions {
@Override
public void reset() {
- work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value);
+ work.obj = new com.tdunning.math.stats.MergingDigest(compression.value);
}
}
@@ -409,13 +409,13 @@ public class TDigestFunctions {
public void setup() {
work = new ObjectHolder();
compression.value = (int) options.getLong(org.apache.drill.exec.ExecConstants.TDIGEST_COMPRESSION);
- work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value);
+ work.obj = new com.tdunning.math.stats.MergingDigest(compression.value);
}
@Override
public void add() {
if (work.obj != null) {
- com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj;
+ com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest) work.obj;
tdigest.add(in.value);
}
}
@@ -423,7 +423,7 @@ public class TDigestFunctions {
@Override
public void output() {
if (work.obj != null) {
- com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj;
+ com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest) work.obj;
try {
if (tdigest.size() > 0) {
int size = tdigest.smallByteSize();
@@ -447,7 +447,7 @@ public class TDigestFunctions {
@Override
public void reset() {
- work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value);
+ work.obj = new com.tdunning.math.stats.MergingDigest(compression.value);
}
}
@@ -464,13 +464,13 @@ public class TDigestFunctions {
public void setup() {
work = new ObjectHolder();
compression.value = (int) options.getLong(org.apache.drill.exec.ExecConstants.TDIGEST_COMPRESSION);
- work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value);
+ work.obj = new com.tdunning.math.stats.MergingDigest(compression.value);
}
@Override
public void add() {
if (work.obj != null) {
- com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj;
+ com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest) work.obj;
if (in.isSet == 1) {
tdigest.add(in.value);
} else {
@@ -482,7 +482,7 @@ public class TDigestFunctions {
@Override
public void output() {
if (work.obj != null) {
- com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj;
+ com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest) work.obj;
try {
if (tdigest.size() > 0) {
int size = tdigest.smallByteSize();
@@ -506,7 +506,7 @@ public class TDigestFunctions {
@Override
public void reset() {
- work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value);
+ work.obj = new com.tdunning.math.stats.MergingDigest(compression.value);
}
}
@@ -523,13 +523,13 @@ public class TDigestFunctions {
public void setup() {
work = new ObjectHolder();
compression.value = (int) options.getLong(org.apache.drill.exec.ExecConstants.TDIGEST_COMPRESSION);
- work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value);
+ work.obj = new com.tdunning.math.stats.MergingDigest(compression.value);
}
@Override
public void add() {
if (work.obj != null) {
- com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj;
+ com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest) work.obj;
tdigest.add(in.value);
}
}
@@ -537,7 +537,7 @@ public class TDigestFunctions {
@Override
public void output() {
if (work.obj != null) {
- com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj;
+ com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest) work.obj;
try {
if (tdigest.size() > 0) {
int size = tdigest.smallByteSize();
@@ -561,7 +561,7 @@ public class TDigestFunctions {
@Override
public void reset() {
- work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value);
+ work.obj = new com.tdunning.math.stats.MergingDigest(compression.value);
}
}
@@ -578,13 +578,13 @@ public class TDigestFunctions {
public void setup() {
work = new ObjectHolder();
compression.value = (int) options.getLong(org.apache.drill.exec.ExecConstants.TDIGEST_COMPRESSION);
- work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value);
+ work.obj = new com.tdunning.math.stats.MergingDigest(compression.value);
}
@Override
public void add() {
if (work.obj != null) {
- com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj;
+ com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest) work.obj;
if (in.isSet == 1) {
tdigest.add(in.value);
} else {
@@ -596,7 +596,7 @@ public class TDigestFunctions {
@Override
public void output() {
if (work.obj != null) {
- com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj;
+ com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest) work.obj;
try {
if (tdigest.size() > 0) {
int size = tdigest.smallByteSize();
@@ -620,7 +620,7 @@ public class TDigestFunctions {
@Override
public void reset() {
- work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value);
+ work.obj = new com.tdunning.math.stats.MergingDigest(compression.value);
}
}
@@ -637,13 +637,13 @@ public class TDigestFunctions {
public void setup() {
work = new ObjectHolder();
compression.value = (int) options.getLong(org.apache.drill.exec.ExecConstants.TDIGEST_COMPRESSION);
- work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value);
+ work.obj = new com.tdunning.math.stats.MergingDigest(compression.value);
}
@Override
public void add() {
if (work.obj != null) {
- com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj;
+ com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest) work.obj;
tdigest.add(in.value);
}
}
@@ -651,7 +651,7 @@ public class TDigestFunctions {
@Override
public void output() {
if (work.obj != null) {
- com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj;
+ com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest) work.obj;
try {
if (tdigest.size() > 0) {
int size = tdigest.smallByteSize();
@@ -675,7 +675,7 @@ public class TDigestFunctions {
@Override
public void reset() {
- work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value);
+ work.obj = new com.tdunning.math.stats.MergingDigest(compression.value);
}
}
@@ -692,13 +692,13 @@ public class TDigestFunctions {
public void setup() {
work = new ObjectHolder();
compression.value = (int) options.getLong(org.apache.drill.exec.ExecConstants.TDIGEST_COMPRESSION);
- work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value);
+ work.obj = new com.tdunning.math.stats.MergingDigest(compression.value);
}
@Override
public void add() {
if (work.obj != null) {
- com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj;
+ com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest) work.obj;
if (in.isSet == 1) {
tdigest.add(in.value);
} else {
@@ -710,7 +710,7 @@ public class TDigestFunctions {
@Override
public void output() {
if (work.obj != null) {
- com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj;
+ com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest) work.obj;
try {
if (tdigest.size() > 0) {
int size = tdigest.smallByteSize();
@@ -734,7 +734,7 @@ public class TDigestFunctions {
@Override
public void reset() {
- work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value);
+ work.obj = new com.tdunning.math.stats.MergingDigest(compression.value);
}
}
@@ -751,13 +751,13 @@ public class TDigestFunctions {
public void setup() {
work = new ObjectHolder();
compression.value = (int) options.getLong(org.apache.drill.exec.ExecConstants.TDIGEST_COMPRESSION);
- work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value);
+ work.obj = new com.tdunning.math.stats.MergingDigest(compression.value);
}
@Override
public void add() {
if (work.obj != null) {
- com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj;
+ com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest) work.obj;
tdigest.add(in.value);
}
}
@@ -765,7 +765,7 @@ public class TDigestFunctions {
@Override
public void output() {
if (work.obj != null) {
- com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj;
+ com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest) work.obj;
try {
if (tdigest.size() > 0) {
int size = tdigest.smallByteSize();
@@ -789,7 +789,7 @@ public class TDigestFunctions {
@Override
public void reset() {
- work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value);
+ work.obj = new com.tdunning.math.stats.MergingDigest(compression.value);
}
}
@@ -806,13 +806,13 @@ public class TDigestFunctions {
public void setup() {
work = new ObjectHolder();
compression.value = (int) options.getLong(org.apache.drill.exec.ExecConstants.TDIGEST_COMPRESSION);
- work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value);
+ work.obj = new com.tdunning.math.stats.MergingDigest(compression.value);
}
@Override
public void add() {
if (work.obj != null) {
- com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj;
+ com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest) work.obj;
if (in.isSet == 1) {
tdigest.add(in.value);
} else {
@@ -824,7 +824,7 @@ public class TDigestFunctions {
@Override
public void output() {
if (work.obj != null) {
- com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj;
+ com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest) work.obj;
try {
if (tdigest.size() > 0) {
int size = tdigest.smallByteSize();
@@ -848,7 +848,7 @@ public class TDigestFunctions {
@Override
public void reset() {
- work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value);
+ work.obj = new com.tdunning.math.stats.MergingDigest(compression.value);
}
}
@@ -865,13 +865,13 @@ public class TDigestFunctions {
public void setup() {
work = new ObjectHolder();
compression.value = (int) options.getLong(org.apache.drill.exec.ExecConstants.TDIGEST_COMPRESSION);
- work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value);
+ work.obj = new com.tdunning.math.stats.MergingDigest(compression.value);
}
@Override
public void add() {
if (work.obj != null) {
- com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj;
+ com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest) work.obj;
tdigest.add(in.value);
}
}
@@ -879,7 +879,7 @@ public class TDigestFunctions {
@Override
public void output() {
if (work.obj != null) {
- com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj;
+ com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest) work.obj;
try {
if (tdigest.size() > 0) {
int size = tdigest.smallByteSize();
@@ -903,7 +903,7 @@ public class TDigestFunctions {
@Override
public void reset() {
- work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value);
+ work.obj = new com.tdunning.math.stats.MergingDigest(compression.value);
}
}
@@ -920,13 +920,13 @@ public class TDigestFunctions {
public void setup() {
work = new ObjectHolder();
compression.value = (int) options.getLong(org.apache.drill.exec.ExecConstants.TDIGEST_COMPRESSION);
- work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value);
+ work.obj = new com.tdunning.math.stats.MergingDigest(compression.value);
}
@Override
public void add() {
if (work.obj != null) {
- com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj;
+ com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest) work.obj;
if (in.isSet == 1) {
tdigest.add(in.value);
} else {
@@ -938,7 +938,7 @@ public class TDigestFunctions {
@Override
public void output() {
if (work.obj != null) {
- com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj;
+ com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest) work.obj;
try {
if (tdigest.size() > 0) {
int size = tdigest.smallByteSize();
@@ -962,7 +962,7 @@ public class TDigestFunctions {
@Override
public void reset() {
- work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value);
+ work.obj = new com.tdunning.math.stats.MergingDigest(compression.value);
}
}
@@ -1097,18 +1097,18 @@ public class TDigestFunctions {
public void setup() {
work = new ObjectHolder();
compression.value = (int) options.getLong(org.apache.drill.exec.ExecConstants.TDIGEST_COMPRESSION);
- work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value);
+ work.obj = new com.tdunning.math.stats.MergingDigest(compression.value);
}
@Override
public void add() {
if (work.obj != null) {
- com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj;
+ com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest) work.obj;
try {
if (in.isSet != 0) {
byte[] buf = org.apache.drill.exec.expr.fn.impl.StringFunctionHelpers.toStringFromUTF8(in.start, in.end, in.buffer).getBytes();
- com.clearspring.analytics.stream.quantile.TDigest other =
- com.clearspring.analytics.stream.quantile.TDigest.fromBytes(java.nio.ByteBuffer.wrap(buf));
+ com.tdunning.math.stats.MergingDigest other =
+ com.tdunning.math.stats.MergingDigest.fromBytes(java.nio.ByteBuffer.wrap(buf));
tdigest.add(other);
}
} catch (Exception e) {
@@ -1120,7 +1120,7 @@ public class TDigestFunctions {
@Override
public void output() {
if (work.obj != null) {
- com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj;
+ com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest) work.obj;
try {
int size = tdigest.smallByteSize();
java.nio.ByteBuffer byteBuf = java.nio.ByteBuffer.allocate(size);
@@ -1140,7 +1140,7 @@ public class TDigestFunctions {
@Override
public void reset() {
- work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value);
+ work.obj = new com.tdunning.math.stats.MergingDigest(compression.value);
}
}
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/statistics/TDigestMergedStatistic.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/statistics/TDigestMergedStatistic.java
index cccbff6..dc84eba 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/statistics/TDigestMergedStatistic.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/statistics/TDigestMergedStatistic.java
@@ -20,7 +20,7 @@ package org.apache.drill.exec.physical.impl.statistics;
// Library implementing TDigest algorithm to derive approximate quantiles. Please refer to:
// 'Computing Extremely Accurate Quantiles using t-Digests' by Ted Dunning and Otmar Ertl
-import com.clearspring.analytics.stream.quantile.TDigest;
+import com.tdunning.math.stats.MergingDigest;
import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.server.options.OptionManager;
@@ -33,7 +33,7 @@ import java.util.Map;
import java.nio.ByteBuffer;
public class TDigestMergedStatistic extends AbstractMergedStatistic {
- private Map<String, TDigest> tdigestHolder;
+ private Map<String, MergingDigest> tdigestHolder;
private int compression;
public TDigestMergedStatistic() {
@@ -63,7 +63,7 @@ public class TDigestMergedStatistic extends AbstractMergedStatistic {
assert (input.getField().getType().getMinorType() == TypeProtos.MinorType.MAP);
for (ValueVector vv : input) {
String colName = vv.getField().getName();
- TDigest colTdigestHolder = null;
+ MergingDigest colTdigestHolder = null;
if (tdigestHolder.get(colName) != null) {
colTdigestHolder = tdigestHolder.get(colName);
}
@@ -71,7 +71,7 @@ public class TDigestMergedStatistic extends AbstractMergedStatistic {
NullableVarBinaryVector.Accessor accessor = tdigestVector.getAccessor();
if (!accessor.isNull(0)) {
- TDigest other = TDigest.fromBytes(ByteBuffer.wrap(accessor.get(0)));
+ MergingDigest other = MergingDigest.fromBytes(ByteBuffer.wrap(accessor.get(0)));
if (colTdigestHolder != null) {
colTdigestHolder.add(other);
tdigestHolder.put(colName, colTdigestHolder);
@@ -82,7 +82,7 @@ public class TDigestMergedStatistic extends AbstractMergedStatistic {
}
}
- public TDigest getStat(String colName) {
+ public MergingDigest getStat(String colName) {
if (state != State.COMPLETE) {
throw new IllegalStateException(String.format("Statistic `%s` has not completed merging statistics",
name));
@@ -98,7 +98,7 @@ public class TDigestMergedStatistic extends AbstractMergedStatistic {
assert (state == State.MERGE);
for (ValueVector outMapCol : output) {
String colName = outMapCol.getField().getName();
- TDigest colTdigestHolder = tdigestHolder.get(colName);
+ MergingDigest colTdigestHolder = tdigestHolder.get(colName);
NullableVarBinaryVector vv = (NullableVarBinaryVector) outMapCol;
vv.allocateNewSafe();
try {
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/NumericEquiDepthHistogram.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/NumericEquiDepthHistogram.java
index 9d5bf6f..0cda7c3 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/NumericEquiDepthHistogram.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/NumericEquiDepthHistogram.java
@@ -26,7 +26,7 @@ import java.util.List;
import org.apache.calcite.rex.RexCall;
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.rex.RexLiteral;
-import com.clearspring.analytics.stream.quantile.TDigest;
+import com.tdunning.math.stats.MergingDigest;
import org.apache.calcite.sql.SqlKind;
import org.apache.calcite.sql.SqlOperator;
import org.apache.drill.shaded.guava.com.google.common.base.Preconditions;
@@ -255,7 +255,7 @@ public class NumericEquiDepthHistogram implements Histogram {
public static NumericEquiDepthHistogram buildFromTDigest(final byte[] tdigest_array,
final int numBuckets,
final long nonNullCount) {
- TDigest tdigest = TDigest.fromBytes(java.nio.ByteBuffer.wrap(tdigest_array));
+ MergingDigest tdigest = MergingDigest.fromBytes(java.nio.ByteBuffer.wrap(tdigest_array));
NumericEquiDepthHistogram histogram = new NumericEquiDepthHistogram(numBuckets);