You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@druid.apache.org by gi...@apache.org on 2022/10/28 15:31:59 UTC
[druid] branch master updated: MSQ: Use long instead of double for estimatedRetainedBytes. (#13272)
This is an automated email from the ASF dual-hosted git repository.
gian pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new 4f0145fb85 MSQ: Use long instead of double for estimatedRetainedBytes. (#13272)
4f0145fb85 is described below
commit 4f0145fb8506257f6531cce04b5dee8354a1a1a1
Author: Gian Merlino <gi...@gmail.com>
AuthorDate: Fri Oct 28 08:31:52 2022 -0700
MSQ: Use long instead of double for estimatedRetainedBytes. (#13272)
Fixes a problem where, due to the inexactness of floating-point math, we
would potentially drift while tracking retained byte counts and run into
assertion failures in assertRetainedByteCountsAreTrackedCorrectly.
---
.../druid/msq/statistics/ClusterByStatisticsCollectorImpl.java | 9 +++++----
.../apache/druid/msq/statistics/DelegateOrMinKeyCollector.java | 2 +-
.../org/apache/druid/msq/statistics/DistinctKeyCollector.java | 2 +-
.../main/java/org/apache/druid/msq/statistics/KeyCollector.java | 2 +-
.../apache/druid/msq/statistics/QuantilesSketchKeyCollector.java | 4 ++--
5 files changed, 10 insertions(+), 9 deletions(-)
diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/statistics/ClusterByStatisticsCollectorImpl.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/statistics/ClusterByStatisticsCollectorImpl.java
index 9e033c8749..13ab5dc01c 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/statistics/ClusterByStatisticsCollectorImpl.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/statistics/ClusterByStatisticsCollectorImpl.java
@@ -58,7 +58,7 @@ public class ClusterByStatisticsCollectorImpl implements ClusterByStatisticsColl
private final int maxRetainedBytes;
private final int maxBuckets;
- private double totalRetainedBytes;
+ private long totalRetainedBytes;
private ClusterByStatisticsCollectorImpl(
final ClusterBy clusterBy,
@@ -369,8 +369,8 @@ public class ClusterByStatisticsCollectorImpl implements ClusterByStatisticsColl
*/
private void downSample()
{
- double newTotalRetainedBytes = totalRetainedBytes;
- final double targetTotalRetainedBytes = totalRetainedBytes / 2;
+ long newTotalRetainedBytes = totalRetainedBytes;
+ final long targetTotalRetainedBytes = totalRetainedBytes / 2;
final List<BucketHolder> sortedHolders = new ArrayList<>(buckets.size());
@@ -384,7 +384,8 @@ public class ClusterByStatisticsCollectorImpl implements ClusterByStatisticsColl
// Downsample least-dense buckets first. (They're less likely to need high resolution.)
sortedHolders.sort(
Comparator.comparing((BucketHolder holder) ->
- (double) holder.keyCollector.estimatedTotalWeight() / holder.keyCollector.estimatedRetainedKeys())
+ (double) holder.keyCollector.estimatedTotalWeight()
+ / holder.keyCollector.estimatedRetainedKeys())
);
int i = 0;
diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/statistics/DelegateOrMinKeyCollector.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/statistics/DelegateOrMinKeyCollector.java
index 179c2bc3ae..e2e2282a6b 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/statistics/DelegateOrMinKeyCollector.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/statistics/DelegateOrMinKeyCollector.java
@@ -128,7 +128,7 @@ public class DelegateOrMinKeyCollector<TDelegate extends KeyCollector<TDelegate>
}
@Override
- public double estimatedRetainedBytes()
+ public long estimatedRetainedBytes()
{
if (delegate != null) {
return delegate.estimatedRetainedBytes();
diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/statistics/DistinctKeyCollector.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/statistics/DistinctKeyCollector.java
index 6868597667..9a1716a9fb 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/statistics/DistinctKeyCollector.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/statistics/DistinctKeyCollector.java
@@ -172,7 +172,7 @@ public class DistinctKeyCollector implements KeyCollector<DistinctKeyCollector>
}
@Override
- public double estimatedRetainedBytes()
+ public long estimatedRetainedBytes()
{
return retainedBytes;
}
diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/statistics/KeyCollector.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/statistics/KeyCollector.java
index 48287e74be..bf059f1996 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/statistics/KeyCollector.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/statistics/KeyCollector.java
@@ -57,7 +57,7 @@ public interface KeyCollector<CollectorType extends KeyCollector<CollectorType>>
* Returns an estimate of the number of bytes currently retained by this collector. This may change over time as
* more keys are added.
*/
- double estimatedRetainedBytes();
+ long estimatedRetainedBytes();
/**
* Downsample this collector, dropping about half of the keys that are currently retained. Returns true if
diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/statistics/QuantilesSketchKeyCollector.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/statistics/QuantilesSketchKeyCollector.java
index 950f9419af..1620324735 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/statistics/QuantilesSketchKeyCollector.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/statistics/QuantilesSketchKeyCollector.java
@@ -102,9 +102,9 @@ public class QuantilesSketchKeyCollector implements KeyCollector<QuantilesSketch
}
@Override
- public double estimatedRetainedBytes()
+ public long estimatedRetainedBytes()
{
- return averageKeyLength * estimatedRetainedKeys();
+ return Math.round(averageKeyLength * estimatedRetainedKeys());
}
@Override
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org