You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2018/08/02 01:19:43 UTC
[kylin] branch master updated: KYLIN-3472 Optimize TopN merge in
for Spark engine
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/master by this push:
new 44926b7 KYLIN-3472 Optimize TopN merge in for Spark engine
44926b7 is described below
commit 44926b777f81d34d4e9da25e900f352be313dd58
Author: shaofengshi <sh...@apache.org>
AuthorDate: Mon Jul 30 18:48:25 2018 +0800
KYLIN-3472 Optimize TopN merge in for Spark engine
---
.../org/apache/kylin/measure/bitmap/BitmapAggregator.java | 10 ++--------
.../apache/kylin/measure/dim/DimCountDistinctAggregator.java | 5 ++---
.../java/org/apache/kylin/measure/hllc/HLLCAggregator.java | 5 ++---
.../kylin/measure/percentile/PercentileAggregator.java | 5 ++---
.../java/org/apache/kylin/measure/raw/RawAggregator.java | 6 ++----
.../java/org/apache/kylin/measure/topn/TopNAggregator.java | 8 ++------
.../main/java/org/apache/kylin/measure/topn/TopNCounter.java | 12 +++++++++---
7 files changed, 21 insertions(+), 30 deletions(-)
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapAggregator.java
index d57af48..19fa49e 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapAggregator.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapAggregator.java
@@ -45,14 +45,8 @@ public class BitmapAggregator extends MeasureAggregator<BitmapCounter> {
@Override
public BitmapCounter aggregate(BitmapCounter value1, BitmapCounter value2) {
- BitmapCounter merged = bitmapFactory.newBitmap();
- if (value1 != null) {
- merged.orWith(value1);
- }
- if (value2 != null) {
- merged.orWith(value2);
- }
- return merged;
+ value1.orWith(value2);
+ return value1;
}
@Override
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/dim/DimCountDistinctAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/dim/DimCountDistinctAggregator.java
index fc43b2e..22daf3a 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/dim/DimCountDistinctAggregator.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/dim/DimCountDistinctAggregator.java
@@ -45,9 +45,8 @@ public class DimCountDistinctAggregator extends MeasureAggregator<DimCountDistin
@Override
public DimCountDistinctCounter aggregate(DimCountDistinctCounter value1, DimCountDistinctCounter value2) {
- DimCountDistinctCounter result = new DimCountDistinctCounter(value1);
- result.addAll(value2);
- return result;
+ value1.addAll(value2);
+ return value1;
}
@Override
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCAggregator.java
index 7d5b216..4e09265 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCAggregator.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCAggregator.java
@@ -47,9 +47,8 @@ public class HLLCAggregator extends MeasureAggregator<HLLCounter> {
@Override
public HLLCounter aggregate(HLLCounter value1, HLLCounter value2) {
- HLLCounter result = new HLLCounter(value1);
- result.merge(value2);
- return result;
+ value1.merge(value2);
+ return value1;
}
@Override
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/percentile/PercentileAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/percentile/PercentileAggregator.java
index d6b93eb..ef8896a 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/percentile/PercentileAggregator.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/percentile/PercentileAggregator.java
@@ -43,9 +43,8 @@ public class PercentileAggregator extends MeasureAggregator<PercentileCounter> {
@Override
public PercentileCounter aggregate(PercentileCounter value1, PercentileCounter value2) {
- PercentileCounter merged = new PercentileCounter(value1);
- merged.merge(value2);
- return merged;
+ value1.merge(value2);
+ return value1;
}
@Override
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawAggregator.java
index c3ecc74..2ee36e3 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawAggregator.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawAggregator.java
@@ -54,10 +54,8 @@ public class RawAggregator extends MeasureAggregator<List<ByteArray>> {
return value1;
}
- List<ByteArray> result = new ArrayList<>(value1.size() + value2.size());
- result.addAll(value1);
- result.addAll(value2);
- return result;
+ value1.addAll(value2);
+ return value1;
}
@Override
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNAggregator.java
index bc2bc36..34ceb9c 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNAggregator.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNAggregator.java
@@ -46,12 +46,8 @@ public class TopNAggregator extends MeasureAggregator<TopNCounter<ByteArray>> {
@Override
public TopNCounter<ByteArray> aggregate(TopNCounter<ByteArray> value1, TopNCounter<ByteArray> value2) {
- int thisCapacity = value1.getCapacity();
- TopNCounter<ByteArray> aggregated = new TopNCounter<>(thisCapacity * 2);
- aggregated.merge(value1);
- aggregated.merge(value2);
- aggregated.retain(thisCapacity);
- return aggregated;
+ value1.merge(value2);
+ return value1;
}
@Override
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounter.java b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounter.java
index 5e4b91e..932248d 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounter.java
@@ -167,12 +167,18 @@ public class TopNCounter<T> implements Iterable<Counter<T>>, java.io.Serializabl
}
for (Map.Entry<T, Counter<T>> entry : another.counterMap.entrySet()) {
- if (this.counterMap.containsKey(entry.getKey())) {
- this.offer(entry.getValue().getItem(), (entry.getValue().count - m2));
+ Counter<T> counter = this.counterMap.get(entry.getKey());
+ if (counter != null) {
+ // this.offer(entry.getValue().getItem(), (entry.getValue().count - m2));
+ counter.setCount(counter.getCount() + (entry.getValue().count - m2));
} else {
- this.offer(entry.getValue().getItem(), entry.getValue().count + m1);
+ // this.offer(entry.getValue().getItem(), entry.getValue().count + m1);
+ counter = new Counter<T>(entry.getValue().getItem(), entry.getValue().count + m1);
+ this.counterMap.put(entry.getValue().getItem(), counter);
+ this.counterList.add(counter);
}
}
+ this.ordered = false;
this.sortAndRetain();
return this;