You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/12/07 04:29:54 UTC
[37/50] [abbrv] kylin git commit: KYLIN-2248 TopN merge further
optimization after KYLIN-1917
KYLIN-2248 TopN merge further optimization after KYLIN-1917
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/59a30f66
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/59a30f66
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/59a30f66
Branch: refs/heads/master-hbase1.x
Commit: 59a30f66d47cc1838e6852405699fd7957bfac29
Parents: af429e5
Author: shaofengshi <sh...@apache.org>
Authored: Sun Dec 4 09:39:45 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Dec 5 17:42:34 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/measure/topn/TopNCounter.java | 47 ++++++--------------
1 file changed, 13 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/59a30f66/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounter.java b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounter.java
index 968e694..caf7961 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounter.java
@@ -26,11 +26,9 @@ import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
-import java.util.Set;
-import com.google.common.collect.Maps;
import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
+import com.google.common.collect.Maps;
/**
* Modified from the StreamSummary.java in https://github.com/addthis/stream-lib
@@ -157,41 +155,22 @@ public class TopNCounter<T> implements Iterable<Counter<T>> {
* @return
*/
public TopNCounter<T> merge(TopNCounter<T> another) {
- double m1 = 0.0, m2 = 0.0;
- if (this.size() >= this.capacity) {
- m1 = this.counterList.getLast().count;
- }
-
- if (another.size() >= another.capacity) {
- m2 = another.counterList.getLast().count;
- }
-
- Set<T> duplicateItems = Sets.newHashSet();
- List<T> notDuplicateItems = Lists.newArrayList();
-
- for (Map.Entry<T, Counter<T>> entry : this.counterMap.entrySet()) {
- T item = entry.getKey();
- Counter<T> existing = another.counterMap.get(item);
- if (existing != null) {
- duplicateItems.add(item);
- } else {
- notDuplicateItems.add(item);
+ boolean thisFull = this.size() >= this.capacity;
+ boolean anotherFull = another.size() >= another.capacity;
+ double m1 = thisFull ? this.counterList.getLast().count : 0.0;
+ double m2 = anotherFull ? another.counterList.getLast().count : 0.0;
+
+ if (anotherFull == true) {
+ for (Counter<T> entry : this.counterMap.values()) {
+ entry.count += m2;
}
}
- for (T item : duplicateItems) {
- this.offer(item, another.counterMap.get(item).count);
- }
-
- for (T item : notDuplicateItems) {
- this.offer(item, m2);
- }
-
for (Map.Entry<T, Counter<T>> entry : another.counterMap.entrySet()) {
- T item = entry.getKey();
- if (duplicateItems.contains(item) == false) {
- double counter = entry.getValue().count;
- this.offer(item, counter + m1);
+ if (this.counterMap.containsKey(entry.getKey())) {
+ this.offer(entry.getValue().getItem(), (entry.getValue().count - m2));
+ } else {
+ this.offer(entry.getValue().getItem(), entry.getValue().count + m1);
}
}