You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2015/10/15 09:26:50 UTC
incubator-kylin git commit: KYLIN-1068 small changes
Repository: incubator-kylin
Updated Branches:
refs/heads/KYLIN-1068 3b2a12faa -> 58dd59130
KYLIN-1068 small changes
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/58dd5913
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/58dd5913
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/58dd5913
Branch: refs/heads/KYLIN-1068
Commit: 58dd59130206d1d47f1cf650a0b491cc328a930e
Parents: 3b2a12f
Author: shaofengshi <sh...@apache.org>
Authored: Thu Oct 15 15:26:24 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Thu Oct 15 15:26:24 2015 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/common/topn/TopNCounter.java | 7 +------
examples/test_case_data/sandbox/kylin.properties | 7 ++-----
.../kylin/storage/hbase/steps/HBaseMROutput2Transition.java | 5 +++--
3 files changed, 6 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/58dd5913/core-common/src/main/java/org/apache/kylin/common/topn/TopNCounter.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/topn/TopNCounter.java b/core-common/src/main/java/org/apache/kylin/common/topn/TopNCounter.java
index 6093a43..593058b 100644
--- a/core-common/src/main/java/org/apache/kylin/common/topn/TopNCounter.java
+++ b/core-common/src/main/java/org/apache/kylin/common/topn/TopNCounter.java
@@ -102,7 +102,6 @@ public class TopNCounter<T> implements ITopK<T>, Iterable<Counter<T>> {
droppedItem = counter.item;
counterMap.remove(droppedItem);
counter.item = item;
-// counter.error = counter.count;
counter.count = 0.0;
}
counterMap.put(item, counterNode);
@@ -217,27 +216,23 @@ public class TopNCounter<T> implements ITopK<T>, Iterable<Counter<T>> {
if (another.size() >= another.capacity) {
m2 = another.counterList.tail().getValue().count;
}
-
+
for (Map.Entry<T, ListNode2<Counter<T>>> entry : this.counterMap.entrySet()) {
T item = entry.getKey();
ListNode2<Counter<T>> existing = another.counterMap.get(item);
if (existing != null) {
this.offer(item, another.counterMap.get(item).getValue().count);
-// entry.getValue().getValue().error += another.counterMap.get(item).getValue().error;
another.counterMap.remove(item);
} else {
this.offer(item, m2);
-// this.counterMap.get(item).getValue().error = entry.getValue().getValue().error + m2;
}
}
for (Map.Entry<T, ListNode2<Counter<T>>> entry : another.counterMap.entrySet()) {
T item = entry.getKey();
double counter = entry.getValue().getValue().count;
-// double error = entry.getValue().getValue().error;
this.offer(item, counter + m1);
-// this.counterMap.get(item).getValue().error = error + m1;
}
return this;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/58dd5913/examples/test_case_data/sandbox/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/kylin.properties b/examples/test_case_data/sandbox/kylin.properties
index c13fcfb..35e2927 100644
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
@@ -36,11 +36,8 @@ kylin.job.concurrent.max.limit=10
# Max reducer number
kylin.job.mapreduce.max.reducer.number=5
-# Whether calculate cube in mem in each mapper;
-kylin.job.cubing.inMem=true
-
-#the percentage of the sampling, default 25%
-kylin.job.cubing.inMem.sampling.percent=25
+#the percentage of the sampling, default 100%
+kylin.job.cubing.inMem.sampling.percent=100
# The cut size for hbase region, in GB.
# E.g, for cube whose capacity be marked as "SMALL", split region per 10GB by default
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/58dd5913/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java
index 6ccb60b..b3bd36e 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java
@@ -366,8 +366,9 @@ public class HBaseMROutput2Transition implements IMROutput2 {
Configuration conf = HBaseConfiguration.create(job.getConfiguration());
HTable htable = new HTable(conf, htableName);
- int regions = htable.getStartKeys().length + 1;
- int reducerNum = regions * 10;
+ int regions = htable.getStartKeys().length;
+ int reducerNum = regions * 3;
+ reducerNum = Math.max(1, reducerNum);
KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
reducerNum = Math.min(kylinConfig.getHadoopJobMaxReducerNumber(), reducerNum);