You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2016/06/24 07:04:45 UTC
[07/50] kylin git commit: minor, adjust config for CI sandbox,
to avoid too many reducers in CreateHFile step
minor, adjust config for CI sandbox, to avoid too many reducers in CreateHFile step
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/74ce81c6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/74ce81c6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/74ce81c6
Branch: refs/heads/stream_m1
Commit: 74ce81c629bc3f6e6e0487ad0a1353479eb4fb2e
Parents: baac10b
Author: Li Yang <li...@apache.org>
Authored: Fri Jun 17 10:50:25 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jun 17 10:50:25 2016 +0800
----------------------------------------------------------------------
examples/test_case_data/sandbox/kylin.properties | 6 ++++--
.../org/apache/kylin/storage/hbase/steps/CreateHTableJob.java | 5 -----
2 files changed, 4 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/74ce81c6/examples/test_case_data/sandbox/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/kylin.properties b/examples/test_case_data/sandbox/kylin.properties
index 3751fe2..410bd74 100644
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
@@ -64,12 +64,14 @@ kylin.job.concurrent.max.limit=10
# Max reducer number
kylin.job.mapreduce.max.reducer.number=5
-#the percentage of the sampling, default 100%
+# The percentage of the sampling, default 100%
kylin.job.cubing.inmem.sampling.percent=100
# The cut size for hbase region, in GB.
# E.g, for cube whose capacity be marked as "SMALL", split region per 10GB by default
-kylin.hbase.region.cut=5
+kylin.hbase.region.cut=0.1
+kylin.hbase.region.count.max=5
+
# The hfile size of GB, smaller hfile leading to the converting hfile MR has more reducers and be faster
# set 0 to disable this optimization
kylin.hbase.hfile.size.gb=2
http://git-wip-us.apache.org/repos/asf/kylin/blob/74ce81c6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java
index c5276a3..2543e9b 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java
@@ -174,11 +174,6 @@ public class CreateHTableJob extends AbstractHadoopJob {
Collections.sort(allCuboids);
int nRegion = Math.round((float) (totalSizeInM / (cut * 1024L)));
- if (kylinConfig.isDevEnv()) {
- // for regression test in sandbox
- nRegion = Math.round((float) (totalSizeInM / cut));
- }
-
nRegion = Math.max(kylinConfig.getHBaseRegionCountMin(), nRegion);
nRegion = Math.min(kylinConfig.getHBaseRegionCountMax(), nRegion);