You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2016/04/05 07:51:32 UTC
kylin git commit: KYLIN-1323 update IT
Repository: kylin
Updated Branches:
refs/heads/KYLIN-1323 c766b300e -> 38c262f03
KYLIN-1323 update IT
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/38c262f0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/38c262f0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/38c262f0
Branch: refs/heads/KYLIN-1323
Commit: 38c262f033df271a97aebe17350806bb44660d5b
Parents: c766b30
Author: shaofengshi <sh...@apache.org>
Authored: Tue Apr 5 13:51:33 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Tue Apr 5 13:51:33 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/storage/hbase/steps/CreateHTableJob.java | 5 +++++
.../kylin/storage/hbase/steps/RangeKeyDistributionJob.java | 5 -----
.../kylin/storage/hbase/steps/RangeKeyDistributionMapper.java | 5 ++++-
3 files changed, 9 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/38c262f0/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java
index 848daca..bb7e53c 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java
@@ -176,6 +176,11 @@ public class CreateHTableJob extends AbstractHadoopJob {
Collections.sort(allCuboids);
int nRegion = Math.round((float) (totalSizeInM / (cut * 1024L)));
+ if (kylinConfig.isDevEnv()) {
+ // for regression test in sandbox
+ nRegion = Math.round((float) (totalSizeInM / cut));
+ }
+
nRegion = Math.max(kylinConfig.getHBaseRegionCountMin(), nRegion);
nRegion = Math.min(kylinConfig.getHBaseRegionCountMax(), nRegion);
http://git-wip-us.apache.org/repos/asf/kylin/blob/38c262f0/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionJob.java
index 4115b3d..87a0e10 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionJob.java
@@ -115,11 +115,6 @@ public class RangeKeyDistributionJob extends AbstractHadoopJob {
// The partition file for hfile is sequenece file consists of ImmutableBytesWritable and NullWritable
TableMapReduceUtil.addDependencyJars(job.getConfiguration(), ImmutableBytesWritable.class, NullWritable.class);
- // Passed the sandbox property to mapper, to simulate large dataset
- if (System.getProperty("useSandbox") != null && System.getProperty("useSandbox").equals("true")) {
- job.getConfiguration().setBoolean("useSandbox", true);
- }
-
return waitForCompletion(job);
} catch (Exception e) {
printUsage(options);
http://git-wip-us.apache.org/repos/asf/kylin/blob/38c262f0/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionMapper.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionMapper.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionMapper.java
index 6f2d2bc..08d97ae 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionMapper.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionMapper.java
@@ -22,7 +22,9 @@ import java.io.IOException;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.engine.mr.KylinMapper;
+import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
/**
* @author ysong1
@@ -43,7 +45,8 @@ public class RangeKeyDistributionMapper extends KylinMapper<Text, Text, Text, Lo
@Override
protected void setup(Context context) throws IOException {
super.bindCurrentConfiguration(context.getConfiguration());
- if (context.getConfiguration().getBoolean("useSandbox", false)) {
+ KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata();
+ if (config.isDevEnv()) {
scaleFactorForSandbox = 1024L;
}
}