You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2016/04/28 12:45:43 UTC
[2/5] kylin git commit: KYLIN-1323 update IT
KYLIN-1323 update IT
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/bd54ef53
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/bd54ef53
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/bd54ef53
Branch: refs/heads/master
Commit: bd54ef530cc7d1fe494c5711cfa485440bb228fc
Parents: fb5e281
Author: shaofengshi <sh...@apache.org>
Authored: Tue Apr 5 13:51:33 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Thu Apr 28 18:36:05 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/storage/hbase/steps/CreateHTableJob.java | 5 +++++
.../kylin/storage/hbase/steps/RangeKeyDistributionJob.java | 5 -----
.../kylin/storage/hbase/steps/RangeKeyDistributionMapper.java | 5 ++++-
3 files changed, 9 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/bd54ef53/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java
index 67a9c4f..204ab58 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java
@@ -174,6 +174,11 @@ public class CreateHTableJob extends AbstractHadoopJob {
Collections.sort(allCuboids);
int nRegion = Math.round((float) (totalSizeInM / (cut * 1024L)));
+ if (kylinConfig.isDevEnv()) {
+ // for regression test in sandbox
+ nRegion = Math.round((float) (totalSizeInM / cut));
+ }
+
nRegion = Math.max(kylinConfig.getHBaseRegionCountMin(), nRegion);
nRegion = Math.min(kylinConfig.getHBaseRegionCountMax(), nRegion);
http://git-wip-us.apache.org/repos/asf/kylin/blob/bd54ef53/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionJob.java
index 9018729..42e1e90 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionJob.java
@@ -114,11 +114,6 @@ public class RangeKeyDistributionJob extends AbstractHadoopJob {
// The partition file for hfile is sequenece file consists of ImmutableBytesWritable and NullWritable
TableMapReduceUtil.addDependencyJars(job.getConfiguration(), ImmutableBytesWritable.class, NullWritable.class);
- // Passed the sandbox property to mapper, to simulate large dataset
- if (System.getProperty("useSandbox") != null && System.getProperty("useSandbox").equals("true")) {
- job.getConfiguration().setBoolean("useSandbox", true);
- }
-
return waitForCompletion(job);
} catch (Exception e) {
printUsage(options);
http://git-wip-us.apache.org/repos/asf/kylin/blob/bd54ef53/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionMapper.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionMapper.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionMapper.java
index 6f2d2bc..08d97ae 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionMapper.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RangeKeyDistributionMapper.java
@@ -22,7 +22,9 @@ import java.io.IOException;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.engine.mr.KylinMapper;
+import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
/**
* @author ysong1
@@ -43,7 +45,8 @@ public class RangeKeyDistributionMapper extends KylinMapper<Text, Text, Text, Lo
@Override
protected void setup(Context context) throws IOException {
super.bindCurrentConfiguration(context.getConfiguration());
- if (context.getConfiguration().getBoolean("useSandbox", false)) {
+ KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata();
+ if (config.isDevEnv()) {
scaleFactorForSandbox = 1024L;
}
}