You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/12/21 09:50:49 UTC
[2/2] kylin git commit: much slimer CI, ready to CI test
much slimer CI, ready to CI test
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/df27eb6b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/df27eb6b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/df27eb6b
Branch: refs/heads/KYLIN-2294
Commit: df27eb6b5426c19873d4e9ed87910cb57ce4426c
Parents: b8e03f2
Author: Li Yang <li...@apache.org>
Authored: Wed Dec 21 17:50:43 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Wed Dec 21 17:50:43 2016 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/job/DeployUtil.java | 4 +-
.../kylin/source/datagen/ColumnGenerator.java | 14 ++-
.../kylin/provision/BuildCubeWithEngine.java | 89 ++++----------------
3 files changed, 28 insertions(+), 79 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/df27eb6b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
----------------------------------------------------------------------
diff --git a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
index c8b0d43..e56c6bc 100644
--- a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
+++ b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
@@ -130,7 +130,7 @@ public class DeployUtil {
static final String[] TABLE_NAMES = new String[] { TABLE_CAL_DT, TABLE_CATEGORY_GROUPINGS, TABLE_KYLIN_FACT, TABLE_SELLER_TYPE_DIM_TABLE, TABLE_SITES };
- public static void prepareTestDataForNormalCubes(String cubeName) throws Exception {
+ public static void prepareTestDataForNormalCubes(String modelName) throws Exception {
boolean buildCubeUsingProvidedData = Boolean.parseBoolean(System.getProperty("buildCubeUsingProvidedData"));
if (!buildCubeUsingProvidedData) {
@@ -138,7 +138,7 @@ public class DeployUtil {
// data is generated according to cube descriptor and saved in resource store
MetadataManager mgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
- DataModelDesc model = mgr.getDataModelDesc("test_kylin_inner_join_model_desc");
+ DataModelDesc model = mgr.getDataModelDesc(modelName);
ModelDataGenerator gen = new ModelDataGenerator(model, 10000);
gen.generate();
} else {
http://git-wip-us.apache.org/repos/asf/kylin/blob/df27eb6b/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenerator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenerator.java b/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenerator.java
index 775f4fc..7c54714 100644
--- a/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenerator.java
+++ b/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenerator.java
@@ -250,8 +250,8 @@ public class ColumnGenerator {
public String next() {
if (values.isEmpty())
return null;
- else
- return values.get(rand.nextInt(values.size()));
+
+ return values.get(rand.nextInt(values.size()));
}
}
@@ -308,7 +308,15 @@ public class ColumnGenerator {
@Override
public String next() {
- return rand.nextDouble() < nullPct || !input.hasNext() ? nullStr : input.next();
+ String r = nullStr;
+ if (input.hasNext()) {
+ r = input.next();
+ }
+
+ if (rand.nextDouble() < nullPct) {
+ r = nullStr;
+ }
+ return r;
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/df27eb6b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
index bfbeb70..66a217a 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
@@ -161,7 +161,7 @@ public class BuildCubeWithEngine {
}
public void build() throws Exception {
- DeployUtil.prepareTestDataForNormalCubes("test_kylin_cube_with_slr_empty");
+ DeployUtil.prepareTestDataForNormalCubes("ci_left_join_model");
KylinConfig.getInstanceFromEnv().setHBaseHFileSizeGB(1.0f);
testInner();
testLeft();
@@ -184,12 +184,12 @@ public class BuildCubeWithEngine {
}
private void testInner() throws Exception {
- String[] testCase = new String[] { "testInnerJoinCubeWithoutSlr", "testInnerJoinCubeWithSlr" };
+ String[] testCase = new String[] { "testInnerJoinCube" };
runTestAndAssertSucceed(testCase);
}
private void testLeft() throws Exception {
- String[] testCase = new String[] { "testLeftJoinCubeWithSlr", "testLeftJoinCubeWithoutSlr" };
+ String[] testCase = new String[] { "testLeftJoinCube" };
runTestAndAssertSucceed(testCase);
}
@@ -243,47 +243,24 @@ public class BuildCubeWithEngine {
@SuppressWarnings("unused")
// called by reflection
- private boolean testInnerJoinCubeWithSlr() throws Exception {
- final String cubeName = "test_kylin_cube_with_slr_empty";
- clearSegment(cubeName);
- SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
- f.setTimeZone(TimeZone.getTimeZone("GMT"));
- long date1 = 0;
- long date2 = f.parse("2015-01-01").getTime();
- long date3 = f.parse("2022-01-01").getTime();
- List<String> result = Lists.newArrayList();
-
- if (fastBuildMode) {
- return buildSegment(cubeName, date1, date3);
- } else {
- if (buildSegment(cubeName, date1, date2) == true) {
- return buildSegment(cubeName, date2, date3);//empty segment
- }
- }
- return false;
- }
-
- @SuppressWarnings("unused")
- // called by reflection
- private boolean testLeftJoinCubeWithSlr() throws Exception {
- String cubeName = "test_kylin_cube_with_slr_left_join_empty";
+ private boolean testLeftJoinCube() throws Exception {
+ String cubeName = "ci_left_join_cube";
clearSegment(cubeName);
SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
f.setTimeZone(TimeZone.getTimeZone("GMT"));
- long date1 = cubeManager.getCube(cubeName).getDescriptor().getPartitionDateStart();
+ long date1 = 0;
long date2 = f.parse("2013-01-01").getTime();
long date3 = f.parse("2013-07-01").getTime();
long date4 = f.parse("2022-01-01").getTime();
- List<String> result = Lists.newArrayList();
if (fastBuildMode) {
return buildSegment(cubeName, date1, date4);
} else {
if (buildSegment(cubeName, date1, date2) == true) {
if (buildSegment(cubeName, date2, date3) == true) {
if (buildSegment(cubeName, date3, date4) == true) {
- return mergeSegment(cubeName, date1, date3);//don't merge all segments
+ return mergeSegment(cubeName, date1, date3); // don't merge all segments
}
}
}
@@ -293,61 +270,25 @@ public class BuildCubeWithEngine {
@SuppressWarnings("unused")
// called by reflection
- private boolean testInnerJoinCubeWithoutSlr() throws Exception {
+ private boolean testInnerJoinCube() throws Exception {
- final String cubeName = "test_kylin_cube_without_slr_empty";
+ String cubeName = "ci_inner_join_cube";
clearSegment(cubeName);
+
SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
f.setTimeZone(TimeZone.getTimeZone("GMT"));
long date1 = 0;
- long date2 = f.parse("2013-01-01").getTime();
- long date3 = f.parse("2013-07-01").getTime();
- long date4 = f.parse("2022-01-01").getTime();
- List<String> result = Lists.newArrayList();
+ long date2 = f.parse("2022-07-01").getTime();
+ long date3 = f.parse("2023-01-01").getTime();
if (fastBuildMode) {
- return buildSegment(cubeName, date1, date4);
- } else {
- if (buildSegment(cubeName, date1, date2) == true) {
- if (buildSegment(cubeName, date2, date3) == true) {
- if (buildSegment(cubeName, date3, date4) == true) {
- return mergeSegment(cubeName, date1, date3);//don't merge all segments
- }
- }
- }
- }
- return false;
-
- }
-
- @SuppressWarnings("unused")
- // called by reflection
- private boolean testLeftJoinCubeWithoutSlr() throws Exception {
- SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
- f.setTimeZone(TimeZone.getTimeZone("GMT"));
- List<String> result = Lists.newArrayList();
- final String cubeName = "test_kylin_cube_without_slr_left_join_empty";
- clearSegment(cubeName);
-
- long date1 = cubeManager.getCube(cubeName).getDescriptor().getPartitionDateStart();
- long date2 = f.parse("2012-06-01").getTime();
- long date3 = f.parse("2022-01-01").getTime();
- long date4 = f.parse("2023-01-01").getTime();
-
- if (fastBuildMode) {
- return buildSegment(cubeName, date1, date4);
+ return buildSegment(cubeName, date1, date3);
} else {
- if (buildSegment(cubeName, date1, date2) == true) {
- if (buildSegment(cubeName, date2, date3) == true) {
- if (buildSegment(cubeName, date3, date4) == true) { //empty segment
- return mergeSegment(cubeName, date1, date3);//don't merge all segments
- }
- }
+ if (buildSegment(cubeName, date1, date2) == true) { // all-in-one build
+ return buildSegment(cubeName, date2, date3); // empty segment
}
}
-
return false;
-
}
private void clearSegment(String cubeName) throws Exception {