You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/02/09 02:31:29 UTC
[16/39] kylin git commit: KYLIN-2424 Optimize the integration test's
performance
KYLIN-2424 Optimize the integration test's performance
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5d83c80f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5d83c80f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5d83c80f
Branch: refs/heads/master-hbase0.98
Commit: 5d83c80fbc0d6c3db434368cc8c29d786eebcd94
Parents: 6f0bc1c
Author: shaofengshi <sh...@apache.org>
Authored: Sat Feb 4 21:37:05 2017 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Sat Feb 4 21:37:05 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/metadata/model/ColumnDesc.java | 8 +++++++
.../spark/SparkBatchCubingJobBuilder2.java | 2 +-
.../table/DEFAULT.STREAMING_TABLE.json | 9 +++++---
.../localmeta/table/DEFAULT.TEST_ACCOUNT.json | 6 +++--
.../table/DEFAULT.TEST_CATEGORY_GROUPINGS.json | 9 +++++---
.../localmeta/table/DEFAULT.TEST_COUNTRY.json | 6 +++--
.../table/DEFAULT.TEST_KYLIN_FACT.json | 21 ++++++++++++------
.../localmeta/table/DEFAULT.TEST_ORDER.json | 6 +++--
.../localmeta/table/EDW.TEST_CAL_DT.json | 15 ++++++++-----
.../table/EDW.TEST_SELLER_TYPE_DIM.json | 6 +++--
.../table/EDW.TEST_SELLER_TYPE_DIM_TABLE.json | 6 +++--
.../localmeta/table/EDW.TEST_SITES.json | 9 +++++---
.../java/org/apache/kylin/query/H2Database.java | 23 ++++++++++++++++++++
.../org/apache/kylin/query/KylinTestBase.java | 4 +++-
14 files changed, 97 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/5d83c80f/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
index 7105ede..e5b51e4 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
@@ -49,6 +49,10 @@ public class ColumnDesc implements Serializable {
@JsonInclude(JsonInclude.Include.NON_NULL)
private String dataGen;
+ @JsonProperty("index")
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ private String index;
+
// parsed from data type
private DataType type;
private DataType upgradedType;
@@ -157,6 +161,10 @@ public class ColumnDesc implements Serializable {
return dataGen;
}
+ public String getIndex() {
+ return index;
+ }
+
public void init(TableDesc table) {
this.table = table;
http://git-wip-us.apache.org/repos/asf/kylin/blob/5d83c80f/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
index 76b73b6..327d215 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
@@ -77,7 +77,7 @@ public class SparkBatchCubingJobBuilder2 extends BatchCubingJobBuilder2 {
try {
return ClassUtil.findContainingJar(Class.forName(className));
} catch (ClassNotFoundException e) {
- logger.error("failed to locate jar for class " + className, e);
+ logger.warn("failed to locate jar for class " + className + ", ignore it", e);
}
return "";
http://git-wip-us.apache.org/repos/asf/kylin/blob/5d83c80f/examples/test_case_data/localmeta/table/DEFAULT.STREAMING_TABLE.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/DEFAULT.STREAMING_TABLE.json b/examples/test_case_data/localmeta/table/DEFAULT.STREAMING_TABLE.json
index f28683f..d67cbe5 100644
--- a/examples/test_case_data/localmeta/table/DEFAULT.STREAMING_TABLE.json
+++ b/examples/test_case_data/localmeta/table/DEFAULT.STREAMING_TABLE.json
@@ -5,17 +5,20 @@
{
"id": "1",
"name": "minute_start",
- "datatype": "timestamp"
+ "datatype": "timestamp",
+ "index": "T"
},
{
"id": "2",
"name": "hour_start",
- "datatype": "timestamp"
+ "datatype": "timestamp",
+ "index": "T"
},
{
"id": "3",
"name": "day_start",
- "datatype": "date"
+ "datatype": "date",
+ "index": "T"
},
{
"id": "4",
http://git-wip-us.apache.org/repos/asf/kylin/blob/5d83c80f/examples/test_case_data/localmeta/table/DEFAULT.TEST_ACCOUNT.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/DEFAULT.TEST_ACCOUNT.json b/examples/test_case_data/localmeta/table/DEFAULT.TEST_ACCOUNT.json
index 89e265b..5945713 100644
--- a/examples/test_case_data/localmeta/table/DEFAULT.TEST_ACCOUNT.json
+++ b/examples/test_case_data/localmeta/table/DEFAULT.TEST_ACCOUNT.json
@@ -8,7 +8,8 @@
"id" : "1",
"name" : "ACCOUNT_ID",
"datatype" : "bigint",
- "data_gen" : "ID|10000000"
+ "data_gen" : "ID|10000000",
+ "index": "T"
}, {
"id" : "2",
"name" : "ACCOUNT_BUYER_LEVEL",
@@ -23,7 +24,8 @@
"id" : "4",
"name" : "ACCOUNT_COUNTRY",
"datatype" : "string",
- "data_gen" : "CN|FR|GB|GE|JP|IT|RU|US"
+ "data_gen" : "CN|FR|GB|GE|JP|IT|RU|US",
+ "index": "T"
}, {
"id" : "5",
"name" : "ACCOUNT_CONTACT",
http://git-wip-us.apache.org/repos/asf/kylin/blob/5d83c80f/examples/test_case_data/localmeta/table/DEFAULT.TEST_CATEGORY_GROUPINGS.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/DEFAULT.TEST_CATEGORY_GROUPINGS.json b/examples/test_case_data/localmeta/table/DEFAULT.TEST_CATEGORY_GROUPINGS.json
index d7ab580..adaf987 100644
--- a/examples/test_case_data/localmeta/table/DEFAULT.TEST_CATEGORY_GROUPINGS.json
+++ b/examples/test_case_data/localmeta/table/DEFAULT.TEST_CATEGORY_GROUPINGS.json
@@ -5,15 +5,18 @@
"columns" : [ {
"id" : "1",
"name" : "LEAF_CATEG_ID",
- "datatype" : "bigint"
+ "datatype" : "bigint",
+ "index": "T"
}, {
"id" : "2",
"name" : "LEAF_CATEG_NAME",
- "datatype" : "string"
+ "datatype" : "string",
+ "index": "T"
}, {
"id" : "3",
"name" : "SITE_ID",
- "datatype" : "int"
+ "datatype" : "int",
+ "index": "T"
}, {
"id" : "4",
"name" : "CATEG_BUSN_MGR",
http://git-wip-us.apache.org/repos/asf/kylin/blob/5d83c80f/examples/test_case_data/localmeta/table/DEFAULT.TEST_COUNTRY.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/DEFAULT.TEST_COUNTRY.json b/examples/test_case_data/localmeta/table/DEFAULT.TEST_COUNTRY.json
index 202182b..870cb2d 100644
--- a/examples/test_case_data/localmeta/table/DEFAULT.TEST_COUNTRY.json
+++ b/examples/test_case_data/localmeta/table/DEFAULT.TEST_COUNTRY.json
@@ -5,7 +5,8 @@
"columns" : [ {
"id" : "1",
"name" : "COUNTRY",
- "datatype" : "string"
+ "datatype" : "string",
+ "index": "T"
}, {
"id" : "2",
"name" : "LATITUDE",
@@ -17,7 +18,8 @@
}, {
"id" : "4",
"name" : "NAME",
- "datatype" : "string"
+ "datatype" : "string",
+ "index": "T"
} ],
"database" : "DEFAULT",
"last_modified" : 0
http://git-wip-us.apache.org/repos/asf/kylin/blob/5d83c80f/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json b/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
index 2b33e66..e82d535 100644
--- a/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
+++ b/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
@@ -10,36 +10,43 @@
}, {
"id" : "2",
"name" : "ORDER_ID",
- "datatype" : "bigint"
+ "datatype" : "bigint",
+ "index": "T"
}, {
"id" : "3",
"name" : "CAL_DT",
"datatype" : "date",
- "data_gen" : "FK,order"
+ "data_gen" : "FK,order",
+ "index": "T"
}, {
"id" : "4",
"name" : "LSTG_FORMAT_NAME",
"datatype" : "string",
- "data_gen" : "FP-GTC|FP-non GTC|ABIN|Auction|Others"
+ "data_gen" : "FP-GTC|FP-non GTC|ABIN|Auction|Others",
+ "index": "T"
}, {
"id" : "5",
"name" : "LEAF_CATEG_ID",
"datatype" : "bigint",
- "data_gen" : "FK,null,nullstr=0"
+ "data_gen" : "FK,null,nullstr=0",
+ "index": "T"
}, {
"id" : "6",
"name" : "LSTG_SITE_ID",
- "datatype" : "int"
+ "datatype" : "int",
+ "index": "T"
}, {
"id" : "7",
"name" : "SLR_SEGMENT_CD",
"datatype" : "smallint",
- "data_gen" : "FK,pk=EDW.TEST_SELLER_TYPE_DIM_TABLE.SELLER_TYPE_CD"
+ "data_gen" : "FK,pk=EDW.TEST_SELLER_TYPE_DIM_TABLE.SELLER_TYPE_CD",
+ "index": "T"
}, {
"id" : "8",
"name" : "SELLER_ID",
"datatype" : "bigint",
- "data_gen" : "RAND||10000000|10001000"
+ "data_gen" : "RAND||10000000|10001000",
+ "index": "T"
}, {
"id" : "9",
"name" : "PRICE",
http://git-wip-us.apache.org/repos/asf/kylin/blob/5d83c80f/examples/test_case_data/localmeta/table/DEFAULT.TEST_ORDER.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/DEFAULT.TEST_ORDER.json b/examples/test_case_data/localmeta/table/DEFAULT.TEST_ORDER.json
index f157846..7232492 100644
--- a/examples/test_case_data/localmeta/table/DEFAULT.TEST_ORDER.json
+++ b/examples/test_case_data/localmeta/table/DEFAULT.TEST_ORDER.json
@@ -6,12 +6,14 @@
"id" : "1",
"name" : "ORDER_ID",
"datatype" : "bigint",
- "data_gen" : "ID"
+ "data_gen" : "ID",
+ "index": "T"
}, {
"id" : "2",
"name" : "BUYER_ID",
"datatype" : "bigint",
- "data_gen" : "RAND||10000500|10001500"
+ "data_gen" : "RAND||10000500|10001500",
+ "index": "T"
}, {
"id" : "3",
"name" : "TEST_DATE_ENC",
http://git-wip-us.apache.org/repos/asf/kylin/blob/5d83c80f/examples/test_case_data/localmeta/table/EDW.TEST_CAL_DT.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/EDW.TEST_CAL_DT.json b/examples/test_case_data/localmeta/table/EDW.TEST_CAL_DT.json
index 02299d9..1230649 100644
--- a/examples/test_case_data/localmeta/table/EDW.TEST_CAL_DT.json
+++ b/examples/test_case_data/localmeta/table/EDW.TEST_CAL_DT.json
@@ -5,23 +5,28 @@
"columns" : [ {
"id" : "1",
"name" : "CAL_DT",
- "datatype" : "date"
+ "datatype" : "date",
+ "index": "T"
}, {
"id" : "2",
"name" : "YEAR_BEG_DT",
- "datatype" : "date"
+ "datatype" : "date",
+ "index": "T"
}, {
"id" : "3",
"name" : "QTR_BEG_DT",
- "datatype" : "date"
+ "datatype" : "date",
+ "index": "T"
}, {
"id" : "4",
"name" : "MONTH_BEG_DT",
- "datatype" : "date"
+ "datatype" : "date",
+ "index": "T"
}, {
"id" : "5",
"name" : "WEEK_BEG_DT",
- "datatype" : "date"
+ "datatype" : "date",
+ "index": "T"
}, {
"id" : "6",
"name" : "AGE_FOR_YEAR_ID",
http://git-wip-us.apache.org/repos/asf/kylin/blob/5d83c80f/examples/test_case_data/localmeta/table/EDW.TEST_SELLER_TYPE_DIM.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/EDW.TEST_SELLER_TYPE_DIM.json b/examples/test_case_data/localmeta/table/EDW.TEST_SELLER_TYPE_DIM.json
index 2bfbb70..136ae57 100644
--- a/examples/test_case_data/localmeta/table/EDW.TEST_SELLER_TYPE_DIM.json
+++ b/examples/test_case_data/localmeta/table/EDW.TEST_SELLER_TYPE_DIM.json
@@ -4,7 +4,8 @@
"columns" : [ {
"id" : "1",
"name" : "SELLER_TYPE_CD",
- "datatype" : "smallint"
+ "datatype" : "smallint",
+ "index": "T"
}, {
"id" : "2",
"name" : "SELLER_TYPE_DESC",
@@ -16,7 +17,8 @@
}, {
"id" : "4",
"name" : "SELLER_GROUP_CD",
- "datatype" : "tinyint"
+ "datatype" : "tinyint",
+ "index": "T"
}, {
"id" : "5",
"name" : "SELLER_GROUP_DESC",
http://git-wip-us.apache.org/repos/asf/kylin/blob/5d83c80f/examples/test_case_data/localmeta/table/EDW.TEST_SELLER_TYPE_DIM_TABLE.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/EDW.TEST_SELLER_TYPE_DIM_TABLE.json b/examples/test_case_data/localmeta/table/EDW.TEST_SELLER_TYPE_DIM_TABLE.json
index 3c5749b..794810d 100644
--- a/examples/test_case_data/localmeta/table/EDW.TEST_SELLER_TYPE_DIM_TABLE.json
+++ b/examples/test_case_data/localmeta/table/EDW.TEST_SELLER_TYPE_DIM_TABLE.json
@@ -4,7 +4,8 @@
"columns" : [ {
"id" : "1",
"name" : "SELLER_TYPE_CD",
- "datatype" : "smallint"
+ "datatype" : "smallint",
+ "index": "T"
}, {
"id" : "2",
"name" : "SELLER_TYPE_DESC",
@@ -16,7 +17,8 @@
}, {
"id" : "4",
"name" : "SELLER_GROUP_CD",
- "datatype" : "tinyint"
+ "datatype" : "tinyint",
+ "index": "T"
}, {
"id" : "5",
"name" : "SELLER_GROUP_DESC",
http://git-wip-us.apache.org/repos/asf/kylin/blob/5d83c80f/examples/test_case_data/localmeta/table/EDW.TEST_SITES.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/EDW.TEST_SITES.json b/examples/test_case_data/localmeta/table/EDW.TEST_SITES.json
index cc801af..4c2d727 100644
--- a/examples/test_case_data/localmeta/table/EDW.TEST_SITES.json
+++ b/examples/test_case_data/localmeta/table/EDW.TEST_SITES.json
@@ -5,11 +5,13 @@
"columns" : [ {
"id" : "1",
"name" : "SITE_ID",
- "datatype" : "int"
+ "datatype" : "int",
+ "index": "T"
}, {
"id" : "2",
"name" : "SITE_NAME",
- "datatype" : "string"
+ "datatype" : "string",
+ "index": "T"
}, {
"id" : "3",
"name" : "SITE_DOMAIN_CODE",
@@ -25,7 +27,8 @@
}, {
"id" : "6",
"name" : "SITE_CNTRY_ID",
- "datatype" : "int"
+ "datatype" : "int",
+ "index": "T"
}, {
"id" : "7",
"name" : "CRE_DATE",
http://git-wip-us.apache.org/repos/asf/kylin/blob/5d83c80f/kylin-it/src/test/java/org/apache/kylin/query/H2Database.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/H2Database.java b/kylin-it/src/test/java/org/apache/kylin/query/H2Database.java
index c449837..8b60c49 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/H2Database.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/H2Database.java
@@ -26,8 +26,10 @@ import java.sql.Connection;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
+import com.clearspring.analytics.util.Lists;
import org.apache.commons.io.IOUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.metadata.MetadataManager;
@@ -103,6 +105,11 @@ public class H2Database {
String sql = generateCreateH2TableSql(tableDesc, cvsFilePath);
stmt.executeUpdate(sql);
+ List<String> createIndexStatements = generateCreateH2IndexSql(tableDesc);
+ for (String indexSql : createIndexStatements) {
+ stmt.executeUpdate(indexSql);
+ }
+
if (tempFile != null)
tempFile.delete();
}
@@ -136,6 +143,22 @@ public class H2Database {
return ddl.toString();
}
+ private List<String> generateCreateH2IndexSql(TableDesc tableDesc) {
+ List<String> result = Lists.newArrayList();
+ int x = 0;
+ for (ColumnDesc col : tableDesc.getColumns()) {
+ if ("T".equalsIgnoreCase(col.getIndex())) {
+ StringBuilder ddl = new StringBuilder();
+ ddl.append("CREATE INDEX IDX_" + tableDesc.getName() + "_" + x + " ON " + tableDesc.getIdentity() + "(" + col.getName() + ")");
+ ddl.append("\n");
+ result.add(ddl.toString());
+ x++;
+ }
+ }
+
+ return result;
+ }
+
private static String getH2DataType(String javaDataType) {
String hiveDataType = javaToH2DataTypeMapping.get(javaDataType.toLowerCase());
if (hiveDataType == null) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/5d83c80f/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
index deb1104..1cc5c76 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
@@ -499,7 +499,9 @@ public class KylinTestBase {
// execute H2
printInfo("Query Result from H2 - " + queryName);
+ long currentTime = System.currentTimeMillis();
ITable h2Table = executeQuery(newH2Connection(), queryName, sql, needSort);
+ printInfo("H2 spent " + (System.currentTimeMillis() - currentTime) + " mili-seconds.");
try {
// compare the result
@@ -644,7 +646,7 @@ public class KylinTestBase {
cubeConnection = DriverManager.getConnection("jdbc:calcite:model=" + olapTmp.getAbsolutePath(), props);
//setup h2
- h2Connection = DriverManager.getConnection("jdbc:h2:mem:db" + (h2InstanceCount++), "sa", "");
+ h2Connection = DriverManager.getConnection("jdbc:h2:mem:db" + (h2InstanceCount++) + ";CACHE_SIZE=32072;DB_CLOSE_DELAY=-1", "sa", "");
// Load H2 Tables (inner join)
H2Database h2DB = new H2Database(h2Connection, config);
h2DB.loadAllTables();